kreuzberg 4.0.8 → 4.1.1

data/vendor/kreuzberg/src/ocr/processor/mod.rs

@@ -0,0 +1,265 @@
+//! OCR processor implementation using Tesseract.
+//!
+//! This module has been split into focused submodules for better organization:
+//! - `validation` - Image and configuration validation
+//! - `config` - Configuration hashing and Tesseract variables
+//! - `execution` - Core OCR execution logic
+
+mod config;
+mod execution;
+mod validation;
+
+use crate::ocr::cache::OcrCache;
+use crate::ocr::error::OcrError;
+use crate::ocr::types::{BatchItemResult, TesseractConfig};
+use crate::types::OcrExtractionResult;
+
+pub struct OcrProcessor {
+    cache: OcrCache,
+}
+
+impl OcrProcessor {
+    pub fn new(cache_dir: Option<std::path::PathBuf>) -> Result<Self, OcrError> {
+        let cache = OcrCache::new(cache_dir)?;
+        Ok(Self { cache })
+    }
+
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, image_bytes),
+        fields(
+            ocr.backend = "tesseract",
+            ocr.language = %config.language,
+            image.size_bytes = image_bytes.len(),
+        )
+    ))]
+    pub fn process_image(&self, image_bytes: &[u8], config: &TesseractConfig) -> Result<OcrExtractionResult, OcrError> {
+        execution::process_image_with_cache(image_bytes, config, &self.cache, None)
+    }
+
+    /// Process an image with OCR and respect the output format from ExtractionConfig.
+    ///
+    /// This variant allows specifying an output format (Plain, Markdown, Djot) which
+    /// affects how the OCR result's mime_type is set when markdown output is requested.
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, image_bytes),
+        fields(
+            ocr.backend = "tesseract",
+            ocr.language = %config.language,
+            image.size_bytes = image_bytes.len(),
+        )
+    ))]
+    pub fn process_image_with_format(
+        &self,
+        image_bytes: &[u8],
+        config: &TesseractConfig,
+        output_format: crate::core::config::OutputFormat,
+    ) -> Result<OcrExtractionResult, OcrError> {
+        execution::process_image_with_cache(image_bytes, config, &self.cache, Some(output_format))
+    }
+
+    pub fn clear_cache(&self) -> Result<(), OcrError> {
+        self.cache.clear()
+    }
+
+    pub fn get_cache_stats(&self) -> Result<super::cache::OcrCacheStats, OcrError> {
+        self.cache.get_stats()
+    }
+
+    pub fn process_file(&self, file_path: &str, config: &TesseractConfig) -> Result<OcrExtractionResult, OcrError> {
+        execution::process_file_with_cache(file_path, config, &self.cache, None)
+    }
+
+    /// Process a file with OCR and respect the output format from ExtractionConfig.
+    ///
+    /// This variant allows specifying an output format (Plain, Markdown, Djot) which
+    /// affects how the OCR result's mime_type is set when markdown output is requested.
+    pub fn process_file_with_format(
+        &self,
+        file_path: &str,
+        config: &TesseractConfig,
+        output_format: crate::core::config::OutputFormat,
+    ) -> Result<OcrExtractionResult, OcrError> {
+        execution::process_file_with_cache(file_path, config, &self.cache, Some(output_format))
+    }
+
+    /// Process multiple image files in parallel using Rayon.
+    ///
+    /// This method processes OCR operations in parallel across CPU cores for improved throughput.
+    /// Results are returned in the same order as the input file paths.
+    pub fn process_files_batch(&self, file_paths: Vec<String>, config: &TesseractConfig) -> Vec<BatchItemResult> {
+        execution::process_files_batch(file_paths, config, &self.cache)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    fn create_test_config() -> TesseractConfig {
+        TesseractConfig {
+            output_format: "text".to_string(),
+            enable_table_detection: false,
+            use_cache: false,
+            ..TesseractConfig::default()
+        }
+    }
+
+    #[test]
+    fn test_processor_creation() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf()));
+        assert!(processor.is_ok());
+    }
+
+    #[test]
+    fn test_processor_creation_default_cache_dir() {
+        let processor = OcrProcessor::new(None);
+        assert!(processor.is_ok());
+    }
+
+    #[test]
+    fn test_cache_operations() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+
+        assert!(processor.clear_cache().is_ok());
+
+        let stats = processor.get_cache_stats();
+        assert!(stats.is_ok());
+    }
+
+    #[test]
+    fn test_process_file_nonexistent() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let result = processor.process_file("/nonexistent/file.png", &config);
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("Failed to read file"));
+    }
+
+    #[test]
+    fn test_process_files_batch_empty() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let results = processor.process_files_batch(vec![], &config);
+        assert_eq!(results.len(), 0);
+    }
+
+    #[test]
+    fn test_process_image_invalid_image_data() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let invalid_data = vec![0, 1, 2, 3, 4];
+        let result = processor.process_image(&invalid_data, &config);
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_process_files_batch_single_file() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let results = processor.process_files_batch(vec!["/nonexistent.png".to_string()], &config);
+        assert_eq!(results.len(), 1);
+        assert!(!results[0].success);
+        assert!(results[0].error.is_some());
+        assert!(results[0].result.is_none());
+    }
+
+    #[test]
+    fn test_process_files_batch_multiple_files() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let file_paths = vec![
+            "/nonexistent1.png".to_string(),
+            "/nonexistent2.png".to_string(),
+            "/nonexistent3.png".to_string(),
+        ];
+
+        let results = processor.process_files_batch(file_paths, &config);
+        assert_eq!(results.len(), 3);
+
+        for result in &results {
+            assert!(!result.success);
+            assert!(result.error.is_some());
+            assert!(result.result.is_none());
+        }
+    }
+
+    #[test]
+    fn test_batch_item_result_structure() {
+        use std::collections::HashMap;
+
+        let success_result = BatchItemResult {
+            file_path: "test.png".to_string(),
+            success: true,
+            result: Some(OcrExtractionResult {
+                content: "test".to_string(),
+                mime_type: "text/plain".to_string(),
+                metadata: HashMap::new(),
+                tables: vec![],
+            }),
+            error: None,
+        };
+
+        assert!(success_result.success);
+        assert!(success_result.result.is_some());
+        assert!(success_result.error.is_none());
+
+        let error_result = BatchItemResult {
+            file_path: "error.png".to_string(),
+            success: false,
+            result: None,
+            error: Some("Test error".to_string()),
+        };
+
+        assert!(!error_result.success);
+        assert!(error_result.result.is_none());
+        assert!(error_result.error.is_some());
+    }
+
+    #[test]
+    fn test_process_files_batch_preserves_order() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+        let config = create_test_config();
+
+        let file_paths = vec![
+            "file1.png".to_string(),
+            "file2.png".to_string(),
+            "file3.png".to_string(),
+        ];
+
+        let results = processor.process_files_batch(file_paths.clone(), &config);
+
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0].file_path, "file1.png");
+        assert_eq!(results[1].file_path, "file2.png");
+        assert_eq!(results[2].file_path, "file3.png");
+    }
+
+    #[test]
+    fn test_process_image_with_cache_disabled() {
+        let temp_dir = tempdir().unwrap();
+        let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+
+        let mut config = create_test_config();
+        config.use_cache = false;
+
+        let invalid_data = vec![0, 1, 2, 3];
+        let result = processor.process_image(&invalid_data, &config);
+
+        assert!(result.is_err());
+    }
+}

data/vendor/kreuzberg/src/ocr/processor/validation.rs

@@ -0,0 +1,145 @@
+//! Image and configuration validation logic.
+//!
+//! This module handles validation of images, language files, and Tesseract configuration
+//! before OCR processing begins.
+
+use crate::ocr::error::OcrError;
+use std::env;
+use std::path::Path;
+
+/// Validate language configuration and check for traineddata files.
+///
+/// This function validates that:
+/// 1. Language string is not empty
+/// 2. Traineddata files exist for all specified languages
+///
+/// # Arguments
+///
+/// * `language` - Language code(s) to validate (can be "eng" or "eng+fra" etc.)
+/// * `tessdata_path` - Path to tessdata directory
+///
+/// # Returns
+///
+/// `Ok(())` if validation passes, otherwise returns an error
+pub(super) fn validate_language_and_traineddata(language: &str, tessdata_path: &str) -> Result<(), OcrError> {
+    // Validate language before initializing to prevent segfault ~keep
+    if language.trim().is_empty() {
+        return Err(OcrError::TesseractInitializationFailed(
+            "Language cannot be empty. Please specify a valid language code (e.g., 'eng')".to_string(),
+        ));
+    }
+
+    // Validate language file exists before initializing to prevent segfault ~keep
+    if !tessdata_path.is_empty() {
+        let languages: Vec<&str> = language.split('+').collect();
+        for lang in languages {
+            let lang = lang.trim();
+            if lang.is_empty() {
+                continue;
+            }
+            let traineddata_path = Path::new(tessdata_path).join(format!("{}.traineddata", lang));
+            if !traineddata_path.exists() {
+                return Err(OcrError::TesseractInitializationFailed(format!(
+                    "Language '{}' not found. Traineddata file does not exist: {}",
+                    lang,
+                    traineddata_path.display()
+                )));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Resolve tessdata path from environment or fallback locations.
+///
+/// Checks TESSDATA_PREFIX environment variable first, then tries common
+/// installation paths for macOS, Linux, and Windows.
+///
+/// # Returns
+///
+/// Path to tessdata directory if found, otherwise empty string
+pub(super) fn resolve_tessdata_path() -> String {
+    let tessdata_env = env::var("TESSDATA_PREFIX").ok();
+    let fallback_paths = [
+        "/opt/homebrew/share/tessdata",
+        "/opt/homebrew/opt/tesseract/share/tessdata",
+        "/usr/local/opt/tesseract/share/tessdata",
+        "/usr/share/tesseract-ocr/5/tessdata",
+        "/usr/share/tesseract-ocr/4/tessdata",
+        "/usr/share/tessdata",
+        "/usr/local/share/tessdata",
+        r#"C:\Program Files\Tesseract-OCR\tessdata"#,
+        r#"C:\ProgramData\Tesseract-OCR\tessdata"#,
+    ];
+
+    tessdata_env
+        .or_else(|| {
+            fallback_paths
+                .iter()
+                .find(|p| Path::new(p).exists())
+                .map(|p| (*p).to_string())
+        })
+        .unwrap_or_default()
+}
+
+/// Strip control characters from text, preserving whitespace.
+///
+/// Removes control characters (0x00-0x1F, 0x7F) except for newlines, carriage returns, and tabs.
+///
+/// # Arguments
+///
+/// * `text` - Text to clean
+///
+/// # Returns
+///
+/// Cleaned text with control characters removed
+pub(super) fn strip_control_characters(text: &str) -> String {
+    if text
+        .chars()
+        .any(|c| matches!(c, '\u{0000}'..='\u{001F}' | '\u{007F}') && c != '\n' && c != '\r' && c != '\t')
+    {
+        text.chars()
+            .filter(|c| !matches!(c, '\u{0000}'..='\u{001F}' | '\u{007F}') || matches!(c, '\n' | '\r' | '\t'))
+            .collect()
+    } else {
+        text.to_string()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_strip_control_characters() {
+        let input = "Hello\x00World\x01Test";
+        let output = strip_control_characters(input);
+        assert_eq!(output, "HelloWorldTest");
+
+        let input_with_newlines = "Hello\nWorld\rTest\t!";
+        let output = strip_control_characters(input_with_newlines);
+        assert_eq!(output, "Hello\nWorld\rTest\t!");
+    }
+
+    #[test]
+    fn test_strip_control_characters_all_control() {
+        let input = "\x00\x01\x02\x03";
+        let output = strip_control_characters(input);
+        assert_eq!(output, "");
+    }
+
+    #[test]
+    fn test_strip_control_characters_no_control() {
+        let input = "Hello World Test";
+        let output = strip_control_characters(input);
+        assert_eq!(output, "Hello World Test");
+    }
+
+    #[test]
+    fn test_strip_control_characters_delete_char() {
+        let input = "Hello\x7FWorld";
+        let output = strip_control_characters(input);
+        assert_eq!(output, "HelloWorld");
+    }
+}

data/vendor/kreuzberg/src/ocr/tesseract_backend.rs

@@ -177,20 +177,24 @@ impl OcrBackend for TesseractBackend {
     async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult> {
         let tess_config = self.config_to_tesseract(config);
         let tess_config_clone = tess_config.clone();
+        let output_format = config.output_format;
 
         let processor = Arc::clone(&self.processor);
         let image_bytes = image_bytes.to_vec();
 
-        let ocr_result = tokio::task::spawn_blocking(move || processor.process_image(&image_bytes, &tess_config_clone))
-            .await
-            .map_err(|e| crate::KreuzbergError::Plugin {
-                message: format!("Tesseract task panicked: {}", e),
-                plugin_name: "tesseract".to_string(),
-            })?
-            .map_err(|e| crate::KreuzbergError::Ocr {
-                message: format!("Tesseract OCR failed: {}", e),
-                source: Some(Box::new(e)),
-            })?;
+        let ocr_result = tokio::task::spawn_blocking(move || match output_format {
+            Some(fmt) => processor.process_image_with_format(&image_bytes, &tess_config_clone, fmt),
+            None => processor.process_image(&image_bytes, &tess_config_clone),
+        })
+        .await
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Tesseract task panicked: {}", e),
+            plugin_name: "tesseract".to_string(),
+        })?
+        .map_err(|e| crate::KreuzbergError::Ocr {
+            message: format!("Tesseract OCR failed: {}", e),
+            source: Some(Box::new(e)),
+        })?;
 
         let metadata = crate::types::Metadata {
             format: Some(crate::types::FormatMetadata::Ocr(crate::types::OcrMetadata {
@@ -225,26 +229,32 @@ impl OcrBackend for TesseractBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            elements: None,
+            djot_content: None,
         })
     }
 
     async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
         let tess_config = self.config_to_tesseract(config);
         let tess_config_clone = tess_config.clone();
+        let output_format = config.output_format;
 
         let processor = Arc::clone(&self.processor);
         let path_str = path.to_string_lossy().to_string();
 
-        let ocr_result = tokio::task::spawn_blocking(move || processor.process_file(&path_str, &tess_config_clone))
-            .await
-            .map_err(|e| crate::KreuzbergError::Plugin {
-                message: format!("Tesseract task panicked: {}", e),
-                plugin_name: "tesseract".to_string(),
-            })?
-            .map_err(|e| crate::KreuzbergError::Ocr {
-                message: format!("Tesseract OCR failed: {}", e),
-                source: Some(Box::new(e)),
-            })?;
+        let ocr_result = tokio::task::spawn_blocking(move || match output_format {
+            Some(fmt) => processor.process_file_with_format(&path_str, &tess_config_clone, fmt),
+            None => processor.process_file(&path_str, &tess_config_clone),
+        })
+        .await
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Tesseract task panicked: {}", e),
+            plugin_name: "tesseract".to_string(),
+        })?
+        .map_err(|e| crate::KreuzbergError::Ocr {
+            message: format!("Tesseract OCR failed: {}", e),
+            source: Some(Box::new(e)),
+        })?;
 
         let metadata = crate::types::Metadata {
             format: Some(crate::types::FormatMetadata::Ocr(crate::types::OcrMetadata {
@@ -279,6 +289,8 @@ impl OcrBackend for TesseractBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            elements: None,
+            djot_content: None,
         })
     }
 
@@ -326,10 +338,11 @@ mod tests {
     #[test]
     fn test_tesseract_backend_supports_language() {
         let backend = TesseractBackend::new().unwrap();
+        // English should always be available
         assert!(backend.supports_language("eng"));
-        assert!(backend.supports_language("deu"));
-        assert!(backend.supports_language("fra"));
+        // Invalid language codes should return false
         assert!(!backend.supports_language("xyz"));
+        assert!(!backend.supports_language("invalid"));
     }
 
     #[test]
@@ -342,9 +355,10 @@ mod tests {
     fn test_tesseract_backend_supported_languages() {
         let backend = TesseractBackend::new().unwrap();
         let languages = backend.supported_languages();
+        // English should always be available
         assert!(languages.contains(&"eng".to_string()));
-        assert!(languages.contains(&"deu".to_string()));
-        assert!(languages.len() > 30);
+        // Should have at least English
+        assert!(!languages.is_empty());
     }
 
     #[test]
@@ -354,6 +368,7 @@ mod tests {
             backend: "tesseract".to_string(),
             language: "deu".to_string(),
             tesseract_config: None,
+            output_format: None,
         };
 
         let tess_config = backend.config_to_tesseract(&ocr_config);
@@ -375,6 +390,7 @@ mod tests {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: Some(custom_tess_config),
+            output_format: None,
         };
 
         let tess_config = backend.config_to_tesseract(&ocr_config);
@@ -418,6 +434,7 @@ mod tests {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: Some(custom_tess_config),
+            output_format: None,
         };
 
         let tess_config = backend.config_to_tesseract(&ocr_config);

data/vendor/kreuzberg/src/pdf/bindings.rs

@@ -263,21 +263,34 @@ mod tests {
 
     #[test]
     fn test_bind_pdfium_multiple_calls() {
-        let result1 = bind_pdfium(PdfError::TextExtractionFailed, "test 1");
-        let result2 = bind_pdfium(PdfError::TextExtractionFailed, "test 2");
+        // First call - acquire lock, test success, then drop handle to release lock
+        {
+            let result1 = bind_pdfium(PdfError::TextExtractionFailed, "test 1");
+            assert!(result1.is_ok(), "First call should succeed");
+        } // result1 dropped here, releasing the lock
 
-        assert!(result1.is_ok(), "First call should succeed");
-        assert!(result2.is_ok(), "Second call should also succeed");
+        // Second call - can now acquire lock since first handle was dropped
+        {
+            let result2 = bind_pdfium(PdfError::TextExtractionFailed, "test 2");
+            assert!(result2.is_ok(), "Second call should also succeed");
+        }
     }
 
     #[test]
     fn test_bind_pdfium_returns_same_instance() {
-        let handle1 = bind_pdfium(PdfError::TextExtractionFailed, "test 1").unwrap();
-        let handle2 = bind_pdfium(PdfError::TextExtractionFailed, "test 2").unwrap();
+        // Get pointer from first handle, then drop it to release lock
+        let ptr1 = {
+            let handle1 = bind_pdfium(PdfError::TextExtractionFailed, "test 1").unwrap();
+            &*handle1 as *const Pdfium
+        }; // handle1 dropped here, releasing the lock
+
+        // Get pointer from second handle
+        let ptr2 = {
+            let handle2 = bind_pdfium(PdfError::TextExtractionFailed, "test 2").unwrap();
+            &*handle2 as *const Pdfium
+        };
 
         // Both handles should dereference to the same Pdfium instance
-        let ptr1 = &*handle1 as *const Pdfium;
-        let ptr2 = &*handle2 as *const Pdfium;
         assert_eq!(ptr1, ptr2, "Both handles should reference the same Pdfium instance");
     }