kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.15

data/vendor/kreuzberg/src/ocr/error.rs

@@ -1,37 +1,37 @@
-use std::fmt;
-
-/// OCR-specific errors (pure Rust, no PyO3)
-#[derive(Debug, Clone)]
-pub enum OcrError {
-    TesseractInitializationFailed(String),
-    UnsupportedVersion(String),
-    InvalidConfiguration(String),
-    InvalidLanguageCode(String),
-    ImageProcessingFailed(String),
-    ProcessingFailed(String),
-    CacheError(String),
-    IOError(String),
-}
-
-impl fmt::Display for OcrError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::TesseractInitializationFailed(msg) => {
-                write!(f, "Tesseract initialization failed: {}", msg)
-            }
-            Self::UnsupportedVersion(msg) => {
-                write!(f, "Unsupported Tesseract version: {}", msg)
-            }
-            Self::InvalidConfiguration(msg) => write!(f, "Invalid configuration: {}", msg),
-            Self::InvalidLanguageCode(msg) => write!(f, "Invalid language code: {}", msg),
-            Self::ImageProcessingFailed(msg) => write!(f, "Image processing failed: {}", msg),
-            Self::ProcessingFailed(msg) => write!(f, "OCR processing failed: {}", msg),
-            Self::CacheError(msg) => write!(f, "Cache error: {}", msg),
-            Self::IOError(msg) => write!(f, "I/O error: {}", msg),
-        }
-    }
-}
-
-impl std::error::Error for OcrError {}
-
-// NOTE: No From<std::io::Error> impl - IO errors must bubble up unchanged per error handling policy
+use std::fmt;
+
+/// OCR-specific errors (pure Rust, no PyO3)
+#[derive(Debug, Clone)]
+pub enum OcrError {
+    TesseractInitializationFailed(String),
+    UnsupportedVersion(String),
+    InvalidConfiguration(String),
+    InvalidLanguageCode(String),
+    ImageProcessingFailed(String),
+    ProcessingFailed(String),
+    CacheError(String),
+    IOError(String),
+}
+
+impl fmt::Display for OcrError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::TesseractInitializationFailed(msg) => {
+                write!(f, "Tesseract initialization failed: {}", msg)
+            }
+            Self::UnsupportedVersion(msg) => {
+                write!(f, "Unsupported Tesseract version: {}", msg)
+            }
+            Self::InvalidConfiguration(msg) => write!(f, "Invalid configuration: {}", msg),
+            Self::InvalidLanguageCode(msg) => write!(f, "Invalid language code: {}", msg),
+            Self::ImageProcessingFailed(msg) => write!(f, "Image processing failed: {}", msg),
+            Self::ProcessingFailed(msg) => write!(f, "OCR processing failed: {}", msg),
+            Self::CacheError(msg) => write!(f, "Cache error: {}", msg),
+            Self::IOError(msg) => write!(f, "I/O error: {}", msg),
+        }
+    }
+}
+
+impl std::error::Error for OcrError {}
+
+// NOTE: No From<std::io::Error> impl - IO errors must bubble up unchanged per error handling policy

data/vendor/kreuzberg/src/ocr/hocr.rs

@@ -1,216 +1,216 @@
-use super::error::OcrError;
-use html_to_markdown_rs::{ConversionOptions, convert};
-
-pub fn convert_hocr_to_markdown(hocr_html: &str, options: Option<ConversionOptions>) -> Result<String, OcrError> {
-    let use_default = options.is_none();
-    let mut opts = options.unwrap_or_default();
-
-    if use_default {
-        opts.hocr_spatial_tables = false;
-        opts.extract_metadata = false;
-    }
-
-    convert(hocr_html, Some(opts)).map_err(|e| OcrError::ProcessingFailed(format!("hOCR conversion failed: {}", e)))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_simple_hocr_conversion() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">Hello</span>
-                <span class="ocrx_word">World</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.contains("Hello"));
-        assert!(markdown.contains("World"));
-    }
-
-    #[test]
-    fn test_hocr_with_formatting() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <strong class="ocrx_word">Bold</strong>
-                <em class="ocrx_word">Italic</em>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-    }
-
-    #[test]
-    fn test_empty_hocr() {
-        let hocr = "";
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.is_empty() || markdown.trim().is_empty());
-    }
-
-    #[test]
-    fn test_hocr_with_headings() {
-        let hocr = r#"<div class="ocr_page">
-            <h1>Title</h1>
-            <p class="ocr_par">
-                <span class="ocrx_word">Content</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-        assert!(markdown.contains("Content"));
-    }
-
-    #[test]
-    fn test_hocr_with_multiple_paragraphs() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">First</span>
-                <span class="ocrx_word">paragraph</span>
-            </p>
-            <p class="ocr_par">
-                <span class="ocrx_word">Second</span>
-                <span class="ocrx_word">paragraph</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.contains("First"));
-        assert!(markdown.contains("Second"));
-    }
-
-    #[test]
-    fn test_hocr_with_line_breaks() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_line">
-                    <span class="ocrx_word">Line</span>
-                    <span class="ocrx_word">one</span>
-                </span>
-                <span class="ocrx_line">
-                    <span class="ocrx_word">Line</span>
-                    <span class="ocrx_word">two</span>
-                </span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_whitespace_handling() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">  Padded  </span>
-                <span class="ocrx_word">  Text  </span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_special_characters() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">&lt;special&gt;</span>
-                <span class="ocrx_word">&amp;chars&amp;</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_nested_structure() {
-        let hocr = r#"<div class="ocr_page">
-            <div class="ocr_carea">
-                <p class="ocr_par">
-                    <span class="ocr_line">
-                        <span class="ocrx_word">Nested</span>
-                    </span>
-                </p>
-            </div>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.contains("Nested"));
-    }
-
-    #[test]
-    fn test_hocr_malformed_html() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">Unclosed
-        </div>"#;
-
-        let result = convert_hocr_to_markdown(hocr, None);
-        assert!(result.is_ok());
-    }
-
-    #[test]
-    fn test_hocr_no_ocr_classes() {
-        let hocr = r#"<div>
-            <p>
-                <span>Regular HTML</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(!markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_mixed_content() {
-        let hocr = r#"<div class="ocr_page">
-            <h1>Heading</h1>
-            <p class="ocr_par">
-                <span class="ocrx_word">Paragraph</span>
-            </p>
-            <ul>
-                <li>List item</li>
-            </ul>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.contains("Heading") || markdown.contains("heading") || !markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_unicode_content() {
-        let hocr = r#"<div class="ocr_page">
-            <p class="ocr_par">
-                <span class="ocrx_word">Ñoño</span>
-                <span class="ocrx_word">日本語</span>
-                <span class="ocrx_word">العربية</span>
-            </p>
-        </div>"#;
-
-        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
-        assert!(markdown.contains("Ñoño") || !markdown.is_empty());
-    }
-
-    #[test]
-    fn test_hocr_large_document() {
-        let mut hocr = String::from(r#"<div class="ocr_page">"#);
-        for i in 0..100 {
-            hocr.push_str(&format!(
-                r#"<p class="ocr_par"><span class="ocrx_word">Word{}</span></p>"#,
-                i
-            ));
-        }
-        hocr.push_str("</div>");
-
-        let result = convert_hocr_to_markdown(&hocr, None);
-        assert!(result.is_ok());
-        let markdown = result.unwrap();
-        assert!(!markdown.is_empty());
-    }
-}
+use super::error::OcrError;
+use html_to_markdown_rs::{ConversionOptions, convert};
+
+pub fn convert_hocr_to_markdown(hocr_html: &str, options: Option<ConversionOptions>) -> Result<String, OcrError> {
+    let use_default = options.is_none();
+    let mut opts = options.unwrap_or_default();
+
+    if use_default {
+        opts.hocr_spatial_tables = false;
+        opts.extract_metadata = false;
+    }
+
+    convert(hocr_html, Some(opts)).map_err(|e| OcrError::ProcessingFailed(format!("hOCR conversion failed: {}", e)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_simple_hocr_conversion() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Hello</span>
+                <span class="ocrx_word">World</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Hello"));
+        assert!(markdown.contains("World"));
+    }
+
+    #[test]
+    fn test_hocr_with_formatting() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <strong class="ocrx_word">Bold</strong>
+                <em class="ocrx_word">Italic</em>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_empty_hocr() {
+        let hocr = "";
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.is_empty() || markdown.trim().is_empty());
+    }
+
+    #[test]
+    fn test_hocr_with_headings() {
+        let hocr = r#"<div class="ocr_page">
+            <h1>Title</h1>
+            <p class="ocr_par">
+                <span class="ocrx_word">Content</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+        assert!(markdown.contains("Content"));
+    }
+
+    #[test]
+    fn test_hocr_with_multiple_paragraphs() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">First</span>
+                <span class="ocrx_word">paragraph</span>
+            </p>
+            <p class="ocr_par">
+                <span class="ocrx_word">Second</span>
+                <span class="ocrx_word">paragraph</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("First"));
+        assert!(markdown.contains("Second"));
+    }
+
+    #[test]
+    fn test_hocr_with_line_breaks() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_line">
+                    <span class="ocrx_word">Line</span>
+                    <span class="ocrx_word">one</span>
+                </span>
+                <span class="ocrx_line">
+                    <span class="ocrx_word">Line</span>
+                    <span class="ocrx_word">two</span>
+                </span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_whitespace_handling() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">  Padded  </span>
+                <span class="ocrx_word">  Text  </span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_special_characters() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">&lt;special&gt;</span>
+                <span class="ocrx_word">&amp;chars&amp;</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_nested_structure() {
+        let hocr = r#"<div class="ocr_page">
+            <div class="ocr_carea">
+                <p class="ocr_par">
+                    <span class="ocr_line">
+                        <span class="ocrx_word">Nested</span>
+                    </span>
+                </p>
+            </div>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Nested"));
+    }
+
+    #[test]
+    fn test_hocr_malformed_html() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Unclosed
+        </div>"#;
+
+        let result = convert_hocr_to_markdown(hocr, None);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_hocr_no_ocr_classes() {
+        let hocr = r#"<div>
+            <p>
+                <span>Regular HTML</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_mixed_content() {
+        let hocr = r#"<div class="ocr_page">
+            <h1>Heading</h1>
+            <p class="ocr_par">
+                <span class="ocrx_word">Paragraph</span>
+            </p>
+            <ul>
+                <li>List item</li>
+            </ul>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Heading") || markdown.contains("heading") || !markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_unicode_content() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Ñoño</span>
+                <span class="ocrx_word">日本語</span>
+                <span class="ocrx_word">العربية</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Ñoño") || !markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_large_document() {
+        let mut hocr = String::from(r#"<div class="ocr_page">"#);
+        for i in 0..100 {
+            hocr.push_str(&format!(
+                r#"<p class="ocr_par"><span class="ocrx_word">Word{}</span></p>"#,
+                i
+            ));
+        }
+        hocr.push_str("</div>");
+
+        let result = convert_hocr_to_markdown(&hocr, None);
+        assert!(result.is_ok());
+        let markdown = result.unwrap();
+        assert!(!markdown.is_empty());
+    }
+}

data/vendor/kreuzberg/src/ocr/mod.rs

@@ -1,58 +1,58 @@
-//! OCR (Optical Character Recognition) subsystem.
-//!
-//! This module provides OCR functionality using Tesseract as the backend.
-//! It includes caching, table reconstruction, hOCR parsing, and batch processing.
-//!
-//! # Features
-//!
-//! - **Tesseract integration**: Native Tesseract backend via `kreuzberg-tesseract`
-//! - **Result caching**: Persistent cache for OCR results using file hashing
-//! - **Table reconstruction**: Extract and reconstruct tables from hOCR/TSV output
-//! - **hOCR to Markdown**: Convert hOCR format to clean Markdown
-//! - **Batch processing**: Process multiple images efficiently
-//! - **Language support**: Validate and configure Tesseract languages
-//! - **PSM modes**: Support for all Tesseract Page Segmentation Modes
-//!
-//! # Example
-//!
-//! ```rust,no_run
-//! use kreuzberg::ocr::{OcrProcessor, TesseractConfig};
-//!
-//! # fn example() -> Result<(), kreuzberg::ocr::error::OcrError> {
-//! let processor = OcrProcessor::new(None)?;
-//! let config = TesseractConfig::default();
-//!
-//! let image_bytes = std::fs::read("scanned.png").expect("failed to read image");
-//! let result = processor.process_image(&image_bytes, &config)?;
-//!
-//! println!("Extracted text: {}", result.content);
-//! # Ok(())
-//! # }
-//! ```
-//!
-//! # Optional Feature
-//!
-//! This module requires the `ocr` feature to be enabled:
-//! ```toml
-//! [dependencies]
-//! kreuzberg = { version = "4.0", features = ["ocr"] }
-//! ```
-pub mod cache;
-pub mod error;
-pub mod hocr;
-pub mod processor;
-pub mod table;
-pub mod tesseract_backend;
-pub mod types;
-pub mod utils;
-pub mod validation;
-
-pub use cache::{OcrCache, OcrCacheStats};
-pub use error::OcrError;
-pub use hocr::convert_hocr_to_markdown;
-pub use processor::OcrProcessor;
-pub use table::{HocrWord, extract_words_from_tsv, reconstruct_table, table_to_markdown};
-pub use tesseract_backend::TesseractBackend;
-pub use types::{BatchItemResult, ExtractionResult, PSMMode, Table, TesseractConfig};
-pub use utils::compute_hash;
-pub use validation::{validate_language_code, validate_tesseract_version};
+//! OCR (Optical Character Recognition) subsystem.
+//!
+//! This module provides OCR functionality using Tesseract as the backend.
+//! It includes caching, table reconstruction, hOCR parsing, and batch processing.
+//!
+//! # Features
+//!
+//! - **Tesseract integration**: Native Tesseract backend via `kreuzberg-tesseract`
+//! - **Result caching**: Persistent cache for OCR results using file hashing
+//! - **Table reconstruction**: Extract and reconstruct tables from hOCR/TSV output
+//! - **hOCR to Markdown**: Convert hOCR format to clean Markdown
+//! - **Batch processing**: Process multiple images efficiently
+//! - **Language support**: Validate and configure Tesseract languages
+//! - **PSM modes**: Support for all Tesseract Page Segmentation Modes
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::ocr::{OcrProcessor, TesseractConfig};
+//!
+//! # fn example() -> Result<(), kreuzberg::ocr::error::OcrError> {
+//! let processor = OcrProcessor::new(None)?;
+//! let config = TesseractConfig::default();
+//!
+//! let image_bytes = std::fs::read("scanned.png").expect("failed to read image");
+//! let result = processor.process_image(&image_bytes, &config)?;
+//!
+//! println!("Extracted text: {}", result.content);
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Optional Feature
+//!
+//! This module requires the `ocr` feature to be enabled:
+//! ```toml
+//! [dependencies]
+//! kreuzberg = { version = "4.0", features = ["ocr"] }
+//! ```
+pub mod cache;
+pub mod error;
+pub mod hocr;
+pub mod processor;
+pub mod table;
+pub mod tesseract_backend;
+pub mod types;
+pub mod utils;
+pub mod validation;
+
+pub use cache::{OcrCache, OcrCacheStats};
+pub use error::OcrError;
+pub use hocr::convert_hocr_to_markdown;
+pub use processor::OcrProcessor;
+pub use table::{HocrWord, extract_words_from_tsv, reconstruct_table, table_to_markdown};
+pub use tesseract_backend::TesseractBackend;
+pub use types::{BatchItemResult, ExtractionResult, PSMMode, Table, TesseractConfig};
+pub use utils::compute_hash;
+pub use validation::{validate_language_code, validate_tesseract_version};