kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/ocr/hocr.rs

@@ -0,0 +1,216 @@
+use super::error::OcrError;
+use html_to_markdown_rs::{ConversionOptions, convert};
+
+pub fn convert_hocr_to_markdown(hocr_html: &str, options: Option<ConversionOptions>) -> Result<String, OcrError> {
+    let use_default = options.is_none();
+    let mut opts = options.unwrap_or_default();
+
+    if use_default {
+        opts.hocr_spatial_tables = false;
+        opts.extract_metadata = false;
+    }
+
+    convert(hocr_html, Some(opts)).map_err(|e| OcrError::ProcessingFailed(format!("hOCR conversion failed: {}", e)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_simple_hocr_conversion() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Hello</span>
+                <span class="ocrx_word">World</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Hello"));
+        assert!(markdown.contains("World"));
+    }
+
+    #[test]
+    fn test_hocr_with_formatting() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <strong class="ocrx_word">Bold</strong>
+                <em class="ocrx_word">Italic</em>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_empty_hocr() {
+        let hocr = "";
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.is_empty() || markdown.trim().is_empty());
+    }
+
+    #[test]
+    fn test_hocr_with_headings() {
+        let hocr = r#"<div class="ocr_page">
+            <h1>Title</h1>
+            <p class="ocr_par">
+                <span class="ocrx_word">Content</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+        assert!(markdown.contains("Content"));
+    }
+
+    #[test]
+    fn test_hocr_with_multiple_paragraphs() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">First</span>
+                <span class="ocrx_word">paragraph</span>
+            </p>
+            <p class="ocr_par">
+                <span class="ocrx_word">Second</span>
+                <span class="ocrx_word">paragraph</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("First"));
+        assert!(markdown.contains("Second"));
+    }
+
+    #[test]
+    fn test_hocr_with_line_breaks() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_line">
+                    <span class="ocrx_word">Line</span>
+                    <span class="ocrx_word">one</span>
+                </span>
+                <span class="ocrx_line">
+                    <span class="ocrx_word">Line</span>
+                    <span class="ocrx_word">two</span>
+                </span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_whitespace_handling() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">  Padded  </span>
+                <span class="ocrx_word">  Text  </span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_special_characters() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">&lt;special&gt;</span>
+                <span class="ocrx_word">&amp;chars&amp;</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_nested_structure() {
+        let hocr = r#"<div class="ocr_page">
+            <div class="ocr_carea">
+                <p class="ocr_par">
+                    <span class="ocr_line">
+                        <span class="ocrx_word">Nested</span>
+                    </span>
+                </p>
+            </div>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Nested"));
+    }
+
+    #[test]
+    fn test_hocr_malformed_html() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Unclosed
+        </div>"#;
+
+        let result = convert_hocr_to_markdown(hocr, None);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_hocr_no_ocr_classes() {
+        let hocr = r#"<div>
+            <p>
+                <span>Regular HTML</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(!markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_mixed_content() {
+        let hocr = r#"<div class="ocr_page">
+            <h1>Heading</h1>
+            <p class="ocr_par">
+                <span class="ocrx_word">Paragraph</span>
+            </p>
+            <ul>
+                <li>List item</li>
+            </ul>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Heading") || markdown.contains("heading") || !markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_unicode_content() {
+        let hocr = r#"<div class="ocr_page">
+            <p class="ocr_par">
+                <span class="ocrx_word">Ñoño</span>
+                <span class="ocrx_word">日本語</span>
+                <span class="ocrx_word">العربية</span>
+            </p>
+        </div>"#;
+
+        let markdown = convert_hocr_to_markdown(hocr, None).unwrap();
+        assert!(markdown.contains("Ñoño") || !markdown.is_empty());
+    }
+
+    #[test]
+    fn test_hocr_large_document() {
+        let mut hocr = String::from(r#"<div class="ocr_page">"#);
+        for i in 0..100 {
+            hocr.push_str(&format!(
+                r#"<p class="ocr_par"><span class="ocrx_word">Word{}</span></p>"#,
+                i
+            ));
+        }
+        hocr.push_str("</div>");
+
+        let result = convert_hocr_to_markdown(&hocr, None);
+        assert!(result.is_ok());
+        let markdown = result.unwrap();
+        assert!(!markdown.is_empty());
+    }
+}

data/vendor/kreuzberg/src/ocr/mod.rs

@@ -0,0 +1,58 @@
+//! OCR (Optical Character Recognition) subsystem.
+//!
+//! This module provides OCR functionality using Tesseract as the backend.
+//! It includes caching, table reconstruction, hOCR parsing, and batch processing.
+//!
+//! # Features
+//!
+//! - **Tesseract integration**: Native Tesseract backend via `kreuzberg-tesseract`
+//! - **Result caching**: Persistent cache for OCR results using file hashing
+//! - **Table reconstruction**: Extract and reconstruct tables from hOCR/TSV output
+//! - **hOCR to Markdown**: Convert hOCR format to clean Markdown
+//! - **Batch processing**: Process multiple images efficiently
+//! - **Language support**: Validate and configure Tesseract languages
+//! - **PSM modes**: Support for all Tesseract Page Segmentation Modes
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::ocr::{OcrProcessor, TesseractConfig};
+//!
+//! # fn example() -> Result<(), kreuzberg::ocr::error::OcrError> {
+//! let processor = OcrProcessor::new(None)?;
+//! let config = TesseractConfig::default();
+//!
+//! let image_bytes = std::fs::read("scanned.png").expect("failed to read image");
+//! let result = processor.process_image(&image_bytes, &config)?;
+//!
+//! println!("Extracted text: {}", result.content);
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Optional Feature
+//!
+//! This module requires the `ocr` feature to be enabled:
+//! ```toml
+//! [dependencies]
+//! kreuzberg = { version = "4.0", features = ["ocr"] }
+//! ```
+pub mod cache;
+pub mod error;
+pub mod hocr;
+pub mod processor;
+pub mod table;
+pub mod tesseract_backend;
+pub mod types;
+pub mod utils;
+pub mod validation;
+
+pub use cache::{OcrCache, OcrCacheStats};
+pub use error::OcrError;
+pub use hocr::convert_hocr_to_markdown;
+pub use processor::OcrProcessor;
+pub use table::{HocrWord, extract_words_from_tsv, reconstruct_table, table_to_markdown};
+pub use tesseract_backend::TesseractBackend;
+pub use types::{BatchItemResult, ExtractionResult, PSMMode, Table, TesseractConfig};
+pub use utils::compute_hash;
+pub use validation::{validate_language_code, validate_tesseract_version};