kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/ocr/language_registry.rs

@@ -23,6 +23,8 @@
 use std::collections::HashMap;
 use std::sync::OnceLock;
 
+use super::backends;
+
 /// Global language registry instance (lazy initialized)
 static LANGUAGE_REGISTRY: OnceLock<LanguageRegistry> = OnceLock::new();
 
@@ -46,241 +48,15 @@ impl LanguageRegistry {
             backends: HashMap::new(),
         };
 
-        registry.backends.insert(
-            "easyocr".to_string(),
-            vec![
-                "abq".to_string(),
-                "ady".to_string(),
-                "af".to_string(),
-                "ang".to_string(),
-                "ar".to_string(),
-                "as".to_string(),
-                "ava".to_string(),
-                "az".to_string(),
-                "be".to_string(),
-                "bg".to_string(),
-                "bh".to_string(),
-                "bho".to_string(),
-                "bn".to_string(),
-                "bs".to_string(),
-                "ch_sim".to_string(),
-                "ch_tra".to_string(),
-                "che".to_string(),
-                "cs".to_string(),
-                "cy".to_string(),
-                "da".to_string(),
-                "dar".to_string(),
-                "de".to_string(),
-                "en".to_string(),
-                "es".to_string(),
-                "et".to_string(),
-                "fa".to_string(),
-                "fr".to_string(),
-                "ga".to_string(),
-                "gom".to_string(),
-                "hi".to_string(),
-                "hr".to_string(),
-                "hu".to_string(),
-                "id".to_string(),
-                "inh".to_string(),
-                "is".to_string(),
-                "it".to_string(),
-                "ja".to_string(),
-                "kbd".to_string(),
-                "kn".to_string(),
-                "ko".to_string(),
-                "ku".to_string(),
-                "la".to_string(),
-                "lbe".to_string(),
-                "lez".to_string(),
-                "lt".to_string(),
-                "lv".to_string(),
-                "mah".to_string(),
-                "mai".to_string(),
-                "mi".to_string(),
-                "mn".to_string(),
-                "mr".to_string(),
-                "ms".to_string(),
-                "mt".to_string(),
-                "ne".to_string(),
-                "new".to_string(),
-                "nl".to_string(),
-                "no".to_string(),
-                "oc".to_string(),
-                "pi".to_string(),
-                "pl".to_string(),
-                "pt".to_string(),
-                "ro".to_string(),
-                "ru".to_string(),
-                "rs_cyrillic".to_string(),
-                "rs_latin".to_string(),
-                "sck".to_string(),
-                "sk".to_string(),
-                "sl".to_string(),
-                "sq".to_string(),
-                "sv".to_string(),
-                "sw".to_string(),
-                "ta".to_string(),
-                "tab".to_string(),
-                "te".to_string(),
-                "th".to_string(),
-                "tjk".to_string(),
-                "tl".to_string(),
-                "tr".to_string(),
-                "ug".to_string(),
-                "uk".to_string(),
-                "ur".to_string(),
-                "uz".to_string(),
-                "vi".to_string(),
-            ],
-        );
-
-        registry.backends.insert(
-            "paddleocr".to_string(),
-            vec![
-                "ch".to_string(),
-                "en".to_string(),
-                "french".to_string(),
-                "german".to_string(),
-                "korean".to_string(),
-                "japan".to_string(),
-                "chinese_cht".to_string(),
-                "ta".to_string(),
-                "te".to_string(),
-                "ka".to_string(),
-                "latin".to_string(),
-                "arabic".to_string(),
-                "cyrillic".to_string(),
-                "devanagari".to_string(),
-            ],
-        );
-
-        registry.backends.insert(
-            "tesseract".to_string(),
-            vec![
-                "afr".to_string(),
-                "amh".to_string(),
-                "ara".to_string(),
-                "asm".to_string(),
-                "aze".to_string(),
-                "aze_cyrl".to_string(),
-                "bel".to_string(),
-                "ben".to_string(),
-                "bod".to_string(),
-                "bos".to_string(),
-                "bre".to_string(),
-                "bul".to_string(),
-                "cat".to_string(),
-                "ceb".to_string(),
-                "ces".to_string(),
-                "chi_sim".to_string(),
-                "chi_tra".to_string(),
-                "chr".to_string(),
-                "cos".to_string(),
-                "cym".to_string(),
-                "dan".to_string(),
-                "deu".to_string(),
-                "div".to_string(),
-                "dzo".to_string(),
-                "ell".to_string(),
-                "eng".to_string(),
-                "enm".to_string(),
-                "epo".to_string(),
-                "equ".to_string(),
-                "est".to_string(),
-                "eus".to_string(),
-                "fao".to_string(),
-                "fas".to_string(),
-                "fil".to_string(),
-                "fin".to_string(),
-                "fra".to_string(),
-                "frk".to_string(),
-                "frm".to_string(),
-                "fry".to_string(),
-                "gla".to_string(),
-                "gle".to_string(),
-                "glg".to_string(),
-                "grc".to_string(),
-                "guj".to_string(),
-                "hat".to_string(),
-                "heb".to_string(),
-                "hin".to_string(),
-                "hrv".to_string(),
-                "hun".to_string(),
-                "hye".to_string(),
-                "iku".to_string(),
-                "ind".to_string(),
-                "isl".to_string(),
-                "ita".to_string(),
-                "ita_old".to_string(),
-                "jav".to_string(),
-                "jpn".to_string(),
-                "kan".to_string(),
-                "kat".to_string(),
-                "kat_old".to_string(),
-                "kaz".to_string(),
-                "khm".to_string(),
-                "kir".to_string(),
-                "kmr".to_string(),
-                "kor".to_string(),
-                "lao".to_string(),
-                "lat".to_string(),
-                "lav".to_string(),
-                "lit".to_string(),
-                "ltz".to_string(),
-                "mal".to_string(),
-                "mar".to_string(),
-                "mkd".to_string(),
-                "mlt".to_string(),
-                "mon".to_string(),
-                "mri".to_string(),
-                "msa".to_string(),
-                "mya".to_string(),
-                "nep".to_string(),
-                "nld".to_string(),
-                "nor".to_string(),
-                "oci".to_string(),
-                "ori".to_string(),
-                "osd".to_string(),
-                "pan".to_string(),
-                "pol".to_string(),
-                "por".to_string(),
-                "pus".to_string(),
-                "que".to_string(),
-                "ron".to_string(),
-                "rus".to_string(),
-                "san".to_string(),
-                "sin".to_string(),
-                "slk".to_string(),
-                "slv".to_string(),
-                "snd".to_string(),
-                "spa".to_string(),
-                "spa_old".to_string(),
-                "sqi".to_string(),
-                "srp".to_string(),
-                "srp_latn".to_string(),
-                "sun".to_string(),
-                "swa".to_string(),
-                "swe".to_string(),
-                "syr".to_string(),
-                "tam".to_string(),
-                "tat".to_string(),
-                "tel".to_string(),
-                "tgk".to_string(),
-                "tha".to_string(),
-                "tir".to_string(),
-                "ton".to_string(),
-                "tur".to_string(),
-                "uig".to_string(),
-                "ukr".to_string(),
-                "urd".to_string(),
-                "uzb".to_string(),
-                "uzb_cyrl".to_string(),
-                "vie".to_string(),
-                "yid".to_string(),
-                "yor".to_string(),
-            ],
-        );
+        registry
+            .backends
+            .insert("easyocr".to_string(), backends::easyocr::languages());
+        registry
+            .backends
+            .insert("paddleocr".to_string(), backends::paddleocr::languages());
+        registry
+            .backends
+            .insert("tesseract".to_string(), backends::tesseract::languages());
 
         registry
     }

data/vendor/kreuzberg/src/ocr/mod.rs

@@ -37,6 +37,7 @@
 //! [dependencies]
 //! kreuzberg = { version = "4.0", features = ["ocr"] }
 //! ```
+mod backends;
 pub mod cache;
 pub mod error;
 pub mod hocr;

data/vendor/kreuzberg/src/ocr/processor/config.rs

@@ -0,0 +1,203 @@
+//! Configuration hashing and Tesseract variable management.
+//!
+//! This module handles configuration hashing for caching and
+//! setting Tesseract variables.
+
+use crate::ocr::error::OcrError;
+use crate::ocr::types::TesseractConfig;
+use kreuzberg_tesseract::TesseractAPI;
+
+/// Compute a deterministic hash of the OCR configuration.
+///
+/// This hash is used as part of the cache key to ensure different
+/// configurations produce different cached results.
+///
+/// # Arguments
+///
+/// * `config` - Configuration to hash
+///
+/// # Returns
+///
+/// Hexadecimal string representation of the configuration hash
+pub(super) fn hash_config(config: &TesseractConfig) -> String {
+    use ahash::AHasher;
+    use std::hash::{Hash, Hasher};
+
+    let mut hasher = AHasher::default();
+    config.language.hash(&mut hasher);
+    config.psm.hash(&mut hasher);
+    config.output_format.hash(&mut hasher);
+    config.enable_table_detection.hash(&mut hasher);
+    config.table_min_confidence.to_bits().hash(&mut hasher);
+    config.table_column_threshold.hash(&mut hasher);
+    config.table_row_threshold_ratio.to_bits().hash(&mut hasher);
+    config.classify_use_pre_adapted_templates.hash(&mut hasher);
+    config.language_model_ngram_on.hash(&mut hasher);
+    config.tessedit_dont_blkrej_good_wds.hash(&mut hasher);
+    config.tessedit_dont_rowrej_good_wds.hash(&mut hasher);
+    config.tessedit_enable_dict_correction.hash(&mut hasher);
+    config.tessedit_char_whitelist.hash(&mut hasher);
+    config.tessedit_use_primary_params_model.hash(&mut hasher);
+    config.textord_space_size_is_variable.hash(&mut hasher);
+    config.thresholding_method.hash(&mut hasher);
+
+    format!("{:016x}", hasher.finish())
+}
+
+/// Apply Tesseract configuration variables to the API.
+///
+/// Sets all the advanced Tesseract variables from the configuration.
+///
+/// # Arguments
+///
+/// * `api` - Tesseract API instance
+/// * `config` - Configuration with variables to apply
+///
+/// # Returns
+///
+/// `Ok(())` if all variables were set successfully, otherwise an error
+pub(super) fn apply_tesseract_variables(api: &TesseractAPI, config: &TesseractConfig) -> Result<(), OcrError> {
+    api.set_variable(
+        "classify_use_pre_adapted_templates",
+        &config.classify_use_pre_adapted_templates.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set classify_use_pre_adapted_templates: {}", e)))?;
+
+    api.set_variable("language_model_ngram_on", &config.language_model_ngram_on.to_string())
+        .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set language_model_ngram_on: {}", e)))?;
+
+    api.set_variable(
+        "tessedit_dont_blkrej_good_wds",
+        &config.tessedit_dont_blkrej_good_wds.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set tessedit_dont_blkrej_good_wds: {}", e)))?;
+
+    api.set_variable(
+        "tessedit_dont_rowrej_good_wds",
+        &config.tessedit_dont_rowrej_good_wds.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set tessedit_dont_rowrej_good_wds: {}", e)))?;
+
+    api.set_variable(
+        "tessedit_enable_dict_correction",
+        &config.tessedit_enable_dict_correction.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set tessedit_enable_dict_correction: {}", e)))?;
+
+    if !config.tessedit_char_whitelist.is_empty() {
+        api.set_variable("tessedit_char_whitelist", &config.tessedit_char_whitelist)
+            .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set tessedit_char_whitelist: {}", e)))?;
+    }
+
+    api.set_variable(
+        "tessedit_use_primary_params_model",
+        &config.tessedit_use_primary_params_model.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set tessedit_use_primary_params_model: {}", e)))?;
+
+    api.set_variable(
+        "textord_space_size_is_variable",
+        &config.textord_space_size_is_variable.to_string(),
+    )
+    .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set textord_space_size_is_variable: {}", e)))?;
+
+    api.set_variable("thresholding_method", &config.thresholding_method.to_string())
+        .map_err(|e| OcrError::InvalidConfiguration(format!("Failed to set thresholding_method: {}", e)))?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn create_test_config() -> TesseractConfig {
+        TesseractConfig {
+            output_format: "text".to_string(),
+            enable_table_detection: false,
+            use_cache: false,
+            ..TesseractConfig::default()
+        }
+    }
+
+    #[test]
+    fn test_hash_config_deterministic() {
+        let config = create_test_config();
+
+        let hash1 = hash_config(&config);
+        let hash2 = hash_config(&config);
+
+        assert_eq!(hash1, hash2);
+        assert_eq!(hash1.len(), 16);
+    }
+
+    #[test]
+    fn test_hash_config_different_languages() {
+        let mut config1 = create_test_config();
+        config1.language = "eng".to_string();
+
+        let mut config2 = create_test_config();
+        config2.language = "fra".to_string();
+
+        let hash1 = hash_config(&config1);
+        let hash2 = hash_config(&config2);
+
+        assert_ne!(hash1, hash2);
+    }
+
+    #[test]
+    fn test_hash_config_different_psm() {
+        let mut config1 = create_test_config();
+        config1.psm = 3;
+
+        let mut config2 = create_test_config();
+        config2.psm = 6;
+
+        let hash1 = hash_config(&config1);
+        let hash2 = hash_config(&config2);
+
+        assert_ne!(hash1, hash2);
+    }
+
+    #[test]
+    fn test_hash_config_different_output_format() {
+        let mut config1 = create_test_config();
+        config1.output_format = "text".to_string();
+
+        let mut config2 = create_test_config();
+        config2.output_format = "markdown".to_string();
+
+        let hash1 = hash_config(&config1);
+        let hash2 = hash_config(&config2);
+
+        assert_ne!(hash1, hash2);
+    }
+
+    #[test]
+    fn test_hash_config_table_detection_flag() {
+        let mut config1 = create_test_config();
+        config1.enable_table_detection = false;
+
+        let mut config2 = create_test_config();
+        config2.enable_table_detection = true;
+
+        let hash1 = hash_config(&config1);
+        let hash2 = hash_config(&config2);
+
+        assert_ne!(hash1, hash2);
+    }
+
+    #[test]
+    fn test_hash_config_whitelist() {
+        let mut config1 = create_test_config();
+        config1.tessedit_char_whitelist = "".to_string();
+
+        let mut config2 = create_test_config();
+        config2.tessedit_char_whitelist = "0123456789".to_string();
+
+        let hash1 = hash_config(&config1);
+        let hash2 = hash_config(&config2);
+
+        assert_ne!(hash1, hash2);
+    }
+}