kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/ocr/utils.rs

@@ -1,47 +1,47 @@
-use ahash::AHasher;
-use std::hash::{Hash, Hasher};
-
-/// Minimal supported Tesseract version
-pub const MINIMAL_SUPPORTED_TESSERACT_VERSION: u32 = 5;
-
-/// TSV parsing constants
-pub const TSV_WORD_LEVEL: u32 = 5;
-pub const TSV_MIN_FIELDS: usize = 12;
-
-/// Table formatting constants
-pub const MIN_COLUMN_WIDTH: usize = 3;
-
-/// Compute a hash string from input data
-pub fn compute_hash(data: &str) -> String {
-    let mut hasher = AHasher::default();
-    data.hash(&mut hasher);
-    let hash = hasher.finish();
-    format!("{:016x}", hash)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_compute_hash_deterministic() {
-        let input = "test_string";
-        let hash1 = compute_hash(input);
-        let hash2 = compute_hash(input);
-        assert_eq!(hash1, hash2);
-        assert_eq!(hash1.len(), 16);
-    }
-
-    #[test]
-    fn test_compute_hash_different_inputs() {
-        let hash1 = compute_hash("test1");
-        let hash2 = compute_hash("test2");
-        assert_ne!(hash1, hash2);
-    }
-
-    #[test]
-    fn test_compute_hash_empty() {
-        let hash = compute_hash("");
-        assert_eq!(hash.len(), 16);
-    }
-}
+use ahash::AHasher;
+use std::hash::{Hash, Hasher};
+
+/// Minimal supported Tesseract version
+pub const MINIMAL_SUPPORTED_TESSERACT_VERSION: u32 = 5;
+
+/// TSV parsing constants
+pub const TSV_WORD_LEVEL: u32 = 5;
+pub const TSV_MIN_FIELDS: usize = 12;
+
+/// Table formatting constants
+pub const MIN_COLUMN_WIDTH: usize = 3;
+
+/// Compute a hash string from input data
+pub fn compute_hash(data: &str) -> String {
+    let mut hasher = AHasher::default();
+    data.hash(&mut hasher);
+    let hash = hasher.finish();
+    format!("{:016x}", hash)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_compute_hash_deterministic() {
+        let input = "test_string";
+        let hash1 = compute_hash(input);
+        let hash2 = compute_hash(input);
+        assert_eq!(hash1, hash2);
+        assert_eq!(hash1.len(), 16);
+    }
+
+    #[test]
+    fn test_compute_hash_different_inputs() {
+        let hash1 = compute_hash("test1");
+        let hash2 = compute_hash("test2");
+        assert_ne!(hash1, hash2);
+    }
+
+    #[test]
+    fn test_compute_hash_empty() {
+        let hash = compute_hash("");
+        assert_eq!(hash.len(), 16);
+    }
+}

data/vendor/kreuzberg/src/ocr/validation.rs

@@ -1,206 +1,206 @@
-use super::error::OcrError;
-use super::utils::MINIMAL_SUPPORTED_TESSERACT_VERSION;
-use std::collections::HashSet;
-
-lazy_static::lazy_static! {
-    pub static ref TESSERACT_SUPPORTED_LANGUAGE_CODES: HashSet<&'static str> = {
-        let mut set = HashSet::new();
-        set.insert("afr");
-        set.insert("amh");
-        set.insert("ara");
-        set.insert("asm");
-        set.insert("aze");
-        set.insert("aze_cyrl");
-        set.insert("bel");
-        set.insert("ben");
-        set.insert("bod");
-        set.insert("bos");
-        set.insert("bre");
-        set.insert("bul");
-        set.insert("cat");
-        set.insert("ceb");
-        set.insert("ces");
-        set.insert("chi_sim");
-        set.insert("chi_tra");
-        set.insert("chr");
-        set.insert("cos");
-        set.insert("cym");
-        set.insert("dan");
-        set.insert("deu");
-        set.insert("div");
-        set.insert("dzo");
-        set.insert("ell");
-        set.insert("eng");
-        set.insert("enm");
-        set.insert("epo");
-        set.insert("equ");
-        set.insert("est");
-        set.insert("eus");
-        set.insert("fao");
-        set.insert("fas");
-        set.insert("fil");
-        set.insert("fin");
-        set.insert("fra");
-        set.insert("frk");
-        set.insert("frm");
-        set.insert("fry");
-        set.insert("gla");
-        set.insert("gle");
-        set.insert("glg");
-        set.insert("grc");
-        set.insert("guj");
-        set.insert("hat");
-        set.insert("heb");
-        set.insert("hin");
-        set.insert("hrv");
-        set.insert("hun");
-        set.insert("hye");
-        set.insert("iku");
-        set.insert("ind");
-        set.insert("isl");
-        set.insert("ita");
-        set.insert("ita_old");
-        set.insert("jav");
-        set.insert("jpn");
-        set.insert("kan");
-        set.insert("kat");
-        set.insert("kat_old");
-        set.insert("kaz");
-        set.insert("khm");
-        set.insert("kir");
-        set.insert("kmr");
-        set.insert("kor");
-        set.insert("lao");
-        set.insert("lat");
-        set.insert("lav");
-        set.insert("lit");
-        set.insert("ltz");
-        set.insert("mal");
-        set.insert("mar");
-        set.insert("mkd");
-        set.insert("mlt");
-        set.insert("mon");
-        set.insert("mri");
-        set.insert("msa");
-        set.insert("mya");
-        set.insert("nep");
-        set.insert("nld");
-        set.insert("nor");
-        set.insert("oci");
-        set.insert("ori");
-        set.insert("osd");
-        set.insert("pan");
-        set.insert("pol");
-        set.insert("por");
-        set.insert("pus");
-        set.insert("que");
-        set.insert("ron");
-        set.insert("rus");
-        set.insert("san");
-        set.insert("sin");
-        set.insert("slk");
-        set.insert("slv");
-        set.insert("snd");
-        set.insert("spa");
-        set.insert("spa_old");
-        set.insert("sqi");
-        set.insert("srp");
-        set.insert("srp_latn");
-        set.insert("sun");
-        set.insert("swa");
-        set.insert("swe");
-        set.insert("syr");
-        set.insert("tam");
-        set.insert("tat");
-        set.insert("tel");
-        set.insert("tgk");
-        set.insert("tha");
-        set.insert("tir");
-        set.insert("ton");
-        set.insert("tur");
-        set.insert("uig");
-        set.insert("ukr");
-        set.insert("urd");
-        set.insert("uzb");
-        set.insert("uzb_cyrl");
-        set.insert("vie");
-        set.insert("yid");
-        set.insert("yor");
-        set
-    };
-}
-
-pub fn validate_language_code(lang_code: &str) -> Result<(), OcrError> {
-    for code in lang_code.split('+') {
-        if !TESSERACT_SUPPORTED_LANGUAGE_CODES.contains(code) {
-            return Err(OcrError::InvalidLanguageCode(format!(
-                "Language code '{}' is not supported by Tesseract",
-                code
-            )));
-        }
-    }
-    Ok(())
-}
-
-pub fn validate_tesseract_version(version: u32) -> Result<(), OcrError> {
-    if version < MINIMAL_SUPPORTED_TESSERACT_VERSION {
-        return Err(OcrError::UnsupportedVersion(format!(
-            "Tesseract version {} is not supported. Minimum required version is {}",
-            version, MINIMAL_SUPPORTED_TESSERACT_VERSION
-        )));
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_validate_language_code_valid() {
-        assert!(validate_language_code("eng").is_ok());
-        assert!(validate_language_code("fra").is_ok());
-        assert!(validate_language_code("deu").is_ok());
-        assert!(validate_language_code("chi_sim").is_ok());
-    }
-
-    #[test]
-    fn test_validate_language_code_multiple() {
-        assert!(validate_language_code("eng+fra").is_ok());
-        assert!(validate_language_code("eng+fra+deu").is_ok());
-    }
-
-    #[test]
-    fn test_validate_language_code_invalid() {
-        let result = validate_language_code("invalid_lang");
-        assert!(result.is_err());
-        assert!(matches!(result.unwrap_err(), OcrError::InvalidLanguageCode(_)));
-    }
-
-    #[test]
-    fn test_validate_language_code_mixed_valid_invalid() {
-        let result = validate_language_code("eng+invalid");
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_validate_tesseract_version_valid() {
-        assert!(validate_tesseract_version(5).is_ok());
-        assert!(validate_tesseract_version(6).is_ok());
-    }
-
-    #[test]
-    fn test_validate_tesseract_version_invalid() {
-        let result = validate_tesseract_version(4);
-        assert!(result.is_err());
-        assert!(matches!(result.unwrap_err(), OcrError::UnsupportedVersion(_)));
-    }
-
-    #[test]
-    fn test_language_codes_exist() {
-        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("eng"));
-        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("fra"));
-        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("chi_sim"));
-        assert!(!TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("fake"));
-    }
-}
+use super::error::OcrError;
+use super::utils::MINIMAL_SUPPORTED_TESSERACT_VERSION;
+use std::collections::HashSet;
+
+lazy_static::lazy_static! {
+    pub static ref TESSERACT_SUPPORTED_LANGUAGE_CODES: HashSet<&'static str> = {
+        let mut set = HashSet::new();
+        set.insert("afr");
+        set.insert("amh");
+        set.insert("ara");
+        set.insert("asm");
+        set.insert("aze");
+        set.insert("aze_cyrl");
+        set.insert("bel");
+        set.insert("ben");
+        set.insert("bod");
+        set.insert("bos");
+        set.insert("bre");
+        set.insert("bul");
+        set.insert("cat");
+        set.insert("ceb");
+        set.insert("ces");
+        set.insert("chi_sim");
+        set.insert("chi_tra");
+        set.insert("chr");
+        set.insert("cos");
+        set.insert("cym");
+        set.insert("dan");
+        set.insert("deu");
+        set.insert("div");
+        set.insert("dzo");
+        set.insert("ell");
+        set.insert("eng");
+        set.insert("enm");
+        set.insert("epo");
+        set.insert("equ");
+        set.insert("est");
+        set.insert("eus");
+        set.insert("fao");
+        set.insert("fas");
+        set.insert("fil");
+        set.insert("fin");
+        set.insert("fra");
+        set.insert("frk");
+        set.insert("frm");
+        set.insert("fry");
+        set.insert("gla");
+        set.insert("gle");
+        set.insert("glg");
+        set.insert("grc");
+        set.insert("guj");
+        set.insert("hat");
+        set.insert("heb");
+        set.insert("hin");
+        set.insert("hrv");
+        set.insert("hun");
+        set.insert("hye");
+        set.insert("iku");
+        set.insert("ind");
+        set.insert("isl");
+        set.insert("ita");
+        set.insert("ita_old");
+        set.insert("jav");
+        set.insert("jpn");
+        set.insert("kan");
+        set.insert("kat");
+        set.insert("kat_old");
+        set.insert("kaz");
+        set.insert("khm");
+        set.insert("kir");
+        set.insert("kmr");
+        set.insert("kor");
+        set.insert("lao");
+        set.insert("lat");
+        set.insert("lav");
+        set.insert("lit");
+        set.insert("ltz");
+        set.insert("mal");
+        set.insert("mar");
+        set.insert("mkd");
+        set.insert("mlt");
+        set.insert("mon");
+        set.insert("mri");
+        set.insert("msa");
+        set.insert("mya");
+        set.insert("nep");
+        set.insert("nld");
+        set.insert("nor");
+        set.insert("oci");
+        set.insert("ori");
+        set.insert("osd");
+        set.insert("pan");
+        set.insert("pol");
+        set.insert("por");
+        set.insert("pus");
+        set.insert("que");
+        set.insert("ron");
+        set.insert("rus");
+        set.insert("san");
+        set.insert("sin");
+        set.insert("slk");
+        set.insert("slv");
+        set.insert("snd");
+        set.insert("spa");
+        set.insert("spa_old");
+        set.insert("sqi");
+        set.insert("srp");
+        set.insert("srp_latn");
+        set.insert("sun");
+        set.insert("swa");
+        set.insert("swe");
+        set.insert("syr");
+        set.insert("tam");
+        set.insert("tat");
+        set.insert("tel");
+        set.insert("tgk");
+        set.insert("tha");
+        set.insert("tir");
+        set.insert("ton");
+        set.insert("tur");
+        set.insert("uig");
+        set.insert("ukr");
+        set.insert("urd");
+        set.insert("uzb");
+        set.insert("uzb_cyrl");
+        set.insert("vie");
+        set.insert("yid");
+        set.insert("yor");
+        set
+    };
+}
+
+pub fn validate_language_code(lang_code: &str) -> Result<(), OcrError> {
+    for code in lang_code.split('+') {
+        if !TESSERACT_SUPPORTED_LANGUAGE_CODES.contains(code) {
+            return Err(OcrError::InvalidLanguageCode(format!(
+                "Language code '{}' is not supported by Tesseract",
+                code
+            )));
+        }
+    }
+    Ok(())
+}
+
+pub fn validate_tesseract_version(version: u32) -> Result<(), OcrError> {
+    if version < MINIMAL_SUPPORTED_TESSERACT_VERSION {
+        return Err(OcrError::UnsupportedVersion(format!(
+            "Tesseract version {} is not supported. Minimum required version is {}",
+            version, MINIMAL_SUPPORTED_TESSERACT_VERSION
+        )));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_validate_language_code_valid() {
+        assert!(validate_language_code("eng").is_ok());
+        assert!(validate_language_code("fra").is_ok());
+        assert!(validate_language_code("deu").is_ok());
+        assert!(validate_language_code("chi_sim").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_multiple() {
+        assert!(validate_language_code("eng+fra").is_ok());
+        assert!(validate_language_code("eng+fra+deu").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_invalid() {
+        let result = validate_language_code("invalid_lang");
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), OcrError::InvalidLanguageCode(_)));
+    }
+
+    #[test]
+    fn test_validate_language_code_mixed_valid_invalid() {
+        let result = validate_language_code("eng+invalid");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_validate_tesseract_version_valid() {
+        assert!(validate_tesseract_version(5).is_ok());
+        assert!(validate_tesseract_version(6).is_ok());
+    }
+
+    #[test]
+    fn test_validate_tesseract_version_invalid() {
+        let result = validate_tesseract_version(4);
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), OcrError::UnsupportedVersion(_)));
+    }
+
+    #[test]
+    fn test_language_codes_exist() {
+        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("eng"));
+        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("fra"));
+        assert!(TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("chi_sim"));
+        assert!(!TESSERACT_SUPPORTED_LANGUAGE_CODES.contains("fake"));
+    }
+}

data/vendor/kreuzberg/src/panic_context.rs

@@ -0,0 +1,154 @@
+use std::any::Any;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+/// Context information captured when a panic occurs.
+///
+/// This struct stores detailed information about where and when a panic happened,
+/// enabling better error reporting across FFI boundaries.
+#[derive(Debug, Clone)]
+pub struct PanicContext {
+    /// Source file where the panic occurred
+    pub file: &'static str,
+    /// Line number where the panic occurred
+    pub line: u32,
+    /// Function name where the panic occurred
+    pub function: &'static str,
+    /// Panic message extracted from the panic payload
+    pub message: String,
+    /// Timestamp when the panic was captured
+    pub timestamp: SystemTime,
+}
+
+impl PanicContext {
+    /// Creates a new PanicContext with the given parameters.
+    ///
+    /// # Arguments
+    ///
+    /// * `file` - Source file path
+    /// * `line` - Line number
+    /// * `function` - Function name
+    /// * `panic_info` - The panic payload to extract message from
+    pub fn new(file: &'static str, line: u32, function: &'static str, panic_info: &dyn Any) -> Self {
+        let timestamp = std::panic::catch_unwind(SystemTime::now).unwrap_or(UNIX_EPOCH);
+
+        Self {
+            file,
+            line,
+            function,
+            message: extract_panic_message(panic_info),
+            timestamp,
+        }
+    }
+
+    /// Formats the panic context as a human-readable string.
+    pub fn format(&self) -> String {
+        format!(
+            "Panic at {}:{}:{} - {}",
+            self.file, self.line, self.function, self.message
+        )
+    }
+}
+
+/// Maximum panic message length to prevent DoS attacks
+const MAX_PANIC_MESSAGE_LEN: usize = 4096;
+
+/// Extracts a human-readable message from a panic payload.
+///
+/// Attempts to downcast the panic payload to common types (String, &str)
+/// to extract a meaningful error message.
+///
+/// Message is truncated to 4KB to prevent DoS attacks via extremely large panic messages.
+///
+/// # Arguments
+///
+/// * `panic_info` - The panic payload from catch_unwind
+///
+/// # Returns
+///
+/// A string representation of the panic message (truncated if necessary)
+pub fn extract_panic_message(panic_info: &dyn Any) -> String {
+    let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
+        s.clone()
+    } else if let Some(s) = panic_info.downcast_ref::<&str>() {
+        (*s).to_string()
+    } else {
+        "Unknown panic payload".to_string()
+    };
+
+    if msg.len() > MAX_PANIC_MESSAGE_LEN {
+        let truncate_at = msg.floor_char_boundary(MAX_PANIC_MESSAGE_LEN);
+        format!("{}... [truncated]", &msg[..truncate_at])
+    } else {
+        msg
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_panic_message_string() {
+        let panic_msg = "test panic".to_string();
+        let msg = extract_panic_message(&panic_msg);
+        assert_eq!(msg, "test panic");
+    }
+
+    #[test]
+    fn test_extract_panic_message_str() {
+        let panic_msg: &str = "test panic";
+        let msg = extract_panic_message(&panic_msg);
+        assert_eq!(msg, "test panic");
+    }
+
+    #[test]
+    fn test_extract_panic_message_unknown() {
+        let panic_msg = 42i32;
+        let msg = extract_panic_message(&panic_msg);
+        assert_eq!(msg, "Unknown panic payload");
+    }
+
+    #[test]
+    fn test_panic_context_format() {
+        let panic_msg = "test error".to_string();
+        let ctx = PanicContext::new("test.rs", 42, "test_function", &panic_msg);
+
+        let formatted = ctx.format();
+        assert!(formatted.contains("test.rs"));
+        assert!(formatted.contains("42"));
+        assert!(formatted.contains("test_function"));
+        assert!(formatted.contains("test error"));
+    }
+
+    #[test]
+    fn test_panic_message_truncation() {
+        let long_msg = "x".repeat(5000);
+        let msg = extract_panic_message(&long_msg);
+        assert!(msg.len() <= MAX_PANIC_MESSAGE_LEN + 20);
+        assert!(msg.ends_with("... [truncated]"));
+    }
+
+    #[test]
+    fn test_panic_message_truncation_utf8_boundary() {
+        let mut msg = "x".repeat(4093);
+        msg.push('🦀');
+        msg.push_str("yyy");
+
+        let truncated = extract_panic_message(&msg);
+
+        assert!(truncated.ends_with("... [truncated]"));
+
+        assert!(std::str::from_utf8(truncated.as_bytes()).is_ok());
+
+        assert!(!truncated.contains("🦀"));
+        assert!(!truncated.contains("yyy"));
+    }
+
+    #[test]
+    fn test_panic_message_no_truncation_needed() {
+        let short_msg = "short".to_string();
+        let msg = extract_panic_message(&short_msg);
+        assert_eq!(msg, "short");
+        assert!(!msg.contains("[truncated]"));
+    }
+}