kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/text/string_utils.rs

@@ -0,0 +1,217 @@
+use chardetng::EncodingDetector;
+use encoding_rs::Encoding;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::sync::RwLock;
+
+// ============================================================================
+
+static CONTROL_CHARS: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F-\x9F]")
+        .expect("Control chars regex pattern is valid and should compile")
+});
+static REPLACEMENT_CHARS: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"\u{FFFD}+").expect("Replacement chars regex pattern is valid and should compile"));
+static ISOLATED_COMBINING: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[\u{0300}-\u{036F}]+")
+        .expect("Isolated combining diacritics regex pattern is valid and should compile")
+});
+static HEBREW_AS_CYRILLIC: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[\u{0400}-\u{04FF}]{3,}")
+        .expect("Hebrew misencoded as Cyrillic regex pattern is valid and should compile")
+});
+
+static ENCODING_CACHE: Lazy<RwLock<HashMap<String, &'static Encoding>>> = Lazy::new(|| RwLock::new(HashMap::new()));
+
+const CACHE_SIZE_LIMIT: usize = 1000;
+
+#[inline]
+fn chain_replacements<'a>(mut text: Cow<'a, str>, replacements: &[(&Regex, &str)]) -> Cow<'a, str> {
+    for (pattern, replacement) in replacements {
+        if pattern.is_match(&text) {
+            text = Cow::Owned(pattern.replace_all(&text, *replacement).into_owned());
+        }
+    }
+    text
+}
+
+fn calculate_cache_key(data: &[u8]) -> String {
+    use std::collections::hash_map::DefaultHasher;
+    use std::hash::{Hash, Hasher};
+
+    let mut hasher = DefaultHasher::new();
+    let sample = if data.len() > 1024 { &data[..1024] } else { data };
+    sample.hash(&mut hasher);
+    data.len().hash(&mut hasher);
+    format!("{:x}", hasher.finish())
+}
+
+pub fn safe_decode(byte_data: &[u8], encoding: Option<&str>) -> String {
+    if byte_data.is_empty() {
+        return String::new();
+    }
+
+    if let Some(enc_name) = encoding
+        && let Some(enc) = Encoding::for_label(enc_name.as_bytes())
+    {
+        let (decoded, _, _) = enc.decode(byte_data);
+        return fix_mojibake_internal(&decoded);
+    }
+
+    let cache_key = calculate_cache_key(byte_data);
+
+    if let Ok(cache) = ENCODING_CACHE.read()
+        && let Some(&cached_encoding) = cache.get(&cache_key)
+    {
+        let (decoded, _, _) = cached_encoding.decode(byte_data);
+        return fix_mojibake_internal(&decoded);
+    }
+
+    let mut detector = EncodingDetector::new();
+    detector.feed(byte_data, true);
+    let encoding = detector.guess(None, true);
+
+    if let Ok(mut cache) = ENCODING_CACHE.write()
+        && cache.len() < CACHE_SIZE_LIMIT
+    {
+        cache.insert(cache_key, encoding);
+    }
+
+    let (decoded, _, had_errors) = encoding.decode(byte_data);
+
+    if had_errors {
+        for enc_name in &[
+            "windows-1255",
+            "iso-8859-8",
+            "windows-1256",
+            "iso-8859-6",
+            "windows-1252",
+            "cp1251",
+        ] {
+            if let Some(enc) = Encoding::for_label(enc_name.as_bytes()) {
+                let (test_decoded, _, test_errors) = enc.decode(byte_data);
+                if !test_errors && calculate_text_confidence_internal(&test_decoded) > 0.5 {
+                    return fix_mojibake_internal(&test_decoded);
+                }
+            }
+        }
+    }
+
+    fix_mojibake_internal(&decoded)
+}
+
+pub fn get_encoding_cache_key(data_hash: &str, size: usize) -> String {
+    format!("{}:{}", data_hash, size)
+}
+
+pub fn calculate_text_confidence(text: &str) -> f64 {
+    calculate_text_confidence_internal(text)
+}
+
+fn calculate_text_confidence_internal(text: &str) -> f64 {
+    if text.is_empty() {
+        return 0.0;
+    }
+
+    let total_chars = text.len() as f64;
+
+    let replacement_count = REPLACEMENT_CHARS.find_iter(text).count() as f64;
+    let control_count = CONTROL_CHARS.find_iter(text).count() as f64;
+
+    let penalty = (replacement_count + control_count * 2.0) / total_chars;
+
+    let readable_chars = text
+        .chars()
+        .filter(|c| c.is_ascii_graphic() || c.is_whitespace())
+        .count() as f64;
+
+    let readability_score = readable_chars / total_chars;
+
+    let cyrillic_matches = HEBREW_AS_CYRILLIC.find_iter(text);
+    let cyrillic_length: usize = cyrillic_matches.map(|m| m.len()).sum();
+
+    let mut final_penalty = penalty;
+    if cyrillic_length as f64 > total_chars * 0.1 {
+        final_penalty += 0.3;
+    }
+
+    (readability_score - final_penalty).clamp(0.0, 1.0)
+}
+
+pub fn fix_mojibake(text: &str) -> String {
+    fix_mojibake_internal(text)
+}
+
+fn fix_mojibake_internal(text: &str) -> String {
+    if text.is_empty() {
+        return text.to_string();
+    }
+
+    let replacements = [
+        (&*CONTROL_CHARS, ""),
+        (&*REPLACEMENT_CHARS, ""),
+        (&*ISOLATED_COMBINING, ""),
+    ];
+
+    chain_replacements(Cow::Borrowed(text), &replacements).into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_safe_decode_empty() {
+        assert_eq!(safe_decode(b"", None), "");
+    }
+
+    #[test]
+    fn test_safe_decode_ascii() {
+        let text = b"Hello, World!";
+        assert_eq!(safe_decode(text, None), "Hello, World!");
+    }
+
+    #[test]
+    fn test_safe_decode_utf8() {
+        let text = "Hello, 世界! مرحبا".as_bytes();
+        assert_eq!(safe_decode(text, None), "Hello, 世界! مرحبا");
+    }
+
+    #[test]
+    fn test_calculate_text_confidence_empty() {
+        assert_eq!(calculate_text_confidence(""), 0.0);
+    }
+
+    #[test]
+    fn test_calculate_text_confidence_clean_text() {
+        let text = "This is clean, readable text without any issues.";
+        let confidence = calculate_text_confidence(text);
+        assert!(confidence > 0.9);
+    }
+
+    #[test]
+    fn test_fix_mojibake_empty() {
+        assert_eq!(fix_mojibake(""), "");
+    }
+
+    #[test]
+    fn test_fix_mojibake_clean_text() {
+        let text = "Clean text without mojibake";
+        assert_eq!(fix_mojibake(text), text);
+    }
+
+    #[test]
+    fn test_fix_mojibake_control_chars() {
+        let text = "Text\x00with\x01control\x1Fchars";
+        let fixed = fix_mojibake(text);
+        assert_eq!(fixed, "Textwithcontrolchars");
+    }
+
+    #[test]
+    fn test_get_encoding_cache_key() {
+        let key = get_encoding_cache_key("hash123", 1024);
+        assert_eq!(key, "hash123:1024");
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs

@@ -0,0 +1,164 @@
+use std::ops::RangeInclusive;
+
+/// CJK text tokenizer for token reduction.
+///
+/// This tokenizer uses bigram (2-character) tokenization for CJK text,
+/// which is appropriate for token reduction where we want to preserve
+/// meaning while reducing token count.
+///
+/// # Unicode Range Coverage
+///
+/// **Currently covers:** CJK Unified Ideographs (U+4E00-U+9FFF)
+/// - Covers ~20,992 common Chinese/Japanese Kanji characters
+/// - Sufficient for token reduction purposes with Chinese and Japanese text
+///
+/// **Intentionally excluded:**
+/// - Hiragana (U+3040-U+309F): Japanese phonetic script
+/// - Katakana (U+30A0-U+30FF): Japanese phonetic script
+/// - Hangul (U+AC00-U+D7AF): Korean alphabet
+///
+/// These exclusions are intentional for token reduction. Hiragana and Katakana
+/// are typically tokenized with whitespace, and Hangul has different tokenization
+/// requirements. If broader CJK support is needed, consider expanding the range
+/// or using language-specific tokenizers.
+pub struct CjkTokenizer {
+    cjk_range: RangeInclusive<u32>,
+}
+
+impl CjkTokenizer {
+    pub fn new() -> Self {
+        Self {
+            cjk_range: 0x4E00..=0x9FFF,
+        }
+    }
+
+    /// Checks if a character is a CJK Unified Ideograph (U+4E00-U+9FFF).
+    ///
+    /// Returns true for Chinese characters and Japanese Kanji, false for
+    /// Hiragana, Katakana, Hangul, and non-CJK characters.
+    #[inline]
+    pub fn is_cjk_char(&self, c: char) -> bool {
+        self.cjk_range.contains(&(c as u32))
+    }
+
+    #[inline]
+    pub fn has_cjk(&self, text: &str) -> bool {
+        text.chars().any(|c| self.is_cjk_char(c))
+    }
+
+    pub fn tokenize_cjk_string(&self, text: &str) -> Vec<String> {
+        let chars: Vec<char> = text.chars().collect();
+        self.tokenize_cjk_chars(&chars)
+    }
+
+    pub fn tokenize_cjk_chars(&self, chars: &[char]) -> Vec<String> {
+        chars
+            .chunks(2)
+            .map(|chunk| {
+                if chunk.len() == 2 {
+                    format!("{}{}", chunk[0], chunk[1])
+                } else {
+                    chunk[0].to_string()
+                }
+            })
+            .collect()
+    }
+
+    pub fn tokenize_mixed_text(&self, text: &str) -> Vec<String> {
+        let whitespace_tokens: Vec<&str> = text.split_whitespace().collect();
+
+        if whitespace_tokens.is_empty() {
+            return if text.is_empty() {
+                vec![]
+            } else {
+                vec![text.to_string()]
+            };
+        }
+
+        if whitespace_tokens.len() == 1 {
+            let token = whitespace_tokens[0];
+            return if self.has_cjk(token) {
+                self.tokenize_cjk_string(token)
+            } else {
+                vec![token.to_string()]
+            };
+        }
+
+        let mut all_tokens = Vec::new();
+        for token in whitespace_tokens {
+            if self.has_cjk(token) {
+                all_tokens.extend(self.tokenize_cjk_string(token));
+            } else {
+                all_tokens.push(token.to_string());
+            }
+        }
+        all_tokens
+    }
+}
+
+impl Default for CjkTokenizer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_cjk_char() {
+        let tokenizer = CjkTokenizer::new();
+
+        assert!(tokenizer.is_cjk_char('中'));
+        assert!(tokenizer.is_cjk_char('国'));
+        assert!(tokenizer.is_cjk_char('日'));
+        assert!(tokenizer.is_cjk_char('本'));
+
+        assert!(!tokenizer.is_cjk_char('a'));
+        assert!(!tokenizer.is_cjk_char('Z'));
+        assert!(!tokenizer.is_cjk_char('1'));
+        assert!(!tokenizer.is_cjk_char(' '));
+    }
+
+    #[test]
+    fn test_has_cjk() {
+        let tokenizer = CjkTokenizer::new();
+
+        assert!(tokenizer.has_cjk("这是中文"));
+        assert!(tokenizer.has_cjk("mixed 中文 text"));
+        assert!(tokenizer.has_cjk("日本語"));
+
+        assert!(!tokenizer.has_cjk("English text"));
+        assert!(!tokenizer.has_cjk("12345"));
+        assert!(!tokenizer.has_cjk(""));
+    }
+
+    #[test]
+    fn test_tokenize_cjk_string() {
+        let tokenizer = CjkTokenizer::new();
+
+        let tokens = tokenizer.tokenize_cjk_string("中国人");
+        assert_eq!(tokens, vec!["中国", "人"]);
+
+        let tokens = tokenizer.tokenize_cjk_string("四个字");
+        assert_eq!(tokens, vec!["四个", "字"]);
+    }
+
+    #[test]
+    fn test_tokenize_mixed_text() {
+        let tokenizer = CjkTokenizer::new();
+
+        let tokens = tokenizer.tokenize_mixed_text("hello world");
+        assert_eq!(tokens, vec!["hello", "world"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("中国");
+        assert_eq!(tokens, vec!["中国"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("hello 中国 world");
+        assert_eq!(tokens, vec!["hello", "中国", "world"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("学习 machine learning 技术");
+        assert_eq!(tokens, vec!["学习", "machine", "learning", "技术"]);
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/config.rs

@@ -0,0 +1,100 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum ReductionLevel {
+    Off = 0,
+    Light = 1,
+    Moderate = 2,
+    Aggressive = 3,
+    Maximum = 4,
+}
+
+impl ReductionLevel {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReductionLevel::Off => "off",
+            ReductionLevel::Light => "light",
+            ReductionLevel::Moderate => "moderate",
+            ReductionLevel::Aggressive => "aggressive",
+            ReductionLevel::Maximum => "maximum",
+        }
+    }
+}
+
+impl From<&str> for ReductionLevel {
+    fn from(s: &str) -> Self {
+        match s.to_lowercase().as_str() {
+            "off" => ReductionLevel::Off,
+            "light" => ReductionLevel::Light,
+            "moderate" => ReductionLevel::Moderate,
+            "aggressive" => ReductionLevel::Aggressive,
+            "maximum" => ReductionLevel::Maximum,
+            _ => ReductionLevel::Moderate,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct TokenReductionConfig {
+    pub level: ReductionLevel,
+    pub language_hint: Option<String>,
+    pub preserve_markdown: bool,
+    pub preserve_code: bool,
+    pub semantic_threshold: f32,
+    pub enable_parallel: bool,
+    pub use_simd: bool,
+    pub custom_stopwords: Option<HashMap<String, Vec<String>>>,
+    pub preserve_patterns: Vec<String>,
+    pub target_reduction: Option<f32>,
+    pub enable_semantic_clustering: bool,
+}
+
+impl Default for TokenReductionConfig {
+    fn default() -> Self {
+        Self {
+            level: ReductionLevel::Moderate,
+            language_hint: None,
+            preserve_markdown: false,
+            preserve_code: true,
+            semantic_threshold: 0.3,
+            enable_parallel: true,
+            use_simd: true,
+            custom_stopwords: None,
+            preserve_patterns: vec![],
+            target_reduction: None,
+            enable_semantic_clustering: false,
+        }
+    }
+}
+
+impl TokenReductionConfig {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        level: ReductionLevel,
+        language_hint: Option<String>,
+        preserve_markdown: bool,
+        preserve_code: bool,
+        semantic_threshold: f32,
+        enable_parallel: bool,
+        use_simd: bool,
+        custom_stopwords: Option<HashMap<String, Vec<String>>>,
+        preserve_patterns: Option<Vec<String>>,
+        target_reduction: Option<f32>,
+        enable_semantic_clustering: bool,
+    ) -> Self {
+        Self {
+            level,
+            language_hint,
+            preserve_markdown,
+            preserve_code,
+            semantic_threshold: semantic_threshold.clamp(0.0, 1.0),
+            enable_parallel,
+            use_simd,
+            custom_stopwords,
+            preserve_patterns: preserve_patterns.unwrap_or_default(),
+            target_reduction: target_reduction.map(|t| t.clamp(0.0, 1.0)),
+            enable_semantic_clustering,
+        }
+    }
+}