kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/text/token_reduction/core.rs

@@ -0,0 +1,796 @@
+use crate::error::Result;
+use crate::text::token_reduction::{
+    cjk_utils::CjkTokenizer,
+    config::{ReductionLevel, TokenReductionConfig},
+    filters::FilterPipeline,
+    semantic::SemanticAnalyzer,
+    simd_text::{SimdTextProcessor, chunk_text_for_parallel},
+};
+use once_cell::sync::Lazy;
+use rayon::prelude::*;
+use regex::Regex;
+use std::sync::Arc;
+use unicode_normalization::UnicodeNormalization;
+
+static REPEATED_EXCLAMATION: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[!]{2,}").expect("Repeated exclamation regex pattern is valid and should compile"));
+static REPEATED_QUESTION: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[?]{2,}").expect("Repeated question regex pattern is valid and should compile"));
+static REPEATED_COMMA: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[,]{2,}").expect("Repeated comma regex pattern is valid and should compile"));
+
+/// Bonus added for sentences at the beginning or end of the document
+const SENTENCE_EDGE_POSITION_BONUS: f32 = 0.3;
+
+/// Bonus added for sentences with ideal word count (neither too short nor too long)
+const IDEAL_WORD_COUNT_BONUS: f32 = 0.2;
+
+/// Minimum word count for ideal sentence length
+const MIN_IDEAL_WORD_COUNT: usize = 3;
+
+/// Maximum word count for ideal sentence length
+const MAX_IDEAL_WORD_COUNT: usize = 25;
+
+/// Weight multiplier for numeric content density in sentences
+const NUMERIC_CONTENT_WEIGHT: f32 = 0.3;
+
+/// Weight multiplier for capitalized/acronym word density in sentences
+const CAPS_ACRONYM_WEIGHT: f32 = 0.25;
+
+/// Weight multiplier for long word density in sentences
+const LONG_WORD_WEIGHT: f32 = 0.2;
+
+/// Minimum character length for a word to be considered "long"
+const LONG_WORD_THRESHOLD: usize = 8;
+
+/// Weight multiplier for punctuation density in sentences
+const PUNCTUATION_DENSITY_WEIGHT: f32 = 0.15;
+
+/// Weight multiplier for word diversity ratio (unique words / total words)
+const DIVERSITY_RATIO_WEIGHT: f32 = 0.15;
+
+/// Weight multiplier for character entropy (measure of text randomness/information)
+const CHAR_ENTROPY_WEIGHT: f32 = 0.1;
+
+pub struct TokenReducer {
+    config: Arc<TokenReductionConfig>,
+    text_processor: SimdTextProcessor,
+    filter_pipeline: FilterPipeline,
+    semantic_analyzer: Option<SemanticAnalyzer>,
+    cjk_tokenizer: CjkTokenizer,
+    language: String,
+}
+
+impl TokenReducer {
+    pub fn new(config: &TokenReductionConfig, language_hint: Option<&str>) -> Result<Self> {
+        let config = Arc::new(config.clone());
+        let language = language_hint
+            .or(config.language_hint.as_deref())
+            .unwrap_or("en")
+            .to_string();
+
+        let text_processor = SimdTextProcessor::new();
+        let filter_pipeline = FilterPipeline::new(&config, &language)?;
+
+        let semantic_analyzer = if matches!(config.level, ReductionLevel::Aggressive | ReductionLevel::Maximum) {
+            Some(SemanticAnalyzer::new(&language))
+        } else {
+            None
+        };
+
+        Ok(Self {
+            config,
+            text_processor,
+            filter_pipeline,
+            semantic_analyzer,
+            cjk_tokenizer: CjkTokenizer::new(),
+            language,
+        })
+    }
+
+    /// Get the language code being used for stopwords and semantic analysis.
+    pub fn language(&self) -> &str {
+        &self.language
+    }
+
+    pub fn reduce(&self, text: &str) -> String {
+        if text.is_empty() || matches!(self.config.level, ReductionLevel::Off) {
+            return text.to_string();
+        }
+
+        let working_text = if text.is_ascii() {
+            text
+        } else {
+            &text.nfc().collect::<String>()
+        };
+
+        match self.config.level {
+            ReductionLevel::Off => working_text.to_string(),
+            ReductionLevel::Light => self.apply_light_reduction_optimized(working_text),
+            ReductionLevel::Moderate => self.apply_moderate_reduction_optimized(working_text),
+            ReductionLevel::Aggressive => self.apply_aggressive_reduction_optimized(working_text),
+            ReductionLevel::Maximum => self.apply_maximum_reduction_optimized(working_text),
+        }
+    }
+
+    pub fn batch_reduce(&self, texts: &[&str]) -> Vec<String> {
+        if !self.config.enable_parallel || texts.len() < 2 {
+            return texts.iter().map(|text| self.reduce(text)).collect();
+        }
+
+        texts.par_iter().map(|text| self.reduce(text)).collect()
+    }
+
+    fn apply_light_reduction_optimized(&self, text: &str) -> String {
+        let mut result = if self.config.use_simd {
+            self.text_processor.clean_punctuation(text)
+        } else {
+            self.clean_punctuation_optimized(text)
+        };
+
+        result = self.filter_pipeline.apply_light_filters(&result);
+        result.trim().to_string()
+    }
+
+    fn apply_moderate_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_light_reduction_optimized(text);
+
+        result = if self.config.enable_parallel && text.len() > 1000 {
+            self.apply_parallel_moderate_reduction(&result)
+        } else {
+            self.filter_pipeline.apply_moderate_filters(&result)
+        };
+
+        result
+    }
+
+    fn apply_aggressive_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_moderate_reduction_optimized(text);
+
+        result = self.remove_additional_common_words(&result);
+        result = self.apply_sentence_selection(&result);
+
+        if let Some(ref analyzer) = self.semantic_analyzer {
+            result = analyzer.apply_semantic_filtering(&result, self.config.semantic_threshold);
+        }
+
+        result
+    }
+
+    fn apply_maximum_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_aggressive_reduction_optimized(text);
+
+        if let Some(ref analyzer) = self.semantic_analyzer
+            && self.config.enable_semantic_clustering
+        {
+            result = analyzer.apply_hypernym_compression(&result, self.config.target_reduction);
+        }
+
+        result
+    }
+
+    fn apply_parallel_moderate_reduction(&self, text: &str) -> String {
+        let num_threads = rayon::current_num_threads();
+        let chunks = chunk_text_for_parallel(text, num_threads);
+
+        let processed_chunks: Vec<String> = chunks
+            .par_iter()
+            .map(|chunk| self.filter_pipeline.apply_moderate_filters(chunk))
+            .collect();
+
+        processed_chunks.join(" ")
+    }
+
+    fn clean_punctuation_optimized(&self, text: &str) -> String {
+        let mut result = text.to_string();
+
+        result = REPEATED_EXCLAMATION.replace_all(&result, "!").to_string();
+        result = REPEATED_QUESTION.replace_all(&result, "?").to_string();
+        result = REPEATED_COMMA.replace_all(&result, ",").to_string();
+
+        result
+    }
+
+    fn remove_additional_common_words(&self, text: &str) -> String {
+        let words = self.universal_tokenize(text);
+
+        if words.len() < 4 {
+            return text.to_string();
+        }
+
+        let mut word_freq = std::collections::HashMap::new();
+        let mut word_lengths = Vec::new();
+
+        for word in &words {
+            let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                word.to_lowercase()
+            } else {
+                word.chars()
+                    .filter(|c| c.is_alphabetic())
+                    .collect::<String>()
+                    .to_lowercase()
+            };
+
+            if !clean_word.is_empty() {
+                *word_freq.entry(clean_word.clone()).or_insert(0) += 1;
+                word_lengths.push(clean_word.chars().count());
+            }
+        }
+
+        let avg_length = if !word_lengths.is_empty() {
+            word_lengths.iter().sum::<usize>() as f32 / word_lengths.len() as f32
+        } else {
+            5.0
+        };
+
+        let original_count = words.len();
+
+        let filtered_words: Vec<String> = words
+            .iter()
+            .filter(|word| {
+                let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                    word.to_lowercase()
+                } else {
+                    word.chars()
+                        .filter(|c| c.is_alphabetic())
+                        .collect::<String>()
+                        .to_lowercase()
+                };
+
+                if clean_word.is_empty() {
+                    return true;
+                }
+
+                let freq = word_freq.get(&clean_word).unwrap_or(&0);
+                let word_len = clean_word.chars().count() as f32;
+
+                self.has_important_characteristics(word)
+                    || (*freq <= 2 && word_len >= avg_length * 0.8)
+                    || (word_len >= avg_length * 1.5)
+            })
+            .cloned()
+            .collect();
+
+        let has_cjk_content = text.chars().any(|c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
+        let fallback_threshold = if has_cjk_content {
+            original_count / 5
+        } else {
+            original_count / 3
+        };
+
+        if filtered_words.len() < fallback_threshold {
+            let fallback_words: Vec<String> = words
+                .iter()
+                .filter(|word| {
+                    let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                        (*word).clone()
+                    } else {
+                        word.chars().filter(|c| c.is_alphabetic()).collect::<String>()
+                    };
+
+                    clean_word.is_empty() || clean_word.chars().count() >= 3 || self.has_important_characteristics(word)
+                })
+                .cloned()
+                .collect();
+            self.smart_join(&fallback_words, has_cjk_content)
+        } else {
+            self.smart_join(&filtered_words, has_cjk_content)
+        }
+    }
+
+    fn smart_join(&self, tokens: &[String], has_cjk_content: bool) -> String {
+        if has_cjk_content {
+            tokens.join("")
+        } else {
+            tokens.join(" ")
+        }
+    }
+
+    fn has_important_characteristics(&self, word: &str) -> bool {
+        if word.len() > 1 && word.chars().all(|c| c.is_uppercase()) {
+            return true;
+        }
+
+        if word.chars().any(|c| c.is_numeric()) {
+            return true;
+        }
+
+        if word.len() > 10 {
+            return true;
+        }
+
+        let uppercase_count = word.chars().filter(|c| c.is_uppercase()).count();
+        if uppercase_count > 1 && uppercase_count < word.len() {
+            return true;
+        }
+
+        if self.has_cjk_importance(word) {
+            return true;
+        }
+
+        false
+    }
+
+    fn has_cjk_importance(&self, word: &str) -> bool {
+        let chars: Vec<char> = word.chars().collect();
+
+        let has_cjk = chars.iter().any(|&c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
+        if !has_cjk {
+            return false;
+        }
+
+        let important_radicals = [
+            '学', '智', '能', '技', '术', '法', '算', '理', '科', '研', '究', '发', '展', '系', '统', '模', '型', '方',
+            '式', '过', '程', '结', '构', '功', '效', '应', '分', '析', '计', '算', '数', '据', '信', '息', '处', '理',
+            '语', '言', '文', '生', '成', '产', '用', '作', '为', '成', '变', '化', '转', '换', '提', '高', '网', '络',
+            '神', '经', '机', '器', '人', '工', '智', '能', '自', '然', '复',
+        ];
+
+        for &char in &chars {
+            if important_radicals.contains(&char) {
+                return true;
+            }
+        }
+
+        if chars.len() == 2 && has_cjk {
+            let has_technical = chars.iter().any(|&c| {
+                let code = c as u32;
+                (0x4E00..=0x4FFF).contains(&code)
+                    || (0x5000..=0x51FF).contains(&code)
+                    || (0x6700..=0x68FF).contains(&code)
+                    || (0x7500..=0x76FF).contains(&code)
+            });
+
+            if has_technical {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn apply_sentence_selection(&self, text: &str) -> String {
+        let sentences: Vec<&str> = text
+            .split(['.', '!', '?'])
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        if sentences.len() <= 2 {
+            return text.to_string();
+        }
+
+        let mut scored_sentences: Vec<(usize, f32, &str)> = sentences
+            .iter()
+            .enumerate()
+            .map(|(i, sentence)| {
+                let score = self.score_sentence_importance(sentence, i, sentences.len());
+                (i, score, *sentence)
+            })
+            .collect();
+
+        scored_sentences.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+
+        let keep_count = ((sentences.len() as f32 * 0.4).ceil() as usize).max(1);
+        let mut selected_indices: Vec<usize> = scored_sentences[..keep_count].iter().map(|(i, _, _)| *i).collect();
+
+        selected_indices.sort();
+
+        let selected_sentences: Vec<&str> = selected_indices
+            .iter()
+            .filter_map(|&i| sentences.get(i))
+            .copied()
+            .collect();
+
+        if selected_sentences.is_empty() {
+            text.to_string()
+        } else {
+            selected_sentences.join(". ")
+        }
+    }
+
+    fn score_sentence_importance(&self, sentence: &str, position: usize, total_sentences: usize) -> f32 {
+        let mut score = 0.0;
+
+        if position == 0 || position == total_sentences - 1 {
+            score += SENTENCE_EDGE_POSITION_BONUS;
+        }
+
+        let words: Vec<&str> = sentence.split_whitespace().collect();
+        if words.is_empty() {
+            return score;
+        }
+
+        let word_count = words.len();
+        if (MIN_IDEAL_WORD_COUNT..=MAX_IDEAL_WORD_COUNT).contains(&word_count) {
+            score += IDEAL_WORD_COUNT_BONUS;
+        }
+
+        let mut numeric_count = 0;
+        let mut caps_count = 0;
+        let mut long_word_count = 0;
+        let mut punct_density = 0;
+
+        for word in &words {
+            if word.chars().any(|c| c.is_numeric()) {
+                numeric_count += 1;
+            }
+
+            if word.len() > 1 && word.chars().all(|c| c.is_uppercase()) {
+                caps_count += 1;
+            }
+
+            if word.len() > LONG_WORD_THRESHOLD {
+                long_word_count += 1;
+            }
+
+            punct_density += word.chars().filter(|c| c.is_ascii_punctuation()).count();
+        }
+
+        score += (numeric_count as f32 / words.len() as f32) * NUMERIC_CONTENT_WEIGHT;
+        score += (caps_count as f32 / words.len() as f32) * CAPS_ACRONYM_WEIGHT;
+        score += (long_word_count as f32 / words.len() as f32) * LONG_WORD_WEIGHT;
+        score += (punct_density as f32 / sentence.len() as f32) * PUNCTUATION_DENSITY_WEIGHT;
+
+        let unique_words: std::collections::HashSet<_> = words
+            .iter()
+            .map(|w| {
+                w.chars()
+                    .filter(|c| c.is_alphabetic())
+                    .collect::<String>()
+                    .to_lowercase()
+            })
+            .collect();
+        let diversity_ratio = unique_words.len() as f32 / words.len() as f32;
+        score += diversity_ratio * DIVERSITY_RATIO_WEIGHT;
+
+        let char_entropy = self.calculate_char_entropy(sentence);
+        score += char_entropy * CHAR_ENTROPY_WEIGHT;
+
+        score
+    }
+
+    fn universal_tokenize(&self, text: &str) -> Vec<String> {
+        self.cjk_tokenizer.tokenize_mixed_text(text)
+    }
+
+    fn calculate_char_entropy(&self, text: &str) -> f32 {
+        let chars: Vec<char> = text.chars().collect();
+        if chars.is_empty() {
+            return 0.0;
+        }
+
+        let mut char_freq = std::collections::HashMap::new();
+        for &ch in &chars {
+            let lowercase_ch = ch
+                .to_lowercase()
+                .next()
+                .expect("to_lowercase() must yield at least one character for valid Unicode");
+            *char_freq.entry(lowercase_ch).or_insert(0) += 1;
+        }
+
+        let total_chars = chars.len() as f32;
+        char_freq
+            .values()
+            .map(|&freq| {
+                let p = freq as f32 / total_chars;
+                if p > 0.0 { -p * p.log2() } else { 0.0 }
+            })
+            .sum::<f32>()
+            .min(5.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_light_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Hello   world!!!   How are you???";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.contains("   "));
+    }
+
+    #[test]
+    fn test_moderate_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(result.contains("quick"));
+        assert!(result.contains("brown"));
+        assert!(result.contains("fox"));
+    }
+
+    #[test]
+    fn test_batch_processing() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            enable_parallel: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let inputs = vec!["Hello  world!", "How   are you?", "Fine,  thanks!"];
+        let results = reducer.batch_reduce(&inputs);
+
+        assert_eq!(results.len(), inputs.len());
+        for result in &results {
+            assert!(!result.contains("  "));
+        }
+    }
+
+    #[test]
+    fn test_aggressive_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog and running through the forest";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_maximum_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Maximum,
+            use_simd: false,
+            enable_semantic_clustering: true,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog and running through the forest";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_empty_text_handling() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        assert_eq!(reducer.reduce(""), "");
+        let result = reducer.reduce("   ");
+        assert!(result == "   " || result.is_empty());
+    }
+
+    #[test]
+    fn test_off_mode_preserves_text() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Off,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Text   with    multiple   spaces!!!";
+        assert_eq!(reducer.reduce(input), input);
+    }
+
+    #[test]
+    fn test_parallel_batch_processing() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            enable_parallel: true,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let inputs = vec![
+            "First text  with spaces",
+            "Second  text with  spaces",
+            "Third   text  with spaces",
+        ];
+        let results = reducer.batch_reduce(&inputs);
+
+        assert_eq!(results.len(), inputs.len());
+        for result in &results {
+            assert!(!result.contains("  "));
+        }
+    }
+
+    #[test]
+    fn test_cjk_text_handling() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("zh")).unwrap();
+        let input = "这是中文文本测试";
+        let result = reducer.reduce(input);
+
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_mixed_language_text() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "This is English text 这是中文 and some more English";
+        let result = reducer.reduce(input);
+
+        assert!(!result.is_empty());
+        assert!(result.contains("English") || result.contains("中"));
+    }
+
+    #[test]
+    fn test_punctuation_normalization() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Text!!!!!! with????? excessive,,,,,, punctuation";
+        let result = reducer.reduce(input);
+
+        assert!(!result.contains("!!!!!!"));
+        assert!(!result.contains("?????"));
+        assert!(!result.contains(",,,,,,"));
+    }
+
+    #[test]
+    fn test_sentence_selection() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "First sentence here. Second sentence with more words. Third one. Fourth sentence is even longer than the others.";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(result.split(". ").count() < 4);
+    }
+
+    #[test]
+    fn test_unicode_normalization_ascii() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Pure ASCII text without special characters";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("ASCII"));
+    }
+
+    #[test]
+    fn test_unicode_normalization_non_ascii() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Café naïve résumé";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("Café") || result.contains("Cafe"));
+    }
+
+    #[test]
+    fn test_single_text_vs_batch() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let text = "The quick brown fox jumps over the lazy dog";
+
+        let single_result = reducer.reduce(text);
+        let batch_results = reducer.batch_reduce(&[text]);
+
+        assert_eq!(single_result, batch_results[0]);
+    }
+
+    #[test]
+    fn test_important_word_preservation() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "The IMPORTANT word COVID-19 and 12345 numbers should be preserved";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("IMPORTANT") || result.contains("COVID") || result.contains("12345"));
+    }
+
+    #[test]
+    fn test_technical_terms_preservation() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "The implementation uses PyTorch and TensorFlow frameworks";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("PyTorch") || result.contains("TensorFlow"));
+    }
+
+    #[test]
+    fn test_calculate_char_entropy() {
+        let config = TokenReductionConfig::default();
+        let reducer = TokenReducer::new(&config, None).unwrap();
+
+        let low_entropy = reducer.calculate_char_entropy("aaaaaaa");
+        assert!(low_entropy < 1.0);
+
+        let high_entropy = reducer.calculate_char_entropy("abcdefg123");
+        assert!(high_entropy > low_entropy);
+    }
+
+    #[test]
+    fn test_universal_tokenize_english() {
+        let config = TokenReductionConfig::default();
+        let reducer = TokenReducer::new(&config, None).unwrap();
+
+        let tokens = reducer.universal_tokenize("hello world test");
+        assert_eq!(tokens, vec!["hello", "world", "test"]);
+    }
+
+    #[test]
+    fn test_universal_tokenize_cjk() {
+        let config = TokenReductionConfig::default();
+        let reducer = TokenReducer::new(&config, None).unwrap();
+
+        let tokens = reducer.universal_tokenize("中文");
+        assert!(!tokens.is_empty());
+    }
+
+    #[test]
+    fn test_fallback_threshold() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+
+        let input = "a the is of to in for on at by";
+        let result = reducer.reduce(input);
+
+        assert!(!result.is_empty());
+    }
+}