kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/text/token_reduction/core/reducer.rs

@@ -0,0 +1,384 @@
+use crate::error::Result;
+use crate::text::token_reduction::{
+    config::{ReductionLevel, TokenReductionConfig},
+    filters::FilterPipeline,
+    semantic::SemanticAnalyzer,
+    simd_text::{SimdTextProcessor, chunk_text_for_parallel},
+};
+use rayon::prelude::*;
+use std::sync::Arc;
+use unicode_normalization::UnicodeNormalization;
+
+use super::punctuation::PunctuationCleaner;
+use super::sentence_selection::SentenceSelector;
+use super::word_filtering::WordFilter;
+
+pub struct TokenReducer {
+    config: Arc<TokenReductionConfig>,
+    text_processor: SimdTextProcessor,
+    filter_pipeline: FilterPipeline,
+    semantic_analyzer: Option<SemanticAnalyzer>,
+    word_filter: WordFilter,
+    language: String,
+}
+
+impl TokenReducer {
+    pub fn new(config: &TokenReductionConfig, language_hint: Option<&str>) -> Result<Self> {
+        let config = Arc::new(config.clone());
+        let language = language_hint
+            .or(config.language_hint.as_deref())
+            .unwrap_or("en")
+            .to_string();
+
+        let text_processor = SimdTextProcessor::new();
+        let filter_pipeline = FilterPipeline::new(&config, &language)?;
+
+        let semantic_analyzer = if matches!(config.level, ReductionLevel::Aggressive | ReductionLevel::Maximum) {
+            Some(SemanticAnalyzer::new(&language))
+        } else {
+            None
+        };
+
+        Ok(Self {
+            config,
+            text_processor,
+            filter_pipeline,
+            semantic_analyzer,
+            word_filter: WordFilter::new(),
+            language,
+        })
+    }
+
+    /// Get the language code being used for stopwords and semantic analysis.
+    pub fn language(&self) -> &str {
+        &self.language
+    }
+
+    pub fn reduce(&self, text: &str) -> String {
+        if text.is_empty() || matches!(self.config.level, ReductionLevel::Off) {
+            return text.to_string();
+        }
+
+        let nfc_string;
+        let working_text = if text.is_ascii() {
+            text
+        } else {
+            nfc_string = text.nfc().collect::<String>();
+            &nfc_string
+        };
+
+        match self.config.level {
+            ReductionLevel::Off => working_text.to_string(),
+            ReductionLevel::Light => self.apply_light_reduction_optimized(working_text),
+            ReductionLevel::Moderate => self.apply_moderate_reduction_optimized(working_text),
+            ReductionLevel::Aggressive => self.apply_aggressive_reduction_optimized(working_text),
+            ReductionLevel::Maximum => self.apply_maximum_reduction_optimized(working_text),
+        }
+    }
+
+    pub fn batch_reduce(&self, texts: &[&str]) -> Vec<String> {
+        if !self.config.enable_parallel || texts.len() < 2 {
+            return texts.iter().map(|text| self.reduce(text)).collect();
+        }
+
+        texts.par_iter().map(|text| self.reduce(text)).collect()
+    }
+
+    fn apply_light_reduction_optimized(&self, text: &str) -> String {
+        let mut result = if self.config.use_simd {
+            self.text_processor.clean_punctuation(text)
+        } else {
+            PunctuationCleaner::clean_punctuation_optimized(text)
+        };
+
+        result = self.filter_pipeline.apply_light_filters(&result);
+        result.trim().to_string()
+    }
+
+    fn apply_moderate_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_light_reduction_optimized(text);
+
+        result = if self.config.enable_parallel && text.len() > 1000 {
+            self.apply_parallel_moderate_reduction(&result)
+        } else {
+            self.filter_pipeline.apply_moderate_filters(&result)
+        };
+
+        result
+    }
+
+    fn apply_aggressive_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_moderate_reduction_optimized(text);
+
+        result = self.word_filter.remove_additional_common_words(&result);
+        result = SentenceSelector::apply_sentence_selection(&result);
+
+        if let Some(ref analyzer) = self.semantic_analyzer {
+            result = analyzer.apply_semantic_filtering(&result, self.config.semantic_threshold);
+        }
+
+        result
+    }
+
+    fn apply_maximum_reduction_optimized(&self, text: &str) -> String {
+        let mut result = self.apply_aggressive_reduction_optimized(text);
+
+        if let Some(ref analyzer) = self.semantic_analyzer
+            && self.config.enable_semantic_clustering
+        {
+            result = analyzer.apply_hypernym_compression(&result, self.config.target_reduction);
+        }
+
+        result
+    }
+
+    fn apply_parallel_moderate_reduction(&self, text: &str) -> String {
+        let num_threads = rayon::current_num_threads();
+        let chunks = chunk_text_for_parallel(text, num_threads);
+
+        let processed_chunks: Vec<String> = chunks
+            .par_iter()
+            .map(|chunk| self.filter_pipeline.apply_moderate_filters(chunk))
+            .collect();
+
+        processed_chunks.join(" ")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_light_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Hello   world!!!   How are you???";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.contains("   "));
+    }
+
+    #[test]
+    fn test_moderate_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(result.contains("quick"));
+        assert!(result.contains("brown"));
+        assert!(result.contains("fox"));
+    }
+
+    #[test]
+    fn test_batch_processing() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            enable_parallel: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let inputs = vec!["Hello  world!", "How   are you?", "Fine,  thanks!"];
+        let results = reducer.batch_reduce(&inputs);
+
+        assert_eq!(results.len(), inputs.len());
+        for result in &results {
+            assert!(!result.contains("  "));
+        }
+    }
+
+    #[test]
+    fn test_aggressive_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            use_simd: false,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog and running through the forest";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_maximum_reduction() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Maximum,
+            use_simd: false,
+            enable_semantic_clustering: true,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("en")).unwrap();
+        let input = "The quick brown fox is jumping over the lazy dog and running through the forest";
+        let result = reducer.reduce(input);
+
+        assert!(result.len() < input.len());
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_empty_text_handling() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        assert_eq!(reducer.reduce(""), "");
+        let result = reducer.reduce("   ");
+        assert!(result == "   " || result.is_empty());
+    }
+
+    #[test]
+    fn test_off_mode_preserves_text() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Off,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Text   with    multiple   spaces!!!";
+        assert_eq!(reducer.reduce(input), input);
+    }
+
+    #[test]
+    fn test_parallel_batch_processing() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            enable_parallel: true,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let inputs = vec![
+            "First text  with spaces",
+            "Second  text with  spaces",
+            "Third   text  with spaces",
+        ];
+        let results = reducer.batch_reduce(&inputs);
+
+        assert_eq!(results.len(), inputs.len());
+        for result in &results {
+            assert!(!result.contains("  "));
+        }
+    }
+
+    #[test]
+    fn test_cjk_text_handling() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, Some("zh")).unwrap();
+        let input = "这是中文文本测试";
+        let result = reducer.reduce(input);
+
+        assert!(!result.is_empty());
+    }
+
+    #[test]
+    fn test_mixed_language_text() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "This is English text 这是中文 and some more English";
+        let result = reducer.reduce(input);
+
+        assert!(!result.is_empty());
+        assert!(result.contains("English") || result.contains("中"));
+    }
+
+    #[test]
+    fn test_unicode_normalization_ascii() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Pure ASCII text without special characters";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("ASCII"));
+    }
+
+    #[test]
+    fn test_unicode_normalization_non_ascii() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Light,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "Café naïve résumé";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("Café") || result.contains("Cafe"));
+    }
+
+    #[test]
+    fn test_single_text_vs_batch() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Moderate,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let text = "The quick brown fox jumps over the lazy dog";
+
+        let single_result = reducer.reduce(text);
+        let batch_results = reducer.batch_reduce(&[text]);
+
+        assert_eq!(single_result, batch_results[0]);
+    }
+
+    #[test]
+    fn test_important_word_preservation() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "The IMPORTANT word COVID-19 and 12345 numbers should be preserved";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("IMPORTANT") || result.contains("COVID") || result.contains("12345"));
+    }
+
+    #[test]
+    fn test_technical_terms_preservation() {
+        let config = TokenReductionConfig {
+            level: ReductionLevel::Aggressive,
+            ..Default::default()
+        };
+
+        let reducer = TokenReducer::new(&config, None).unwrap();
+        let input = "The implementation uses PyTorch and TensorFlow frameworks";
+        let result = reducer.reduce(input);
+
+        assert!(result.contains("PyTorch") || result.contains("TensorFlow"));
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/core/sentence_selection.rs

@@ -0,0 +1,68 @@
+use super::analysis::TextAnalyzer;
+
+/// Handles sentence selection and filtering based on importance scoring.
+pub struct SentenceSelector;
+
+impl SentenceSelector {
+    /// Applies sentence selection to keep only the most important sentences.
+    pub fn apply_sentence_selection(text: &str) -> String {
+        let sentences: Vec<&str> = text
+            .split(['.', '!', '?'])
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        if sentences.len() <= 2 {
+            return text.to_string();
+        }
+
+        let mut scored_sentences: Vec<(usize, f32, &str)> = sentences
+            .iter()
+            .enumerate()
+            .map(|(i, sentence)| {
+                let score = TextAnalyzer::score_sentence_importance(sentence, i, sentences.len());
+                (i, score, *sentence)
+            })
+            .collect();
+
+        scored_sentences.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+
+        let keep_count = ((sentences.len() as f32 * 0.4).ceil() as usize).max(1);
+        let mut selected_indices: Vec<usize> = scored_sentences[..keep_count].iter().map(|(i, _, _)| *i).collect();
+
+        selected_indices.sort();
+
+        let selected_sentences: Vec<&str> = selected_indices
+            .iter()
+            .filter_map(|&i| sentences.get(i))
+            .copied()
+            .collect();
+
+        if selected_sentences.is_empty() {
+            text.to_string()
+        } else {
+            selected_sentences.join(". ")
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_sentence_selection() {
+        let input = "First sentence here. Second sentence with more words. Third one. Fourth sentence is even longer than the others.";
+        let result = SentenceSelector::apply_sentence_selection(input);
+
+        assert!(result.len() < input.len());
+        assert!(result.split(". ").count() < 4);
+    }
+
+    #[test]
+    fn test_sentence_selection_short_text() {
+        let input = "Only one sentence.";
+        let result = SentenceSelector::apply_sentence_selection(input);
+        assert_eq!(result, input);
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/core/word_filtering.rs

@@ -0,0 +1,156 @@
+use crate::text::token_reduction::cjk_utils::CjkTokenizer;
+use ahash::AHashMap;
+
+use super::analysis::TextAnalyzer;
+
+/// Handles word filtering and token removal operations.
+pub struct WordFilter {
+    cjk_tokenizer: CjkTokenizer,
+}
+
+impl WordFilter {
+    pub fn new() -> Self {
+        Self {
+            cjk_tokenizer: CjkTokenizer::new(),
+        }
+    }
+
+    /// Removes additional common words based on frequency and characteristics.
+    pub fn remove_additional_common_words(&self, text: &str) -> String {
+        let words = self.universal_tokenize(text);
+
+        if words.len() < 4 {
+            return text.to_string();
+        }
+
+        let estimated_unique = (words.len() as f32 * 0.7).ceil() as usize;
+        let mut word_freq = AHashMap::with_capacity(estimated_unique);
+
+        let mut word_lengths = Vec::with_capacity(words.len());
+
+        for word in &words {
+            let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                word.to_lowercase()
+            } else {
+                word.chars()
+                    .filter(|c| c.is_alphabetic())
+                    .collect::<String>()
+                    .to_lowercase()
+            };
+
+            if !clean_word.is_empty() {
+                *word_freq.entry(clean_word.clone()).or_insert(0) += 1;
+                word_lengths.push(clean_word.chars().count());
+            }
+        }
+
+        let avg_length = if !word_lengths.is_empty() {
+            word_lengths.iter().sum::<usize>() as f32 / word_lengths.len() as f32
+        } else {
+            5.0
+        };
+
+        let original_count = words.len();
+        let has_cjk_content = text.chars().any(|c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
+
+        let mut filtered_words = Vec::with_capacity(words.len());
+        for word in &words {
+            let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                word.to_lowercase()
+            } else {
+                word.chars()
+                    .filter(|c| c.is_alphabetic())
+                    .collect::<String>()
+                    .to_lowercase()
+            };
+
+            if clean_word.is_empty() {
+                filtered_words.push(word.clone());
+            } else {
+                let freq = word_freq.get(&clean_word).unwrap_or(&0);
+                let word_len = clean_word.chars().count() as f32;
+
+                if TextAnalyzer::has_important_characteristics(word)
+                    || (*freq <= 2 && word_len >= avg_length * 0.8)
+                    || (word_len >= avg_length * 1.5)
+                {
+                    filtered_words.push(word.clone());
+                }
+            }
+        }
+
+        let fallback_threshold = if has_cjk_content {
+            original_count / 5
+        } else {
+            original_count / 3
+        };
+
+        if filtered_words.len() < fallback_threshold {
+            let mut fallback_words = Vec::with_capacity(words.len());
+            for word in &words {
+                let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                    word.to_lowercase()
+                } else {
+                    word.chars().filter(|c| c.is_alphabetic()).collect::<String>()
+                };
+
+                if clean_word.is_empty()
+                    || clean_word.chars().count() >= 3
+                    || TextAnalyzer::has_important_characteristics(word)
+                {
+                    fallback_words.push(word.clone());
+                }
+            }
+            self.smart_join(&fallback_words, has_cjk_content)
+        } else {
+            self.smart_join(&filtered_words, has_cjk_content)
+        }
+    }
+
+    /// Smart joins tokens based on language type (CJK vs. other).
+    pub fn smart_join(&self, tokens: &[String], has_cjk_content: bool) -> String {
+        if has_cjk_content {
+            tokens.join("")
+        } else {
+            tokens.join(" ")
+        }
+    }
+
+    /// Universal tokenizer that handles both CJK and non-CJK text.
+    pub fn universal_tokenize(&self, text: &str) -> Vec<String> {
+        self.cjk_tokenizer.tokenize_mixed_text(text)
+    }
+}
+
+impl Default for WordFilter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_universal_tokenize_english() {
+        let filter = WordFilter::new();
+        let tokens = filter.universal_tokenize("hello world test");
+        assert_eq!(tokens, vec!["hello", "world", "test"]);
+    }
+
+    #[test]
+    fn test_universal_tokenize_cjk() {
+        let filter = WordFilter::new();
+        let tokens = filter.universal_tokenize("中文");
+        assert!(!tokens.is_empty());
+    }
+
+    #[test]
+    fn test_fallback_threshold() {
+        let filter = WordFilter::new();
+        let input = "a the is of to in for on at by";
+        let result = filter.remove_additional_common_words(input);
+        assert!(!result.is_empty());
+    }
+}