kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/text/token_reduction/core.rs

@@ -6,7 +6,6 @@ use crate::text::token_reduction::{
     semantic::SemanticAnalyzer,
     simd_text::{SimdTextProcessor, chunk_text_for_parallel},
 };
-use ahash::AHashMap;
 use once_cell::sync::Lazy;
 use rayon::prelude::*;
 use regex::Regex;
@@ -99,12 +98,10 @@ impl TokenReducer {
             return text.to_string();
         }
 
-        let nfc_string;
         let working_text = if text.is_ascii() {
             text
         } else {
-            nfc_string = text.nfc().collect::<String>();
-            &nfc_string
+            &text.nfc().collect::<String>()
         };
 
         match self.config.level {
@@ -185,21 +182,13 @@ impl TokenReducer {
     }
 
     fn clean_punctuation_optimized(&self, text: &str) -> String {
-        use std::borrow::Cow;
+        let mut result = text.to_string();
 
-        let mut result = Cow::Borrowed(text);
+        result = REPEATED_EXCLAMATION.replace_all(&result, "!").to_string();
+        result = REPEATED_QUESTION.replace_all(&result, "?").to_string();
+        result = REPEATED_COMMA.replace_all(&result, ",").to_string();
 
-        if REPEATED_EXCLAMATION.is_match(&result) {
-            result = Cow::Owned(REPEATED_EXCLAMATION.replace_all(&result, "!").into_owned());
-        }
-        if REPEATED_QUESTION.is_match(&result) {
-            result = Cow::Owned(REPEATED_QUESTION.replace_all(&result, "?").into_owned());
-        }
-        if REPEATED_COMMA.is_match(&result) {
-            result = Cow::Owned(REPEATED_COMMA.replace_all(&result, ",").into_owned());
-        }
-
-        result.into_owned()
+        result
     }
 
     fn remove_additional_common_words(&self, text: &str) -> String {
@@ -209,10 +198,8 @@ impl TokenReducer {
             return text.to_string();
         }
 
-        let estimated_unique = (words.len() as f32 * 0.7).ceil() as usize;
-        let mut word_freq = AHashMap::with_capacity(estimated_unique);
-
-        let mut word_lengths = Vec::with_capacity(words.len());
+        let mut word_freq = std::collections::HashMap::new();
+        let mut word_lengths = Vec::new();
 
         for word in &words {
             let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
@@ -237,34 +224,34 @@ impl TokenReducer {
         };
 
         let original_count = words.len();
-        let has_cjk_content = text.chars().any(|c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
 
-        let mut filtered_words = Vec::with_capacity(words.len());
-        for word in &words {
-            let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
-                word.to_lowercase()
-            } else {
-                word.chars()
-                    .filter(|c| c.is_alphabetic())
-                    .collect::<String>()
-                    .to_lowercase()
-            };
+        let filtered_words: Vec<String> = words
+            .iter()
+            .filter(|word| {
+                let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                    word.to_lowercase()
+                } else {
+                    word.chars()
+                        .filter(|c| c.is_alphabetic())
+                        .collect::<String>()
+                        .to_lowercase()
+                };
+
+                if clean_word.is_empty() {
+                    return true;
+                }
 
-            if clean_word.is_empty() {
-                filtered_words.push(word.clone());
-            } else {
                 let freq = word_freq.get(&clean_word).unwrap_or(&0);
                 let word_len = clean_word.chars().count() as f32;
 
-                if self.has_important_characteristics(word)
+                self.has_important_characteristics(word)
                     || (*freq <= 2 && word_len >= avg_length * 0.8)
                     || (word_len >= avg_length * 1.5)
-                {
-                    filtered_words.push(word.clone());
-                }
-            }
-        }
+            })
+            .cloned()
+            .collect();
 
+        let has_cjk_content = text.chars().any(|c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
         let fallback_threshold = if has_cjk_content {
             original_count / 5
         } else {
@@ -272,19 +259,19 @@ impl TokenReducer {
         };
 
         if filtered_words.len() < fallback_threshold {
-            let mut fallback_words = Vec::with_capacity(words.len());
-            for word in &words {
-                let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
-                    word.to_lowercase()
-                } else {
-                    word.chars().filter(|c| c.is_alphabetic()).collect::<String>()
-                };
-
-                if clean_word.is_empty() || clean_word.chars().count() >= 3 || self.has_important_characteristics(word)
-                {
-                    fallback_words.push(word.clone());
-                }
-            }
+            let fallback_words: Vec<String> = words
+                .iter()
+                .filter(|word| {
+                    let clean_word = if word.chars().all(|c| c.is_alphabetic()) {
+                        (*word).clone()
+                    } else {
+                        word.chars().filter(|c| c.is_alphabetic()).collect::<String>()
+                    };
+
+                    clean_word.is_empty() || clean_word.chars().count() >= 3 || self.has_important_characteristics(word)
+                })
+                .cloned()
+                .collect();
             self.smart_join(&fallback_words, has_cjk_content)
         } else {
             self.smart_join(&filtered_words, has_cjk_content)
@@ -445,37 +432,16 @@ impl TokenReducer {
         score += (long_word_count as f32 / words.len() as f32) * LONG_WORD_WEIGHT;
         score += (punct_density as f32 / sentence.len() as f32) * PUNCTUATION_DENSITY_WEIGHT;
 
-        let estimated_unique = (words.len() as f32 * 0.6).ceil() as usize;
-        let mut unique_words: ahash::AHashSet<String> = ahash::AHashSet::with_capacity(estimated_unique.max(10));
-
-        for w in &words {
-            let clean = w
-                .chars()
-                .filter(|c| c.is_alphabetic())
-                .collect::<String>()
-                .to_lowercase();
-            unique_words.insert(clean);
-
-            if unique_words.len() >= estimated_unique {
-                break;
-            }
-        }
-
-        let final_unique_count = if unique_words.len() >= estimated_unique {
-            unique_words.len()
-        } else {
-            for w in &words {
-                let clean = w
-                    .chars()
+        let unique_words: std::collections::HashSet<_> = words
+            .iter()
+            .map(|w| {
+                w.chars()
                     .filter(|c| c.is_alphabetic())
                     .collect::<String>()
-                    .to_lowercase();
-                unique_words.insert(clean);
-            }
-            unique_words.len()
-        };
-
-        let diversity_ratio = final_unique_count as f32 / words.len() as f32;
+                    .to_lowercase()
+            })
+            .collect();
+        let diversity_ratio = unique_words.len() as f32 / words.len() as f32;
         score += diversity_ratio * DIVERSITY_RATIO_WEIGHT;
 
         let char_entropy = self.calculate_char_entropy(sentence);
@@ -494,9 +460,7 @@ impl TokenReducer {
             return 0.0;
         }
 
-        let estimated_unique = (chars.len() as f32 * 0.1).ceil() as usize;
-        let mut char_freq = AHashMap::with_capacity(estimated_unique.max(26));
-
+        let mut char_freq = std::collections::HashMap::new();
         for &ch in &chars {
             let lowercase_ch = ch
                 .to_lowercase()

data/vendor/kreuzberg/src/text/token_reduction/filters.rs

@@ -1,7 +1,6 @@
 use crate::error::{KreuzbergError, Result};
 use crate::stopwords::STOPWORDS;
 use crate::text::token_reduction::config::TokenReductionConfig;
-use crate::text::utf8_validation;
 use ahash::{AHashMap, AHashSet};
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -64,48 +63,34 @@ impl FilterPipeline {
     }
 
     pub fn apply_light_filters(&self, text: &str) -> String {
-        use std::borrow::Cow;
-
-        let mut result = Cow::Borrowed(text);
+        let mut result = text.to_string();
 
-        let mut preserved_blocks: Option<AHashMap<String, String>> = None;
+        let mut preserved_blocks = AHashMap::new();
         if self.config.preserve_markdown {
-            let mut blocks = AHashMap::new();
-            result = Cow::Owned(self.extract_and_preserve_code(result.as_ref(), &mut blocks));
-            preserved_blocks = Some(blocks);
+            result = self.extract_and_preserve_code(&result, &mut preserved_blocks);
         }
 
-        if HTML_COMMENT_REGEX.is_match(&result) {
-            result = Cow::Owned(HTML_COMMENT_REGEX.replace_all(&result, "").into_owned());
-        }
+        result = HTML_COMMENT_REGEX.replace_all(&result, "").to_string();
 
-        if MULTIPLE_SPACES_REGEX.is_match(&result) {
-            result = Cow::Owned(MULTIPLE_SPACES_REGEX.replace_all(&result, " ").into_owned());
-        }
+        result = MULTIPLE_SPACES_REGEX.replace_all(&result, " ").to_string();
 
-        if EXCESSIVE_NEWLINES_REGEX.is_match(&result) {
-            result = Cow::Owned(EXCESSIVE_NEWLINES_REGEX.replace_all(&result, "\n\n").into_owned());
-        }
+        result = EXCESSIVE_NEWLINES_REGEX.replace_all(&result, "\n\n").to_string();
 
         if self.config.preserve_markdown {
-            result = Cow::Owned(self.preserve_markdown_structure(&result));
+            result = self.preserve_markdown_structure(&result);
         }
 
-        if let Some(blocks) = &preserved_blocks {
-            result = Cow::Owned(self.restore_preserved_blocks(&result, blocks));
-        }
+        result = self.restore_preserved_blocks(&result, &preserved_blocks);
 
-        result.into_owned()
+        result
     }
 
     pub fn apply_moderate_filters(&self, text: &str) -> String {
         let mut result = self.apply_light_filters(text);
 
-        let mut preserved_blocks: Option<AHashMap<String, String>> = None;
+        let mut preserved_blocks = AHashMap::new();
         if self.config.preserve_code {
-            let mut blocks = AHashMap::new();
-            result = self.extract_and_preserve_code(&result, &mut blocks);
-            preserved_blocks = Some(blocks);
+            result = self.extract_and_preserve_code(&result, &mut preserved_blocks);
         }
 
         if self.config.preserve_markdown {
@@ -114,16 +99,14 @@ impl FilterPipeline {
             result = self.remove_stopwords(&result);
         }
 
-        if let Some(blocks) = &preserved_blocks {
-            result = self.restore_preserved_blocks(&result, blocks);
-        }
+        result = self.restore_preserved_blocks(&result, &preserved_blocks);
 
         result
     }
 
     fn remove_stopwords_preserving_markdown(&self, text: &str) -> String {
         let lines: Vec<&str> = text.lines().collect();
-        let mut processed_lines = Vec::with_capacity(lines.len());
+        let mut processed_lines = Vec::new();
 
         for line in lines {
             if MARKDOWN_HEADERS_REGEX.is_match(line) {
@@ -150,7 +133,7 @@ impl FilterPipeline {
 
     fn remove_stopwords(&self, text: &str) -> String {
         let words: Vec<&str> = text.split_whitespace().collect();
-        let mut filtered_words = Vec::with_capacity((words.len() as f32 * 0.7).ceil() as usize);
+        let mut filtered_words = Vec::with_capacity(words.len());
 
         for word in words {
             if word.is_empty() {
@@ -178,7 +161,7 @@ impl FilterPipeline {
                     .filter(|&b| b.is_ascii_alphabetic())
                     .map(|b| b.to_ascii_lowercase())
                     .collect();
-                utf8_validation::string_from_utf8(clean_bytes).unwrap_or_else(|_| {
+                String::from_utf8(clean_bytes).unwrap_or_else(|_| {
                     word.chars()
                         .filter(|c| c.is_alphabetic())
                         .collect::<String>()
@@ -249,7 +232,7 @@ impl FilterPipeline {
 
     fn preserve_markdown_structure(&self, text: &str) -> String {
         let lines: Vec<&str> = text.lines().collect();
-        let mut processed_lines = Vec::with_capacity(lines.len());
+        let mut processed_lines = Vec::new();
 
         for line in lines {
             if MARKDOWN_HEADERS_REGEX.is_match(line) {
@@ -297,10 +280,6 @@ impl FilterPipeline {
     }
 
     fn restore_preserved_blocks(&self, text: &str, preserved: &AHashMap<String, String>) -> String {
-        if preserved.is_empty() {
-            return text.to_string();
-        }
-
         let mut result = text.to_string();
 
         for (placeholder, original_content) in preserved {

data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs

@@ -1,4 +1,3 @@
-use crate::text::utf8_validation;
 use memchr::{memchr, memchr3};
 
 pub struct SimdTextProcessor;
@@ -51,7 +50,7 @@ impl SimdTextProcessor {
             i = sequence_end;
         }
 
-        utf8_validation::string_from_utf8(result).unwrap_or_else(|_| text.to_string())
+        String::from_utf8(result).unwrap_or_else(|_| text.to_string())
     }
 
     #[inline]