kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/stopwords/languages/germanic.rs

@@ -0,0 +1,36 @@
+//! Germanic language stopwords.
+//!
+//! Includes: English (en), German (de), Dutch (nl), Swedish (sv),
+//! Norwegian (no), Danish (da), Afrikaans (af).
+
+use ahash::{AHashMap, AHashSet};
+
+/// Macro to generate embedded stopwords for Germanic languages.
+macro_rules! embed_stopwords {
+    ($map:expr, $($lang:literal),* $(,)?) => {
+        $(
+            {
+                const JSON: &str = include_str!(concat!("../../../stopwords/", $lang, "_stopwords.json"));
+                match serde_json::from_str::<Vec<String>>(JSON) {
+                    Ok(words) => {
+                        let set: AHashSet<String> = words.into_iter().collect();
+                        $map.insert($lang.to_string(), set);
+                    }
+                    Err(e) => {
+                        panic!(
+                            "Failed to parse embedded stopwords for language '{}': {}. \
+                            This indicates corrupted or malformed JSON in the embedded stopwords data. \
+                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
+                            $lang, e
+                        );
+                    }
+                }
+            }
+        )*
+    };
+}
+
+/// Load Germanic language stopwords into the provided map.
+pub(in crate::stopwords) fn load_stopwords(map: &mut AHashMap<String, AHashSet<String>>) {
+    embed_stopwords!(map, "en", "de", "nl", "sv", "no", "da", "af");
+}

data/vendor/kreuzberg/src/stopwords/languages/mod.rs

@@ -0,0 +1,10 @@
+//! Language family modules for stopword loading.
+//!
+//! Stopwords are organized by language family for easier maintenance
+//! and navigation. Each module handles loading stopwords for related languages.
+
+pub(super) mod asian;
+pub(super) mod germanic;
+pub(super) mod other;
+pub(super) mod romance;
+pub(super) mod slavic;

data/vendor/kreuzberg/src/stopwords/languages/other.rs

@@ -0,0 +1,44 @@
+//! Other language stopwords.
+//!
+//! Includes: Arabic (ar), Hebrew (he), Turkish (tr), Persian (fa),
+//! Kurdish (ku), Armenian (hy), Estonian (et), Basque (eu),
+//! Breton (br), Esperanto (eo), Finnish (fi), Irish (ga),
+//! Hungarian (hu), Indonesian (id), Latin (la), Lithuanian (lt),
+//! Latvian (lv), Malay (ms), Tagalog (tl), Greek (el),
+//! Hausa (ha), Swahili (sw), Yoruba (yo), Zulu (zu),
+//! Somali (so), Sesotho (st).
+
+use ahash::{AHashMap, AHashSet};
+
+/// Macro to generate embedded stopwords for other languages.
+macro_rules! embed_stopwords {
+    ($map:expr, $($lang:literal),* $(,)?) => {
+        $(
+            {
+                const JSON: &str = include_str!(concat!("../../../stopwords/", $lang, "_stopwords.json"));
+                match serde_json::from_str::<Vec<String>>(JSON) {
+                    Ok(words) => {
+                        let set: AHashSet<String> = words.into_iter().collect();
+                        $map.insert($lang.to_string(), set);
+                    }
+                    Err(e) => {
+                        panic!(
+                            "Failed to parse embedded stopwords for language '{}': {}. \
+                            This indicates corrupted or malformed JSON in the embedded stopwords data. \
+                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
+                            $lang, e
+                        );
+                    }
+                }
+            }
+        )*
+    };
+}
+
+/// Load other language stopwords into the provided map.
+pub(in crate::stopwords) fn load_stopwords(map: &mut AHashMap<String, AHashSet<String>>) {
+    embed_stopwords!(
+        map, "ar", "he", "tr", "fa", "ku", "hy", "et", "eu", "br", "eo", "fi", "ga", "hu", "id", "la", "lt", "lv",
+        "ms", "tl", "el", "ha", "sw", "yo", "zu", "so", "st"
+    );
+}

data/vendor/kreuzberg/src/stopwords/languages/romance.rs

@@ -0,0 +1,36 @@
+//! Romance language stopwords.
+//!
+//! Includes: French (fr), Spanish (es), Italian (it), Portuguese (pt),
+//! Romanian (ro), Catalan (ca), Galician (gl).
+
+use ahash::{AHashMap, AHashSet};
+
+/// Macro to generate embedded stopwords for Romance languages.
+macro_rules! embed_stopwords {
+    ($map:expr, $($lang:literal),* $(,)?) => {
+        $(
+            {
+                const JSON: &str = include_str!(concat!("../../../stopwords/", $lang, "_stopwords.json"));
+                match serde_json::from_str::<Vec<String>>(JSON) {
+                    Ok(words) => {
+                        let set: AHashSet<String> = words.into_iter().collect();
+                        $map.insert($lang.to_string(), set);
+                    }
+                    Err(e) => {
+                        panic!(
+                            "Failed to parse embedded stopwords for language '{}': {}. \
+                            This indicates corrupted or malformed JSON in the embedded stopwords data. \
+                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
+                            $lang, e
+                        );
+                    }
+                }
+            }
+        )*
+    };
+}
+
+/// Load Romance language stopwords into the provided map.
+pub(in crate::stopwords) fn load_stopwords(map: &mut AHashMap<String, AHashSet<String>>) {
+    embed_stopwords!(map, "fr", "es", "it", "pt", "ro", "ca", "gl");
+}

data/vendor/kreuzberg/src/stopwords/languages/slavic.rs

@@ -0,0 +1,36 @@
+//! Slavic language stopwords.
+//!
+//! Includes: Russian (ru), Polish (pl), Czech (cs), Ukrainian (uk),
+//! Bulgarian (bg), Slovak (sk), Croatian (hr), Slovenian (sl).
+
+use ahash::{AHashMap, AHashSet};
+
+/// Macro to generate embedded stopwords for Slavic languages.
+macro_rules! embed_stopwords {
+    ($map:expr, $($lang:literal),* $(,)?) => {
+        $(
+            {
+                const JSON: &str = include_str!(concat!("../../../stopwords/", $lang, "_stopwords.json"));
+                match serde_json::from_str::<Vec<String>>(JSON) {
+                    Ok(words) => {
+                        let set: AHashSet<String> = words.into_iter().collect();
+                        $map.insert($lang.to_string(), set);
+                    }
+                    Err(e) => {
+                        panic!(
+                            "Failed to parse embedded stopwords for language '{}': {}. \
+                            This indicates corrupted or malformed JSON in the embedded stopwords data. \
+                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
+                            $lang, e
+                        );
+                    }
+                }
+            }
+        )*
+    };
+}
+
+/// Load Slavic language stopwords into the provided map.
+pub(in crate::stopwords) fn load_stopwords(map: &mut AHashMap<String, AHashSet<String>>) {
+    embed_stopwords!(map, "ru", "pl", "cs", "uk", "bg", "sk", "hr", "sl");
+}

data/vendor/kreuzberg/src/stopwords/mod.rs

@@ -82,33 +82,7 @@
 use ahash::{AHashMap, AHashSet};
 use once_cell::sync::Lazy;
 
-/// Macro to generate embedded stopwords for all languages.
-///
-/// This macro embeds the JSON files at compile time using `include_str!()` and
-/// generates code to parse and insert them into the stopwords map.
-macro_rules! embed_stopwords {
-    ($map:expr, $($lang:literal),* $(,)?) => {
-        $(
-            {
-                const JSON: &str = include_str!(concat!("../../stopwords/", $lang, "_stopwords.json"));
-                match serde_json::from_str::<Vec<String>>(JSON) {
-                    Ok(words) => {
-                        let set: AHashSet<String> = words.into_iter().collect();
-                        $map.insert($lang.to_string(), set);
-                    }
-                    Err(e) => {
-                        panic!(
-                            "Failed to parse embedded stopwords for language '{}': {}. \
-                            This indicates corrupted or malformed JSON in the embedded stopwords data. \
-                            Please report this issue at https://github.com/kreuzberg-dev/kreuzberg/issues",
-                            $lang, e
-                        );
-                    }
-                }
-            }
-        )*
-    };
-}
+mod languages;
 
 /// Global stopwords registry.
 ///
@@ -146,12 +120,12 @@ macro_rules! embed_stopwords {
 pub static STOPWORDS: Lazy<AHashMap<String, AHashSet<String>>> = Lazy::new(|| {
     let mut map = AHashMap::new();
 
-    embed_stopwords!(
-        map, "af", "ar", "bg", "bn", "br", "ca", "cs", "da", "de", "el", "en", "eo", "es", "et", "eu", "fa", "fi",
-        "fr", "ga", "gl", "gu", "ha", "he", "hi", "hr", "hu", "hy", "id", "it", "ja", "kn", "ko", "ku", "la", "lt",
-        "lv", "ml", "mr", "ms", "ne", "nl", "no", "pl", "pt", "ro", "ru", "si", "sk", "sl", "so", "st", "sv", "sw",
-        "ta", "te", "th", "tl", "tr", "uk", "ur", "vi", "yo", "zh", "zu",
-    );
+    // Load stopwords by language family
+    languages::germanic::load_stopwords(&mut map);
+    languages::romance::load_stopwords(&mut map);
+    languages::slavic::load_stopwords(&mut map);
+    languages::asian::load_stopwords(&mut map);
+    languages::other::load_stopwords(&mut map);
 
     apply_stopword_whitelist(&mut map);
 

data/vendor/kreuzberg/src/text/quality.rs

@@ -677,7 +677,7 @@ mod tests {
     fn test_normalize_whitespace_cow_no_changes() {
         let text = Cow::Borrowed("normaltext");
         let result = normalize_whitespace_cow(text);
-        assert_eq!(result.as_ref(), "normaltext");
+        assert_eq!(&*result, "normaltext");
     }
 
     #[test]

data/vendor/kreuzberg/src/text/quality_processor.rs

@@ -123,6 +123,8 @@ mod tests {
 	            chunks: None,
 	            images: None,
 	            pages: None,
+	            elements: None,
+	            djot_content: None,
 	        };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -148,7 +150,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -181,7 +185,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let config_with_quality = ExtractionConfig {
@@ -209,7 +215,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let long_result = ExtractionResult {
@@ -220,7 +228,9 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         };
 
         let short_duration = processor.estimated_duration_ms(&short_result);

data/vendor/kreuzberg/src/text/token_reduction/core/analysis.rs

@@ -0,0 +1,238 @@
+use ahash::AHashMap;
+
+/// Bonus added for sentences at the beginning or end of the document
+const SENTENCE_EDGE_POSITION_BONUS: f32 = 0.3;
+
+/// Bonus added for sentences with ideal word count (neither too short nor too long)
+const IDEAL_WORD_COUNT_BONUS: f32 = 0.2;
+
+/// Minimum word count for ideal sentence length
+const MIN_IDEAL_WORD_COUNT: usize = 3;
+
+/// Maximum word count for ideal sentence length
+const MAX_IDEAL_WORD_COUNT: usize = 25;
+
+/// Weight multiplier for numeric content density in sentences
+const NUMERIC_CONTENT_WEIGHT: f32 = 0.3;
+
+/// Weight multiplier for capitalized/acronym word density in sentences
+const CAPS_ACRONYM_WEIGHT: f32 = 0.25;
+
+/// Weight multiplier for long word density in sentences
+const LONG_WORD_WEIGHT: f32 = 0.2;
+
+/// Minimum character length for a word to be considered "long"
+const LONG_WORD_THRESHOLD: usize = 8;
+
+/// Weight multiplier for punctuation density in sentences
+const PUNCTUATION_DENSITY_WEIGHT: f32 = 0.15;
+
+/// Weight multiplier for word diversity ratio (unique words / total words)
+const DIVERSITY_RATIO_WEIGHT: f32 = 0.15;
+
+/// Weight multiplier for character entropy (measure of text randomness/information)
+const CHAR_ENTROPY_WEIGHT: f32 = 0.1;
+
+/// Analyzes text characteristics and scores content importance.
+pub struct TextAnalyzer;
+
+impl TextAnalyzer {
+    /// Scores the importance of a sentence based on various characteristics.
+    pub fn score_sentence_importance(sentence: &str, position: usize, total_sentences: usize) -> f32 {
+        let mut score = 0.0;
+
+        if position == 0 || position == total_sentences - 1 {
+            score += SENTENCE_EDGE_POSITION_BONUS;
+        }
+
+        let words: Vec<&str> = sentence.split_whitespace().collect();
+        if words.is_empty() {
+            return score;
+        }
+
+        let word_count = words.len();
+        if (MIN_IDEAL_WORD_COUNT..=MAX_IDEAL_WORD_COUNT).contains(&word_count) {
+            score += IDEAL_WORD_COUNT_BONUS;
+        }
+
+        let mut numeric_count = 0;
+        let mut caps_count = 0;
+        let mut long_word_count = 0;
+        let mut punct_density = 0;
+
+        for word in &words {
+            if word.chars().any(|c| c.is_numeric()) {
+                numeric_count += 1;
+            }
+
+            if word.len() > 1 && word.chars().all(|c| c.is_uppercase()) {
+                caps_count += 1;
+            }
+
+            if word.len() > LONG_WORD_THRESHOLD {
+                long_word_count += 1;
+            }
+
+            punct_density += word.chars().filter(|c| c.is_ascii_punctuation()).count();
+        }
+
+        score += (numeric_count as f32 / words.len() as f32) * NUMERIC_CONTENT_WEIGHT;
+        score += (caps_count as f32 / words.len() as f32) * CAPS_ACRONYM_WEIGHT;
+        score += (long_word_count as f32 / words.len() as f32) * LONG_WORD_WEIGHT;
+        score += (punct_density as f32 / sentence.len() as f32) * PUNCTUATION_DENSITY_WEIGHT;
+
+        let estimated_unique = (words.len() as f32 * 0.6).ceil() as usize;
+        let mut unique_words: ahash::AHashSet<String> = ahash::AHashSet::with_capacity(estimated_unique.max(10));
+
+        for w in &words {
+            let clean = w
+                .chars()
+                .filter(|c| c.is_alphabetic())
+                .collect::<String>()
+                .to_lowercase();
+            unique_words.insert(clean);
+
+            if unique_words.len() >= estimated_unique {
+                break;
+            }
+        }
+
+        let final_unique_count = if unique_words.len() >= estimated_unique {
+            unique_words.len()
+        } else {
+            for w in &words {
+                let clean = w
+                    .chars()
+                    .filter(|c| c.is_alphabetic())
+                    .collect::<String>()
+                    .to_lowercase();
+                unique_words.insert(clean);
+            }
+            unique_words.len()
+        };
+
+        let diversity_ratio = final_unique_count as f32 / words.len() as f32;
+        score += diversity_ratio * DIVERSITY_RATIO_WEIGHT;
+
+        let char_entropy = Self::calculate_char_entropy(sentence);
+        score += char_entropy * CHAR_ENTROPY_WEIGHT;
+
+        score
+    }
+
+    /// Calculates character entropy (measure of text randomness/information content).
+    pub fn calculate_char_entropy(text: &str) -> f32 {
+        let chars: Vec<char> = text.chars().collect();
+        if chars.is_empty() {
+            return 0.0;
+        }
+
+        let estimated_unique = (chars.len() as f32 * 0.1).ceil() as usize;
+        let mut char_freq = AHashMap::with_capacity(estimated_unique.max(26));
+
+        for &ch in &chars {
+            let lowercase_ch = ch
+                .to_lowercase()
+                .next()
+                .expect("to_lowercase() must yield at least one character for valid Unicode");
+            *char_freq.entry(lowercase_ch).or_insert(0) += 1;
+        }
+
+        let total_chars = chars.len() as f32;
+        char_freq
+            .values()
+            .map(|&freq| {
+                let p = freq as f32 / total_chars;
+                if p > 0.0 { -p * p.log2() } else { 0.0 }
+            })
+            .sum::<f32>()
+            .min(5.0)
+    }
+
+    /// Checks if a word has important characteristics that should be preserved.
+    pub fn has_important_characteristics(word: &str) -> bool {
+        if word.len() > 1 && word.chars().all(|c| c.is_uppercase()) {
+            return true;
+        }
+
+        if word.chars().any(|c| c.is_numeric()) {
+            return true;
+        }
+
+        if word.len() > 10 {
+            return true;
+        }
+
+        let uppercase_count = word.chars().filter(|c| c.is_uppercase()).count();
+        if uppercase_count > 1 && uppercase_count < word.len() {
+            return true;
+        }
+
+        if Self::has_cjk_importance(word) {
+            return true;
+        }
+
+        false
+    }
+
+    /// Checks if a CJK word has important characteristics.
+    pub fn has_cjk_importance(word: &str) -> bool {
+        let chars: Vec<char> = word.chars().collect();
+
+        let has_cjk = chars.iter().any(|&c| c as u32 >= 0x4E00 && (c as u32) <= 0x9FFF);
+        if !has_cjk {
+            return false;
+        }
+
+        let important_radicals = [
+            '学', '智', '能', '技', '术', '法', '算', '理', '科', '研', '究', '发', '展', '系', '统', '模', '型', '方',
+            '式', '过', '程', '结', '构', '功', '效', '应', '分', '析', '计', '算', '数', '据', '信', '息', '处', '理',
+            '语', '言', '文', '生', '成', '产', '用', '作', '为', '成', '变', '化', '转', '换', '提', '高', '网', '络',
+            '神', '经', '机', '器', '人', '工', '智', '能', '自', '然', '复',
+        ];
+
+        for &char in &chars {
+            if important_radicals.contains(&char) {
+                return true;
+            }
+        }
+
+        if chars.len() == 2 && has_cjk {
+            let has_technical = chars.iter().any(|&c| {
+                let code = c as u32;
+                (0x4E00..=0x4FFF).contains(&code)
+                    || (0x5000..=0x51FF).contains(&code)
+                    || (0x6700..=0x68FF).contains(&code)
+                    || (0x7500..=0x76FF).contains(&code)
+            });
+
+            if has_technical {
+                return true;
+            }
+        }
+
+        false
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_calculate_char_entropy() {
+        let low_entropy = TextAnalyzer::calculate_char_entropy("aaaaaaa");
+        assert!(low_entropy < 1.0);
+
+        let high_entropy = TextAnalyzer::calculate_char_entropy("abcdefg123");
+        assert!(high_entropy > low_entropy);
+    }
+
+    #[test]
+    fn test_important_word_characteristics() {
+        assert!(TextAnalyzer::has_important_characteristics("IMPORTANT"));
+        assert!(TextAnalyzer::has_important_characteristics("COVID-19"));
+        assert!(TextAnalyzer::has_important_characteristics("PyTorch"));
+        assert!(TextAnalyzer::has_important_characteristics("verylongword123"));
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/core/mod.rs

@@ -0,0 +1,8 @@
+mod analysis;
+mod punctuation;
+mod reducer;
+mod sentence_selection;
+mod word_filtering;
+
+// Re-export the main public interface
+pub use reducer::TokenReducer;

data/vendor/kreuzberg/src/text/token_reduction/core/punctuation.rs

@@ -0,0 +1,54 @@
+use once_cell::sync::Lazy;
+use regex::Regex;
+use std::borrow::Cow;
+
+static REPEATED_EXCLAMATION: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[!]{2,}").expect("Repeated exclamation regex pattern is valid and should compile"));
+static REPEATED_QUESTION: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[?]{2,}").expect("Repeated question regex pattern is valid and should compile"));
+static REPEATED_COMMA: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"[,]{2,}").expect("Repeated comma regex pattern is valid and should compile"));
+
+/// Handles punctuation cleaning and normalization.
+pub struct PunctuationCleaner;
+
+impl PunctuationCleaner {
+    /// Cleans excessive punctuation from text using optimized Cow pattern.
+    pub fn clean_punctuation_optimized(text: &str) -> String {
+        let mut result = Cow::Borrowed(text);
+
+        if REPEATED_EXCLAMATION.is_match(&result) {
+            result = Cow::Owned(REPEATED_EXCLAMATION.replace_all(&result, "!").into_owned());
+        }
+        if REPEATED_QUESTION.is_match(&result) {
+            result = Cow::Owned(REPEATED_QUESTION.replace_all(&result, "?").into_owned());
+        }
+        if REPEATED_COMMA.is_match(&result) {
+            result = Cow::Owned(REPEATED_COMMA.replace_all(&result, ",").into_owned());
+        }
+
+        result.into_owned()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_punctuation_normalization() {
+        let input = "Text!!!!!! with????? excessive,,,,,, punctuation";
+        let result = PunctuationCleaner::clean_punctuation_optimized(input);
+
+        assert!(!result.contains("!!!!!!"));
+        assert!(!result.contains("?????"));
+        assert!(!result.contains(",,,,,,"));
+    }
+
+    #[test]
+    fn test_punctuation_no_change() {
+        let input = "Text with normal punctuation!";
+        let result = PunctuationCleaner::clean_punctuation_optimized(input);
+        assert_eq!(result, input);
+    }
+}