kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs

@@ -1,164 +1,164 @@
-use std::ops::RangeInclusive;
-
-/// CJK text tokenizer for token reduction.
-///
-/// This tokenizer uses bigram (2-character) tokenization for CJK text,
-/// which is appropriate for token reduction where we want to preserve
-/// meaning while reducing token count.
-///
-/// # Unicode Range Coverage
-///
-/// **Currently covers:** CJK Unified Ideographs (U+4E00-U+9FFF)
-/// - Covers ~20,992 common Chinese/Japanese Kanji characters
-/// - Sufficient for token reduction purposes with Chinese and Japanese text
-///
-/// **Intentionally excluded:**
-/// - Hiragana (U+3040-U+309F): Japanese phonetic script
-/// - Katakana (U+30A0-U+30FF): Japanese phonetic script
-/// - Hangul (U+AC00-U+D7AF): Korean alphabet
-///
-/// These exclusions are intentional for token reduction. Hiragana and Katakana
-/// are typically tokenized with whitespace, and Hangul has different tokenization
-/// requirements. If broader CJK support is needed, consider expanding the range
-/// or using language-specific tokenizers.
-pub struct CjkTokenizer {
-    cjk_range: RangeInclusive<u32>,
-}
-
-impl CjkTokenizer {
-    pub fn new() -> Self {
-        Self {
-            cjk_range: 0x4E00..=0x9FFF,
-        }
-    }
-
-    /// Checks if a character is a CJK Unified Ideograph (U+4E00-U+9FFF).
-    ///
-    /// Returns true for Chinese characters and Japanese Kanji, false for
-    /// Hiragana, Katakana, Hangul, and non-CJK characters.
-    #[inline]
-    pub fn is_cjk_char(&self, c: char) -> bool {
-        self.cjk_range.contains(&(c as u32))
-    }
-
-    #[inline]
-    pub fn has_cjk(&self, text: &str) -> bool {
-        text.chars().any(|c| self.is_cjk_char(c))
-    }
-
-    pub fn tokenize_cjk_string(&self, text: &str) -> Vec<String> {
-        let chars: Vec<char> = text.chars().collect();
-        self.tokenize_cjk_chars(&chars)
-    }
-
-    pub fn tokenize_cjk_chars(&self, chars: &[char]) -> Vec<String> {
-        chars
-            .chunks(2)
-            .map(|chunk| {
-                if chunk.len() == 2 {
-                    format!("{}{}", chunk[0], chunk[1])
-                } else {
-                    chunk[0].to_string()
-                }
-            })
-            .collect()
-    }
-
-    pub fn tokenize_mixed_text(&self, text: &str) -> Vec<String> {
-        let whitespace_tokens: Vec<&str> = text.split_whitespace().collect();
-
-        if whitespace_tokens.is_empty() {
-            return if text.is_empty() {
-                vec![]
-            } else {
-                vec![text.to_string()]
-            };
-        }
-
-        if whitespace_tokens.len() == 1 {
-            let token = whitespace_tokens[0];
-            return if self.has_cjk(token) {
-                self.tokenize_cjk_string(token)
-            } else {
-                vec![token.to_string()]
-            };
-        }
-
-        let mut all_tokens = Vec::new();
-        for token in whitespace_tokens {
-            if self.has_cjk(token) {
-                all_tokens.extend(self.tokenize_cjk_string(token));
-            } else {
-                all_tokens.push(token.to_string());
-            }
-        }
-        all_tokens
-    }
-}
-
-impl Default for CjkTokenizer {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_is_cjk_char() {
-        let tokenizer = CjkTokenizer::new();
-
-        assert!(tokenizer.is_cjk_char('中'));
-        assert!(tokenizer.is_cjk_char('国'));
-        assert!(tokenizer.is_cjk_char('日'));
-        assert!(tokenizer.is_cjk_char('本'));
-
-        assert!(!tokenizer.is_cjk_char('a'));
-        assert!(!tokenizer.is_cjk_char('Z'));
-        assert!(!tokenizer.is_cjk_char('1'));
-        assert!(!tokenizer.is_cjk_char(' '));
-    }
-
-    #[test]
-    fn test_has_cjk() {
-        let tokenizer = CjkTokenizer::new();
-
-        assert!(tokenizer.has_cjk("这是中文"));
-        assert!(tokenizer.has_cjk("mixed 中文 text"));
-        assert!(tokenizer.has_cjk("日本語"));
-
-        assert!(!tokenizer.has_cjk("English text"));
-        assert!(!tokenizer.has_cjk("12345"));
-        assert!(!tokenizer.has_cjk(""));
-    }
-
-    #[test]
-    fn test_tokenize_cjk_string() {
-        let tokenizer = CjkTokenizer::new();
-
-        let tokens = tokenizer.tokenize_cjk_string("中国人");
-        assert_eq!(tokens, vec!["中国", "人"]);
-
-        let tokens = tokenizer.tokenize_cjk_string("四个字");
-        assert_eq!(tokens, vec!["四个", "字"]);
-    }
-
-    #[test]
-    fn test_tokenize_mixed_text() {
-        let tokenizer = CjkTokenizer::new();
-
-        let tokens = tokenizer.tokenize_mixed_text("hello world");
-        assert_eq!(tokens, vec!["hello", "world"]);
-
-        let tokens = tokenizer.tokenize_mixed_text("中国");
-        assert_eq!(tokens, vec!["中国"]);
-
-        let tokens = tokenizer.tokenize_mixed_text("hello 中国 world");
-        assert_eq!(tokens, vec!["hello", "中国", "world"]);
-
-        let tokens = tokenizer.tokenize_mixed_text("学习 machine learning 技术");
-        assert_eq!(tokens, vec!["学习", "machine", "learning", "技术"]);
-    }
-}
+use std::ops::RangeInclusive;
+
+/// CJK text tokenizer for token reduction.
+///
+/// This tokenizer uses bigram (2-character) tokenization for CJK text,
+/// which is appropriate for token reduction where we want to preserve
+/// meaning while reducing token count.
+///
+/// # Unicode Range Coverage
+///
+/// **Currently covers:** CJK Unified Ideographs (U+4E00-U+9FFF)
+/// - Covers ~20,992 common Chinese/Japanese Kanji characters
+/// - Sufficient for token reduction purposes with Chinese and Japanese text
+///
+/// **Intentionally excluded:**
+/// - Hiragana (U+3040-U+309F): Japanese phonetic script
+/// - Katakana (U+30A0-U+30FF): Japanese phonetic script
+/// - Hangul (U+AC00-U+D7AF): Korean alphabet
+///
+/// These exclusions are intentional for token reduction. Hiragana and Katakana
+/// are typically tokenized with whitespace, and Hangul has different tokenization
+/// requirements. If broader CJK support is needed, consider expanding the range
+/// or using language-specific tokenizers.
+pub struct CjkTokenizer {
+    cjk_range: RangeInclusive<u32>,
+}
+
+impl CjkTokenizer {
+    pub fn new() -> Self {
+        Self {
+            cjk_range: 0x4E00..=0x9FFF,
+        }
+    }
+
+    /// Checks if a character is a CJK Unified Ideograph (U+4E00-U+9FFF).
+    ///
+    /// Returns true for Chinese characters and Japanese Kanji, false for
+    /// Hiragana, Katakana, Hangul, and non-CJK characters.
+    #[inline]
+    pub fn is_cjk_char(&self, c: char) -> bool {
+        self.cjk_range.contains(&(c as u32))
+    }
+
+    #[inline]
+    pub fn has_cjk(&self, text: &str) -> bool {
+        text.chars().any(|c| self.is_cjk_char(c))
+    }
+
+    pub fn tokenize_cjk_string(&self, text: &str) -> Vec<String> {
+        let chars: Vec<char> = text.chars().collect();
+        self.tokenize_cjk_chars(&chars)
+    }
+
+    pub fn tokenize_cjk_chars(&self, chars: &[char]) -> Vec<String> {
+        chars
+            .chunks(2)
+            .map(|chunk| {
+                if chunk.len() == 2 {
+                    format!("{}{}", chunk[0], chunk[1])
+                } else {
+                    chunk[0].to_string()
+                }
+            })
+            .collect()
+    }
+
+    pub fn tokenize_mixed_text(&self, text: &str) -> Vec<String> {
+        let whitespace_tokens: Vec<&str> = text.split_whitespace().collect();
+
+        if whitespace_tokens.is_empty() {
+            return if text.is_empty() {
+                vec![]
+            } else {
+                vec![text.to_string()]
+            };
+        }
+
+        if whitespace_tokens.len() == 1 {
+            let token = whitespace_tokens[0];
+            return if self.has_cjk(token) {
+                self.tokenize_cjk_string(token)
+            } else {
+                vec![token.to_string()]
+            };
+        }
+
+        let mut all_tokens = Vec::new();
+        for token in whitespace_tokens {
+            if self.has_cjk(token) {
+                all_tokens.extend(self.tokenize_cjk_string(token));
+            } else {
+                all_tokens.push(token.to_string());
+            }
+        }
+        all_tokens
+    }
+}
+
+impl Default for CjkTokenizer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_cjk_char() {
+        let tokenizer = CjkTokenizer::new();
+
+        assert!(tokenizer.is_cjk_char('中'));
+        assert!(tokenizer.is_cjk_char('国'));
+        assert!(tokenizer.is_cjk_char('日'));
+        assert!(tokenizer.is_cjk_char('本'));
+
+        assert!(!tokenizer.is_cjk_char('a'));
+        assert!(!tokenizer.is_cjk_char('Z'));
+        assert!(!tokenizer.is_cjk_char('1'));
+        assert!(!tokenizer.is_cjk_char(' '));
+    }
+
+    #[test]
+    fn test_has_cjk() {
+        let tokenizer = CjkTokenizer::new();
+
+        assert!(tokenizer.has_cjk("这是中文"));
+        assert!(tokenizer.has_cjk("mixed 中文 text"));
+        assert!(tokenizer.has_cjk("日本語"));
+
+        assert!(!tokenizer.has_cjk("English text"));
+        assert!(!tokenizer.has_cjk("12345"));
+        assert!(!tokenizer.has_cjk(""));
+    }
+
+    #[test]
+    fn test_tokenize_cjk_string() {
+        let tokenizer = CjkTokenizer::new();
+
+        let tokens = tokenizer.tokenize_cjk_string("中国人");
+        assert_eq!(tokens, vec!["中国", "人"]);
+
+        let tokens = tokenizer.tokenize_cjk_string("四个字");
+        assert_eq!(tokens, vec!["四个", "字"]);
+    }
+
+    #[test]
+    fn test_tokenize_mixed_text() {
+        let tokenizer = CjkTokenizer::new();
+
+        let tokens = tokenizer.tokenize_mixed_text("hello world");
+        assert_eq!(tokens, vec!["hello", "world"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("中国");
+        assert_eq!(tokens, vec!["中国"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("hello 中国 world");
+        assert_eq!(tokens, vec!["hello", "中国", "world"]);
+
+        let tokens = tokenizer.tokenize_mixed_text("学习 machine learning 技术");
+        assert_eq!(tokens, vec!["学习", "machine", "learning", "技术"]);
+    }
+}

data/vendor/kreuzberg/src/text/token_reduction/config.rs

@@ -1,100 +1,100 @@
-use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub enum ReductionLevel {
-    Off = 0,
-    Light = 1,
-    Moderate = 2,
-    Aggressive = 3,
-    Maximum = 4,
-}
-
-impl ReductionLevel {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            ReductionLevel::Off => "off",
-            ReductionLevel::Light => "light",
-            ReductionLevel::Moderate => "moderate",
-            ReductionLevel::Aggressive => "aggressive",
-            ReductionLevel::Maximum => "maximum",
-        }
-    }
-}
-
-impl From<&str> for ReductionLevel {
-    fn from(s: &str) -> Self {
-        match s.to_lowercase().as_str() {
-            "off" => ReductionLevel::Off,
-            "light" => ReductionLevel::Light,
-            "moderate" => ReductionLevel::Moderate,
-            "aggressive" => ReductionLevel::Aggressive,
-            "maximum" => ReductionLevel::Maximum,
-            _ => ReductionLevel::Moderate,
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct TokenReductionConfig {
-    pub level: ReductionLevel,
-    pub language_hint: Option<String>,
-    pub preserve_markdown: bool,
-    pub preserve_code: bool,
-    pub semantic_threshold: f32,
-    pub enable_parallel: bool,
-    pub use_simd: bool,
-    pub custom_stopwords: Option<HashMap<String, Vec<String>>>,
-    pub preserve_patterns: Vec<String>,
-    pub target_reduction: Option<f32>,
-    pub enable_semantic_clustering: bool,
-}
-
-impl Default for TokenReductionConfig {
-    fn default() -> Self {
-        Self {
-            level: ReductionLevel::Moderate,
-            language_hint: None,
-            preserve_markdown: false,
-            preserve_code: true,
-            semantic_threshold: 0.3,
-            enable_parallel: true,
-            use_simd: true,
-            custom_stopwords: None,
-            preserve_patterns: vec![],
-            target_reduction: None,
-            enable_semantic_clustering: false,
-        }
-    }
-}
-
-impl TokenReductionConfig {
-    #[allow(clippy::too_many_arguments)]
-    pub fn new(
-        level: ReductionLevel,
-        language_hint: Option<String>,
-        preserve_markdown: bool,
-        preserve_code: bool,
-        semantic_threshold: f32,
-        enable_parallel: bool,
-        use_simd: bool,
-        custom_stopwords: Option<HashMap<String, Vec<String>>>,
-        preserve_patterns: Option<Vec<String>>,
-        target_reduction: Option<f32>,
-        enable_semantic_clustering: bool,
-    ) -> Self {
-        Self {
-            level,
-            language_hint,
-            preserve_markdown,
-            preserve_code,
-            semantic_threshold: semantic_threshold.clamp(0.0, 1.0),
-            enable_parallel,
-            use_simd,
-            custom_stopwords,
-            preserve_patterns: preserve_patterns.unwrap_or_default(),
-            target_reduction: target_reduction.map(|t| t.clamp(0.0, 1.0)),
-            enable_semantic_clustering,
-        }
-    }
-}
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum ReductionLevel {
+    Off = 0,
+    Light = 1,
+    Moderate = 2,
+    Aggressive = 3,
+    Maximum = 4,
+}
+
+impl ReductionLevel {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReductionLevel::Off => "off",
+            ReductionLevel::Light => "light",
+            ReductionLevel::Moderate => "moderate",
+            ReductionLevel::Aggressive => "aggressive",
+            ReductionLevel::Maximum => "maximum",
+        }
+    }
+}
+
+impl From<&str> for ReductionLevel {
+    fn from(s: &str) -> Self {
+        match s.to_lowercase().as_str() {
+            "off" => ReductionLevel::Off,
+            "light" => ReductionLevel::Light,
+            "moderate" => ReductionLevel::Moderate,
+            "aggressive" => ReductionLevel::Aggressive,
+            "maximum" => ReductionLevel::Maximum,
+            _ => ReductionLevel::Moderate,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct TokenReductionConfig {
+    pub level: ReductionLevel,
+    pub language_hint: Option<String>,
+    pub preserve_markdown: bool,
+    pub preserve_code: bool,
+    pub semantic_threshold: f32,
+    pub enable_parallel: bool,
+    pub use_simd: bool,
+    pub custom_stopwords: Option<HashMap<String, Vec<String>>>,
+    pub preserve_patterns: Vec<String>,
+    pub target_reduction: Option<f32>,
+    pub enable_semantic_clustering: bool,
+}
+
+impl Default for TokenReductionConfig {
+    fn default() -> Self {
+        Self {
+            level: ReductionLevel::Moderate,
+            language_hint: None,
+            preserve_markdown: false,
+            preserve_code: true,
+            semantic_threshold: 0.3,
+            enable_parallel: true,
+            use_simd: true,
+            custom_stopwords: None,
+            preserve_patterns: vec![],
+            target_reduction: None,
+            enable_semantic_clustering: false,
+        }
+    }
+}
+
+impl TokenReductionConfig {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        level: ReductionLevel,
+        language_hint: Option<String>,
+        preserve_markdown: bool,
+        preserve_code: bool,
+        semantic_threshold: f32,
+        enable_parallel: bool,
+        use_simd: bool,
+        custom_stopwords: Option<HashMap<String, Vec<String>>>,
+        preserve_patterns: Option<Vec<String>>,
+        target_reduction: Option<f32>,
+        enable_semantic_clustering: bool,
+    ) -> Self {
+        Self {
+            level,
+            language_hint,
+            preserve_markdown,
+            preserve_code,
+            semantic_threshold: semantic_threshold.clamp(0.0, 1.0),
+            enable_parallel,
+            use_simd,
+            custom_stopwords,
+            preserve_patterns: preserve_patterns.unwrap_or_default(),
+            target_reduction: target_reduction.map(|t| t.clamp(0.0, 1.0)),
+            enable_semantic_clustering,
+        }
+    }
+}