kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/text/token_reduction/mod.rs

@@ -1,160 +1,160 @@
-mod cjk_utils;
-mod config;
-mod core;
-mod filters;
-mod semantic;
-mod simd_text;
-
-pub use config::{ReductionLevel, TokenReductionConfig};
-pub use core::TokenReducer;
-
-// TODO: reorganize token_reduction - move out of text, and reorganize text properly into utils etc.
-
-/// Reduces token count in text while preserving meaning and structure.
-///
-/// This function removes stopwords, redundancy, and applies compression techniques
-/// based on the specified reduction level. Supports 64 languages with automatic
-/// stopword removal and optional semantic clustering.
-///
-/// # Arguments
-///
-/// * `text` - The input text to reduce
-/// * `config` - Configuration specifying reduction level and options
-/// * `language_hint` - Optional ISO 639-3 language code (e.g., "eng", "spa")
-///
-/// # Returns
-///
-/// Returns the reduced text with preserved structure (markdown, code blocks).
-///
-/// # Errors
-///
-/// Returns an error if the language hint is invalid or stopwords cannot be loaded.
-///
-/// # Examples
-///
-/// ```rust
-/// use kreuzberg::text::token_reduction::{reduce_tokens, TokenReductionConfig, ReductionLevel};
-///
-/// let text = "This is a simple example text with some stopwords.";
-/// let config = TokenReductionConfig::default();
-/// let reduced = reduce_tokens(text, &config, Some("eng"))?;
-/// println!("Reduced: {}", reduced);
-/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
-/// ```
-pub fn reduce_tokens(
-    text: &str,
-    config: &TokenReductionConfig,
-    language_hint: Option<&str>,
-) -> crate::error::Result<String> {
-    let reducer = TokenReducer::new(config, language_hint)?;
-    Ok(reducer.reduce(text))
-}
-
-/// Reduces token count for multiple texts efficiently using parallel processing.
-///
-/// This function processes multiple texts in parallel using Rayon, providing
-/// significant performance improvements for batch operations. All texts use the
-/// same configuration and language hint for consistency.
-///
-/// # Arguments
-///
-/// * `texts` - Slice of text references to reduce
-/// * `config` - Configuration specifying reduction level and options
-/// * `language_hint` - Optional ISO 639-3 language code (e.g., "eng", "spa")
-///
-/// # Returns
-///
-/// Returns a vector of reduced texts in the same order as the input.
-///
-/// # Errors
-///
-/// Returns an error if the language hint is invalid or stopwords cannot be loaded.
-///
-/// # Examples
-///
-/// ```rust
-/// use kreuzberg::text::token_reduction::{batch_reduce_tokens, TokenReductionConfig, ReductionLevel};
-///
-/// let texts = vec![
-///     "This is the first document with some text.",
-///     "Here is another document with different content.",
-///     "And finally, a third document to process.",
-/// ];
-/// let config = TokenReductionConfig::default();
-/// let reduced = batch_reduce_tokens(&texts, &config, Some("eng"))?;
-/// assert_eq!(reduced.len(), 3);
-/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
-/// ```
-pub fn batch_reduce_tokens(
-    texts: &[&str],
-    config: &TokenReductionConfig,
-    language_hint: Option<&str>,
-) -> crate::error::Result<Vec<String>> {
-    let reducer = TokenReducer::new(config, language_hint)?;
-    Ok(reducer.batch_reduce(texts))
-}
-
-/// Calculates detailed statistics comparing original and reduced text.
-///
-/// Provides comprehensive metrics including reduction percentages and absolute
-/// counts for both characters and tokens. Useful for analyzing the effectiveness
-/// of token reduction and monitoring compression ratios.
-///
-/// # Arguments
-///
-/// * `original` - The original text before reduction
-/// * `reduced` - The reduced text after applying token reduction
-///
-/// # Returns
-///
-/// Returns a tuple with the following statistics (in order):
-/// 1. `char_reduction` (f64) - Character reduction ratio (0.0 to 1.0)
-/// 2. `token_reduction` (f64) - Token reduction ratio (0.0 to 1.0)
-/// 3. `original_chars` (usize) - Original character count
-/// 4. `reduced_chars` (usize) - Reduced character count
-/// 5. `original_tokens` (usize) - Original token count (whitespace-delimited)
-/// 6. `reduced_tokens` (usize) - Reduced token count (whitespace-delimited)
-///
-/// # Examples
-///
-/// ```rust
-/// use kreuzberg::text::token_reduction::{reduce_tokens, get_reduction_statistics, TokenReductionConfig, ReductionLevel};
-///
-/// let original = "This is a simple example text with some stopwords and redundancy.";
-/// let config = TokenReductionConfig::default();
-/// let reduced = reduce_tokens(original, &config, Some("eng"))?;
-///
-/// let (char_ratio, token_ratio, orig_chars, red_chars, orig_tokens, red_tokens) =
-///     get_reduction_statistics(original, &reduced);
-///
-/// println!("Reduced {:.1}% of characters ({} -> {})", char_ratio * 100.0, orig_chars, red_chars);
-/// println!("Reduced {:.1}% of tokens ({} -> {})", token_ratio * 100.0, orig_tokens, red_tokens);
-/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
-/// ```
-pub fn get_reduction_statistics(original: &str, reduced: &str) -> (f64, f64, usize, usize, usize, usize) {
-    let original_chars = original.chars().count();
-    let reduced_chars = reduced.chars().count();
-    let original_tokens = original.split_whitespace().count();
-    let reduced_tokens = reduced.split_whitespace().count();
-
-    let char_reduction = if original_chars > 0 {
-        1.0 - (reduced_chars as f64 / original_chars as f64)
-    } else {
-        0.0
-    };
-
-    let token_reduction = if original_tokens > 0 {
-        1.0 - (reduced_tokens as f64 / original_tokens as f64)
-    } else {
-        0.0
-    };
-
-    (
-        char_reduction,
-        token_reduction,
-        original_chars,
-        reduced_chars,
-        original_tokens,
-        reduced_tokens,
-    )
-}
+mod cjk_utils;
+mod config;
+mod core;
+mod filters;
+mod semantic;
+mod simd_text;
+
+pub use config::{ReductionLevel, TokenReductionConfig};
+pub use core::TokenReducer;
+
+// TODO: reorganize token_reduction - move out of text, and reorganize text properly into utils etc.
+
+/// Reduces token count in text while preserving meaning and structure.
+///
+/// This function removes stopwords, redundancy, and applies compression techniques
+/// based on the specified reduction level. Supports 64 languages with automatic
+/// stopword removal and optional semantic clustering.
+///
+/// # Arguments
+///
+/// * `text` - The input text to reduce
+/// * `config` - Configuration specifying reduction level and options
+/// * `language_hint` - Optional ISO 639-3 language code (e.g., "eng", "spa")
+///
+/// # Returns
+///
+/// Returns the reduced text with preserved structure (markdown, code blocks).
+///
+/// # Errors
+///
+/// Returns an error if the language hint is invalid or stopwords cannot be loaded.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::text::token_reduction::{reduce_tokens, TokenReductionConfig, ReductionLevel};
+///
+/// let text = "This is a simple example text with some stopwords.";
+/// let config = TokenReductionConfig::default();
+/// let reduced = reduce_tokens(text, &config, Some("eng"))?;
+/// println!("Reduced: {}", reduced);
+/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
+/// ```
+pub fn reduce_tokens(
+    text: &str,
+    config: &TokenReductionConfig,
+    language_hint: Option<&str>,
+) -> crate::error::Result<String> {
+    let reducer = TokenReducer::new(config, language_hint)?;
+    Ok(reducer.reduce(text))
+}
+
+/// Reduces token count for multiple texts efficiently using parallel processing.
+///
+/// This function processes multiple texts in parallel using Rayon, providing
+/// significant performance improvements for batch operations. All texts use the
+/// same configuration and language hint for consistency.
+///
+/// # Arguments
+///
+/// * `texts` - Slice of text references to reduce
+/// * `config` - Configuration specifying reduction level and options
+/// * `language_hint` - Optional ISO 639-3 language code (e.g., "eng", "spa")
+///
+/// # Returns
+///
+/// Returns a vector of reduced texts in the same order as the input.
+///
+/// # Errors
+///
+/// Returns an error if the language hint is invalid or stopwords cannot be loaded.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::text::token_reduction::{batch_reduce_tokens, TokenReductionConfig, ReductionLevel};
+///
+/// let texts = vec![
+///     "This is the first document with some text.",
+///     "Here is another document with different content.",
+///     "And finally, a third document to process.",
+/// ];
+/// let config = TokenReductionConfig::default();
+/// let reduced = batch_reduce_tokens(&texts, &config, Some("eng"))?;
+/// assert_eq!(reduced.len(), 3);
+/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
+/// ```
+pub fn batch_reduce_tokens(
+    texts: &[&str],
+    config: &TokenReductionConfig,
+    language_hint: Option<&str>,
+) -> crate::error::Result<Vec<String>> {
+    let reducer = TokenReducer::new(config, language_hint)?;
+    Ok(reducer.batch_reduce(texts))
+}
+
+/// Calculates detailed statistics comparing original and reduced text.
+///
+/// Provides comprehensive metrics including reduction percentages and absolute
+/// counts for both characters and tokens. Useful for analyzing the effectiveness
+/// of token reduction and monitoring compression ratios.
+///
+/// # Arguments
+///
+/// * `original` - The original text before reduction
+/// * `reduced` - The reduced text after applying token reduction
+///
+/// # Returns
+///
+/// Returns a tuple with the following statistics (in order):
+/// 1. `char_reduction` (f64) - Character reduction ratio (0.0 to 1.0)
+/// 2. `token_reduction` (f64) - Token reduction ratio (0.0 to 1.0)
+/// 3. `original_chars` (usize) - Original character count
+/// 4. `reduced_chars` (usize) - Reduced character count
+/// 5. `original_tokens` (usize) - Original token count (whitespace-delimited)
+/// 6. `reduced_tokens` (usize) - Reduced token count (whitespace-delimited)
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::text::token_reduction::{reduce_tokens, get_reduction_statistics, TokenReductionConfig, ReductionLevel};
+///
+/// let original = "This is a simple example text with some stopwords and redundancy.";
+/// let config = TokenReductionConfig::default();
+/// let reduced = reduce_tokens(original, &config, Some("eng"))?;
+///
+/// let (char_ratio, token_ratio, orig_chars, red_chars, orig_tokens, red_tokens) =
+///     get_reduction_statistics(original, &reduced);
+///
+/// println!("Reduced {:.1}% of characters ({} -> {})", char_ratio * 100.0, orig_chars, red_chars);
+/// println!("Reduced {:.1}% of tokens ({} -> {})", token_ratio * 100.0, orig_tokens, red_tokens);
+/// # Ok::<(), kreuzberg::error::KreuzbergError>(())
+/// ```
+pub fn get_reduction_statistics(original: &str, reduced: &str) -> (f64, f64, usize, usize, usize, usize) {
+    let original_chars = original.chars().count();
+    let reduced_chars = reduced.chars().count();
+    let original_tokens = original.split_whitespace().count();
+    let reduced_tokens = reduced.split_whitespace().count();
+
+    let char_reduction = if original_chars > 0 {
+        1.0 - (reduced_chars as f64 / original_chars as f64)
+    } else {
+        0.0
+    };
+
+    let token_reduction = if original_tokens > 0 {
+        1.0 - (reduced_tokens as f64 / original_tokens as f64)
+    } else {
+        0.0
+    };
+
+    (
+        char_reduction,
+        token_reduction,
+        original_chars,
+        reduced_chars,
+        original_tokens,
+        reduced_tokens,
+    )
+}