kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/text/quality_processor.rs

@@ -1,231 +0,0 @@
-//! Quality processing post-processor.
-//!
-//! This module provides a PostProcessor plugin that performs quality assessment and
-//! text cleaning on extraction results.
-//!
-//! # Performance
-//!
-//! This processor optimizes metadata handling by:
-//! - Checking if important metadata fields exist before allocating
-//! - Converting to HashMap only when beneficial metadata is present
-//! - Skipping allocation entirely for documents without metadata
-//!
-//! This avoids unnecessary string cloning for sparse metadata scenarios.
-
-use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
-use crate::{ExtractionConfig, ExtractionResult, Result};
-use async_trait::async_trait;
-
-/// Post-processor that calculates quality score and cleans text.
-///
-/// This processor:
-/// - Runs in the Early processing stage
-/// - Calculates quality score when `config.enable_quality_processing` is true
-/// - Stores quality score in `metadata.additional["quality_score"]`
-/// - Cleans and normalizes extracted text
-///
-/// # Example
-///
-/// ```rust,no_run
-/// use kreuzberg::plugins::{Plugin, PostProcessor};
-/// use kreuzberg::text::QualityProcessor;
-///
-/// let processor = QualityProcessor;
-/// assert_eq!(processor.name(), "quality-processing");
-/// ```
-#[derive(Debug, Clone, Copy)]
-pub struct QualityProcessor;
-
-impl Plugin for QualityProcessor {
-    fn name(&self) -> &str {
-        "quality-processing"
-    }
-
-    fn version(&self) -> String {
-        env!("CARGO_PKG_VERSION").to_string()
-    }
-
-    fn initialize(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn shutdown(&self) -> Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
-impl PostProcessor for QualityProcessor {
-    async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-        let quality_score = if should_use_metadata(&result.metadata) {
-            crate::text::quality::calculate_quality_score(&result.content, Some(&result.metadata.additional))
-        } else {
-            crate::text::quality::calculate_quality_score(&result.content, None)
-        };
-
-        result.metadata.additional.insert(
-            "quality_score".to_string(),
-            serde_json::Value::Number(
-                serde_json::Number::from_f64(quality_score).unwrap_or(serde_json::Number::from(0)),
-            ),
-        );
-
-        Ok(())
-    }
-
-    fn processing_stage(&self) -> ProcessingStage {
-        ProcessingStage::Early
-    }
-
-    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
-        config.enable_quality_processing
-    }
-
-    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
-        let text_length = result.content.len();
-        (text_length / 102400).max(1) as u64
-    }
-}
-
-/// Check if metadata contains any important fields without allocation.
-///
-/// # Performance
-///
-/// O(1) check avoiding HashMap allocation when metadata is sparse.
-/// Only allocates HashMap when important metadata fields are present.
-fn should_use_metadata(metadata: &crate::types::Metadata) -> bool {
-    const IMPORTANT_FIELDS: &[&str] = &["title", "author", "subject", "description", "keywords"];
-    IMPORTANT_FIELDS
-        .iter()
-        .any(|field| metadata.additional.contains_key(*field))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::types::Metadata;
-
-    #[tokio::test]
-    async fn test_quality_processor() {
-        let processor = QualityProcessor;
-        let config = ExtractionConfig {
-            enable_quality_processing: true,
-            ..Default::default()
-        };
-
-        let mut result = ExtractionResult {
-	            content: "This is a well-written paragraph with proper structure. It contains multiple sentences. The quality should be good.".to_string(),
-	            mime_type: "text/plain".to_string(),
-	            metadata: Metadata::default(),
-	            tables: vec![],
-	            detected_languages: None,
-	            chunks: None,
-	            images: None,
-	            pages: None,
-	        };
-
-        processor.process(&mut result, &config).await.unwrap();
-
-        assert!(result.metadata.additional.contains_key("quality_score"));
-        let score = result.metadata.additional.get("quality_score").unwrap();
-        assert!(score.is_number());
-    }
-
-    #[tokio::test]
-    async fn test_quality_processor_disabled() {
-        let processor = QualityProcessor;
-        let config = ExtractionConfig {
-            enable_quality_processing: false,
-            ..Default::default()
-        };
-
-        let mut result = ExtractionResult {
-            content: "Some text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        processor.process(&mut result, &config).await.unwrap();
-    }
-
-    #[test]
-    fn test_quality_processor_plugin_interface() {
-        let processor = QualityProcessor;
-        assert_eq!(processor.name(), "quality-processing");
-        assert!(!processor.version().is_empty());
-        assert!(processor.initialize().is_ok());
-        assert!(processor.shutdown().is_ok());
-    }
-
-    #[test]
-    fn test_quality_processor_stage() {
-        let processor = QualityProcessor;
-        assert_eq!(processor.processing_stage(), ProcessingStage::Early);
-    }
-
-    #[test]
-    fn test_quality_processor_should_process() {
-        let processor = QualityProcessor;
-
-        let result = ExtractionResult {
-            content: "Sample text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let config_with_quality = ExtractionConfig {
-            enable_quality_processing: true,
-            ..Default::default()
-        };
-        assert!(processor.should_process(&result, &config_with_quality));
-
-        let config_without_quality = ExtractionConfig {
-            enable_quality_processing: false,
-            ..Default::default()
-        };
-        assert!(!processor.should_process(&result, &config_without_quality));
-    }
-
-    #[test]
-    fn test_quality_processor_estimated_duration() {
-        let processor = QualityProcessor;
-
-        let short_result = ExtractionResult {
-            content: "Short".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let long_result = ExtractionResult {
-            content: "a".repeat(1000000),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let short_duration = processor.estimated_duration_ms(&short_result);
-        let long_duration = processor.estimated_duration_ms(&long_result);
-
-        assert!(long_duration > short_duration);
-    }
-}

data/vendor/kreuzberg/src/text/utf8_validation.rs

@@ -1,193 +0,0 @@
-//! SIMD-accelerated UTF-8 validation.
-//!
-//! This module provides high-performance UTF-8 validation using SIMD instructions
-//! when available. On platforms without SIMD support, it falls back to standard
-//! validation.
-//!
-//! # Performance
-//!
-//! SIMD validation can process 16-32 bytes per cycle, providing 15-20% improvement
-//! over standard byte-by-byte validation on text-heavy operations.
-//!
-//! # Example
-//!
-//! ```rust
-//! use kreuzberg::text::utf8_validation::from_utf8;
-//!
-//! let bytes = b"Hello, UTF-8 world!";
-//! let result = from_utf8(bytes).expect("valid UTF-8");
-//! assert_eq!(result, "Hello, UTF-8 world!");
-//! ```
-
-/// Validates and converts bytes to string using SIMD when available.
-///
-/// This function attempts to use SIMD UTF-8 validation if the `simd-utf8` feature
-/// is enabled and the platform supports it. Otherwise, it falls back to the standard
-/// `std::str::from_utf8()` validation.
-///
-/// # Arguments
-///
-/// * `bytes` - The byte slice to validate and convert
-///
-/// # Returns
-///
-/// `Ok(&str)` if the bytes are valid UTF-8, `Err(std::str::Utf8Error)` otherwise.
-///
-/// # Safety
-///
-/// This function is safe and does not use any unsafe code directly. The underlying
-/// SIMD validation (when enabled) is contained within the simdutf8 crate and is safe.
-#[inline]
-pub fn from_utf8(bytes: &[u8]) -> Result<&str, std::str::Utf8Error> {
-    #[cfg(feature = "simd-utf8")]
-    {
-        simdutf8::basic::from_utf8(bytes).map_err(|_| {
-            #[allow(invalid_from_utf8)]
-            let err = std::str::from_utf8(&[0xFF, 0xFF, 0xFF, 0xFF]).unwrap_err();
-            err
-        })
-    }
-
-    #[cfg(not(feature = "simd-utf8"))]
-    {
-        std::str::from_utf8(bytes)
-    }
-}
-
-/// Validates and converts owned bytes to String using SIMD when available.
-///
-/// This function converts bytes to an owned String, validating UTF-8 using SIMD
-/// when available. The caller's bytes are consumed to create the String.
-///
-/// # Arguments
-///
-/// * `bytes` - The byte vector to validate and convert
-///
-/// # Returns
-///
-/// `Ok(String)` if the bytes are valid UTF-8, `Err(std::string::FromUtf8Error)` otherwise.
-///
-/// # Performance
-///
-/// When enabled, SIMD validation significantly reduces the time spent on validation,
-/// especially for large text documents.
-#[inline]
-pub fn string_from_utf8(bytes: Vec<u8>) -> Result<String, std::string::FromUtf8Error> {
-    #[cfg(feature = "simd-utf8")]
-    {
-        #[allow(clippy::collapsible_if)]
-        if simdutf8::basic::from_utf8(&bytes).is_ok() {
-            #[allow(unsafe_code)]
-            Ok(unsafe { String::from_utf8_unchecked(bytes) })
-        } else {
-            String::from_utf8(bytes)
-        }
-    }
-
-    #[cfg(not(feature = "simd-utf8"))]
-    {
-        String::from_utf8(bytes)
-    }
-}
-
-/// Validates bytes as UTF-8 without conversion to string slice.
-///
-/// Returns `true` if the bytes represent valid UTF-8, `false` otherwise.
-/// This is useful when you only need to check validity without constructing a string.
-///
-/// # Arguments
-///
-/// * `bytes` - The byte slice to validate
-///
-/// # Returns
-///
-/// `true` if valid UTF-8, `false` otherwise.
-///
-/// # Performance
-///
-/// This function is optimized for early exit on invalid sequences.
-#[inline]
-pub fn is_valid_utf8(bytes: &[u8]) -> bool {
-    #[cfg(feature = "simd-utf8")]
-    {
-        simdutf8::basic::from_utf8(bytes).is_ok()
-    }
-
-    #[cfg(not(feature = "simd-utf8"))]
-    {
-        std::str::from_utf8(bytes).is_ok()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_valid_ascii() {
-        let bytes = b"Hello, world!";
-        let result = from_utf8(bytes).unwrap();
-        assert_eq!(result, "Hello, world!");
-    }
-
-    #[test]
-    fn test_valid_utf8_multibyte() {
-        let bytes = "Hello, 世界! 🌍".as_bytes();
-        let result = from_utf8(bytes).unwrap();
-        assert_eq!(result, "Hello, 世界! 🌍");
-    }
-
-    #[test]
-    fn test_empty_bytes() {
-        let bytes = b"";
-        let result = from_utf8(bytes).unwrap();
-        assert_eq!(result, "");
-    }
-
-    #[test]
-    fn test_invalid_utf8() {
-        let bytes: &[u8] = &[0xFF, 0xFE];
-        assert!(from_utf8(bytes).is_err());
-    }
-
-    #[test]
-    fn test_is_valid_utf8_true() {
-        let bytes = "Valid UTF-8 text".as_bytes();
-        assert!(is_valid_utf8(bytes));
-    }
-
-    #[test]
-    fn test_is_valid_utf8_false() {
-        let bytes: &[u8] = &[0xC0, 0x80];
-        assert!(!is_valid_utf8(bytes));
-    }
-
-    #[test]
-    fn test_string_from_utf8_valid() {
-        let bytes = b"Test string".to_vec();
-        let result = string_from_utf8(bytes).unwrap();
-        assert_eq!(result, "Test string");
-    }
-
-    #[test]
-    fn test_string_from_utf8_invalid() {
-        let bytes: Vec<u8> = vec![0xFF, 0xFE];
-        assert!(string_from_utf8(bytes).is_err());
-    }
-
-    #[test]
-    fn test_large_valid_text() {
-        let text = "a".repeat(100_000);
-        let bytes = text.as_bytes();
-        let result = from_utf8(bytes).unwrap();
-        assert_eq!(result.len(), 100_000);
-    }
-
-    #[test]
-    fn test_mixed_unicode() {
-        let text = "Latin αλφα 中文 العربية עברית Кириллица";
-        let bytes = text.as_bytes();
-        let result = from_utf8(bytes).unwrap();
-        assert_eq!(result, text);
-    }
-}