kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.7

data/vendor/kreuzberg/src/language_detection/mod.rs

@@ -4,8 +4,13 @@
 
 use crate::Result;
 use crate::core::config::LanguageDetectionConfig;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
 use whatlang::{Lang, detect};
 
+pub mod processor;
+pub use processor::LanguageDetector;
+
 /// Detect languages in text using whatlang.
 ///
 /// Returns a list of detected language codes (ISO 639-3 format).
@@ -940,3 +945,41 @@ mod tests {
         assert_eq!(langs[0], "eng");
     }
 }
+
+/// Lazy-initialized flag that ensures language detection processor is registered exactly once.
+///
+/// This static is accessed on first use to automatically register the
+/// language detection processor with the plugin registry.
+static PROCESSOR_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_language_detection_processor);
+
+/// Ensure the language detection processor is registered.
+///
+/// This function is called automatically when needed.
+/// It's safe to call multiple times - registration only happens once.
+pub fn ensure_initialized() -> Result<()> {
+    PROCESSOR_INITIALIZED
+        .as_ref()
+        .map(|_| ())
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to register language detection processor: {}", e),
+            plugin_name: "language-detection".to_string(),
+        })
+}
+
+/// Register the language detection processor with the global registry.
+///
+/// This function should be called once at application startup to register
+/// the language detection post-processor.
+///
+/// **Note:** This is called automatically on first use.
+/// Explicit calling is optional.
+pub fn register_language_detection_processor() -> Result<()> {
+    let registry = crate::plugins::registry::get_post_processor_registry();
+    let mut registry = registry
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Post-processor registry lock poisoned: {}", e)))?;
+
+    registry.register(Arc::new(LanguageDetector), 40)?;
+
+    Ok(())
+}

data/vendor/kreuzberg/src/language_detection/processor.rs

@@ -0,0 +1,219 @@
+//! Language detection post-processor.
+//!
+//! This module provides a PostProcessor plugin that detects languages in
+//! extraction results and stores them in the result.
+
+use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
+use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
+use async_trait::async_trait;
+
+/// Post-processor that detects languages in document content.
+///
+/// This processor:
+/// - Runs in the Early processing stage
+/// - Only processes when `config.language_detection` is configured
+/// - Stores detected languages in `result.detected_languages`
+/// - Uses the whatlang library for detection
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::plugins::{Plugin, PostProcessor};
+/// use kreuzberg::language_detection::processor::LanguageDetector;
+///
+/// let processor = LanguageDetector;
+/// assert_eq!(processor.name(), "language-detection");
+/// ```
+#[derive(Debug, Clone, Copy)]
+pub struct LanguageDetector;
+
+impl Plugin for LanguageDetector {
+    fn name(&self) -> &str {
+        "language-detection"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl PostProcessor for LanguageDetector {
+    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+        let lang_config = match &config.language_detection {
+            Some(cfg) => cfg,
+            None => return Ok(()),
+        };
+
+        match super::detect_languages(&result.content, lang_config)
+            .map_err(|e| KreuzbergError::Other(format!("Language detection failed: {}", e)))?
+        {
+            Some(languages) => {
+                result.detected_languages = Some(languages);
+            }
+            None => {
+                result.detected_languages = None;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn processing_stage(&self) -> ProcessingStage {
+        ProcessingStage::Early
+    }
+
+    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
+        config.language_detection.is_some()
+    }
+
+    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
+        let text_length = result.content.len();
+        // Language detection is relatively fast: ~1ms per 1KB
+        (text_length / 1024).max(1) as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::LanguageDetectionConfig;
+    use crate::types::Metadata;
+
+    #[tokio::test]
+    async fn test_language_detector_processor() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+
+        let mut result = ExtractionResult {
+            content: "Hello world! This is a test of the language detection system.".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_some());
+        let langs = result.detected_languages.unwrap();
+        assert!(!langs.is_empty());
+        assert_eq!(langs[0], "eng");
+    }
+
+    #[tokio::test]
+    async fn test_language_detector_no_config() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig::default();
+
+        let mut result = ExtractionResult {
+            content: "Hello world!".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_none());
+    }
+
+    #[test]
+    fn test_language_detector_plugin_interface() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.name(), "language-detection");
+        assert!(!processor.version().is_empty());
+        assert!(processor.initialize().is_ok());
+        assert!(processor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_language_detector_stage() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.processing_stage(), ProcessingStage::Early);
+    }
+
+    #[test]
+    fn test_language_detector_should_process() {
+        let processor = LanguageDetector;
+
+        let result = ExtractionResult {
+            content: "Sample text".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let config_with_lang = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+        assert!(processor.should_process(&result, &config_with_lang));
+
+        let config_without_lang = ExtractionConfig::default();
+        assert!(!processor.should_process(&result, &config_without_lang));
+    }
+
+    #[test]
+    fn test_language_detector_estimated_duration() {
+        let processor = LanguageDetector;
+
+        let short_result = ExtractionResult {
+            content: "Short".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let long_result = ExtractionResult {
+            content: "a".repeat(10000),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let short_duration = processor.estimated_duration_ms(&short_result);
+        let long_duration = processor.estimated_duration_ms(&long_result);
+
+        assert!(long_duration > short_duration);
+    }
+}

data/vendor/kreuzberg/src/lib.rs

@@ -84,13 +84,21 @@ pub use types::*;
 pub use core::extractor::{batch_extract_bytes, batch_extract_file};
 pub use core::extractor::{extract_bytes, extract_file};
 
-pub use core::extractor::{batch_extract_bytes_sync, batch_extract_file_sync, extract_bytes_sync, extract_file_sync};
+// Available in WASM (bytes-based)
+pub use core::extractor::{batch_extract_bytes_sync, extract_bytes_sync};
+
+// Only available with filesystem access
+#[cfg(feature = "tokio-runtime")]
+pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
 
 pub use core::config::{
     ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
-    LanguageDetectionConfig, OcrConfig, PdfConfig, PostProcessorConfig, TokenReductionConfig,
+    LanguageDetectionConfig, OcrConfig, PostProcessorConfig, TokenReductionConfig,
 };
 
+#[cfg(feature = "pdf")]
+pub use core::config::PdfConfig;
+
 pub use core::mime::{
     DOCX_MIME_TYPE, EXCEL_MIME_TYPE, HTML_MIME_TYPE, JSON_MIME_TYPE, MARKDOWN_MIME_TYPE, PDF_MIME_TYPE,
     PLAIN_TEXT_MIME_TYPE, POWER_POINT_MIME_TYPE, XML_MIME_TYPE, detect_mime_type, detect_mime_type_from_bytes,

data/vendor/kreuzberg/src/mcp/mod.rs

@@ -1,3 +1,5 @@
+#![cfg(feature = "mcp")]
+
 //! Model Context Protocol (MCP) server implementation.
 //!
 //! Provides an MCP server that exposes Kreuzberg's document extraction

data/vendor/kreuzberg/src/mcp/server.rs

@@ -750,6 +750,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -786,6 +787,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -807,6 +809,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -825,6 +828,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -1622,19 +1626,17 @@ mod tests {
 
             let result = server.batch_extract_files(Parameters(params)).await;
 
-            if result.is_ok() {
-                let call_result = result.unwrap();
-                if let Some(content) = call_result.content.first()
-                    && let RawContent::Text(text) = &content.raw
-                {
-                    assert!(text.text.contains("Document 1"));
-                    assert!(text.text.contains("Document 2"));
+            if let Ok(call_result) = result
+                && let Some(content) = call_result.content.first()
+                && let RawContent::Text(text) = &content.raw
+            {
+                assert!(text.text.contains("Document 1"));
+                assert!(text.text.contains("Document 2"));
 
-                    let doc1_pos = text.text.find("Document 1");
-                    let doc2_pos = text.text.find("Document 2");
-                    if let (Some(pos1), Some(pos2)) = (doc1_pos, doc2_pos) {
-                        assert!(pos1 < pos2, "Documents should be in order");
-                    }
+                let doc1_pos = text.text.find("Document 1");
+                let doc2_pos = text.text.find("Document 2");
+                if let (Some(pos1), Some(pos2)) = (doc1_pos, doc2_pos) {
+                    assert!(pos1 < pos2, "Documents should be in order");
                 }
             }
         }

data/vendor/kreuzberg/src/ocr/tesseract_backend.rs

@@ -161,6 +161,7 @@ impl OcrBackend for TesseractBackend {
             content: ocr_result.content,
             mime_type: ocr_result.mime_type,
             metadata,
+            pages: None,
             tables: ocr_result
                 .tables
                 .into_iter()
@@ -214,6 +215,7 @@ impl OcrBackend for TesseractBackend {
             content: ocr_result.content,
             mime_type: ocr_result.mime_type,
             metadata,
+            pages: None,
             tables: ocr_result
                 .tables
                 .into_iter()

data/vendor/kreuzberg/src/pdf/error.rs

@@ -10,6 +10,7 @@ pub enum PdfError {
     TextExtractionFailed(String),
     RenderingFailed(String),
     MetadataExtractionFailed(String),
+    ExtractionFailed(String),
     IOError(String),
 }
 
@@ -28,6 +29,7 @@ impl fmt::Display for PdfError {
             PdfError::MetadataExtractionFailed(msg) => {
                 write!(f, "Metadata extraction failed: {}", msg)
             }
+            PdfError::ExtractionFailed(msg) => write!(f, "Extraction failed: {}", msg),
             PdfError::IOError(msg) => write!(f, "I/O error: {}", msg),
         }
     }
@@ -119,4 +121,10 @@ mod tests {
         let err2 = err1.clone();
         assert_eq!(err1.to_string(), err2.to_string());
     }
+
+    #[test]
+    fn test_extraction_failed_error() {
+        let err = PdfError::ExtractionFailed("page data mismatch".to_string());
+        assert_eq!(err.to_string(), "Extraction failed: page data mismatch");
+    }
 }