kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/src/language_detection/mod.rs

@@ -4,8 +4,13 @@
 
 use crate::Result;
 use crate::core::config::LanguageDetectionConfig;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
 use whatlang::{Lang, detect};
 
+pub mod processor;
+pub use processor::LanguageDetector;
+
 /// Detect languages in text using whatlang.
 ///
 /// Returns a list of detected language codes (ISO 639-3 format).
@@ -940,3 +945,41 @@ mod tests {
         assert_eq!(langs[0], "eng");
     }
 }
+
+/// Lazy-initialized flag that ensures language detection processor is registered exactly once.
+///
+/// This static is accessed on first use to automatically register the
+/// language detection processor with the plugin registry.
+static PROCESSOR_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_language_detection_processor);
+
+/// Ensure the language detection processor is registered.
+///
+/// This function is called automatically when needed.
+/// It's safe to call multiple times - registration only happens once.
+pub fn ensure_initialized() -> Result<()> {
+    PROCESSOR_INITIALIZED
+        .as_ref()
+        .map(|_| ())
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to register language detection processor: {}", e),
+            plugin_name: "language-detection".to_string(),
+        })
+}
+
+/// Register the language detection processor with the global registry.
+///
+/// This function should be called once at application startup to register
+/// the language detection post-processor.
+///
+/// **Note:** This is called automatically on first use.
+/// Explicit calling is optional.
+pub fn register_language_detection_processor() -> Result<()> {
+    let registry = crate::plugins::registry::get_post_processor_registry();
+    let mut registry = registry
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Post-processor registry lock poisoned: {}", e)))?;
+
+    registry.register(Arc::new(LanguageDetector), 40)?;
+
+    Ok(())
+}

data/vendor/kreuzberg/src/language_detection/processor.rs

@@ -0,0 +1,219 @@
+//! Language detection post-processor.
+//!
+//! This module provides a PostProcessor plugin that detects languages in
+//! extraction results and stores them in the result.
+
+use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
+use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
+use async_trait::async_trait;
+
+/// Post-processor that detects languages in document content.
+///
+/// This processor:
+/// - Runs in the Early processing stage
+/// - Only processes when `config.language_detection` is configured
+/// - Stores detected languages in `result.detected_languages`
+/// - Uses the whatlang library for detection
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::plugins::{Plugin, PostProcessor};
+/// use kreuzberg::language_detection::processor::LanguageDetector;
+///
+/// let processor = LanguageDetector;
+/// assert_eq!(processor.name(), "language-detection");
+/// ```
+#[derive(Debug, Clone, Copy)]
+pub struct LanguageDetector;
+
+impl Plugin for LanguageDetector {
+    fn name(&self) -> &str {
+        "language-detection"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl PostProcessor for LanguageDetector {
+    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+        let lang_config = match &config.language_detection {
+            Some(cfg) => cfg,
+            None => return Ok(()),
+        };
+
+        match super::detect_languages(&result.content, lang_config)
+            .map_err(|e| KreuzbergError::Other(format!("Language detection failed: {}", e)))?
+        {
+            Some(languages) => {
+                result.detected_languages = Some(languages);
+            }
+            None => {
+                result.detected_languages = None;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn processing_stage(&self) -> ProcessingStage {
+        ProcessingStage::Early
+    }
+
+    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
+        config.language_detection.is_some()
+    }
+
+    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
+        let text_length = result.content.len();
+        // Language detection is relatively fast: ~1ms per 1KB
+        (text_length / 1024).max(1) as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::LanguageDetectionConfig;
+    use crate::types::Metadata;
+
+    #[tokio::test]
+    async fn test_language_detector_processor() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+
+        let mut result = ExtractionResult {
+            content: "Hello world! This is a test of the language detection system.".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_some());
+        let langs = result.detected_languages.unwrap();
+        assert!(!langs.is_empty());
+        assert_eq!(langs[0], "eng");
+    }
+
+    #[tokio::test]
+    async fn test_language_detector_no_config() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig::default();
+
+        let mut result = ExtractionResult {
+            content: "Hello world!".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_none());
+    }
+
+    #[test]
+    fn test_language_detector_plugin_interface() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.name(), "language-detection");
+        assert!(!processor.version().is_empty());
+        assert!(processor.initialize().is_ok());
+        assert!(processor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_language_detector_stage() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.processing_stage(), ProcessingStage::Early);
+    }
+
+    #[test]
+    fn test_language_detector_should_process() {
+        let processor = LanguageDetector;
+
+        let result = ExtractionResult {
+            content: "Sample text".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let config_with_lang = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+        assert!(processor.should_process(&result, &config_with_lang));
+
+        let config_without_lang = ExtractionConfig::default();
+        assert!(!processor.should_process(&result, &config_without_lang));
+    }
+
+    #[test]
+    fn test_language_detector_estimated_duration() {
+        let processor = LanguageDetector;
+
+        let short_result = ExtractionResult {
+            content: "Short".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let long_result = ExtractionResult {
+            content: "a".repeat(10000),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let short_duration = processor.estimated_duration_ms(&short_result);
+        let long_duration = processor.estimated_duration_ms(&long_result);
+
+        assert!(long_duration > short_duration);
+    }
+}

data/vendor/kreuzberg/src/lib.rs

@@ -84,13 +84,21 @@ pub use types::*;
 pub use core::extractor::{batch_extract_bytes, batch_extract_file};
 pub use core::extractor::{extract_bytes, extract_file};
 
-pub use core::extractor::{batch_extract_bytes_sync, batch_extract_file_sync, extract_bytes_sync, extract_file_sync};
+// Available in WASM (bytes-based)
+pub use core::extractor::{batch_extract_bytes_sync, extract_bytes_sync};
+
+// Only available with filesystem access
+#[cfg(feature = "tokio-runtime")]
+pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
 
 pub use core::config::{
     ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
-    LanguageDetectionConfig, OcrConfig, PdfConfig, PostProcessorConfig, TokenReductionConfig,
+    LanguageDetectionConfig, OcrConfig, PostProcessorConfig, TokenReductionConfig,
 };
 
+#[cfg(feature = "pdf")]
+pub use core::config::PdfConfig;
+
 pub use core::mime::{
     DOCX_MIME_TYPE, EXCEL_MIME_TYPE, HTML_MIME_TYPE, JSON_MIME_TYPE, MARKDOWN_MIME_TYPE, PDF_MIME_TYPE,
     PLAIN_TEXT_MIME_TYPE, POWER_POINT_MIME_TYPE, XML_MIME_TYPE, detect_mime_type, detect_mime_type_from_bytes,

data/vendor/kreuzberg/src/mcp/mod.rs

@@ -26,6 +26,9 @@ mod server;
 
 pub use server::{start_mcp_server, start_mcp_server_with_config};
 
+#[cfg(feature = "mcp-http")]
+pub use server::{start_mcp_server_http, start_mcp_server_http_with_config};
+
 pub use server::{BatchExtractFilesParams, DetectMimeTypeParams, ExtractBytesParams, ExtractFileParams, KreuzbergMcp};
 
 #[doc(hidden)]

data/vendor/kreuzberg/src/mcp/server.rs

@@ -12,6 +12,9 @@ use rmcp::{
     transport::stdio,
 };
 
+#[cfg(feature = "mcp-http")]
+use rmcp::transport::streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager};
+
 use crate::{
     ExtractionConfig, ExtractionResult as KreuzbergResult, KreuzbergError, batch_extract_file, batch_extract_file_sync,
     cache, detect_mime_type, extract_bytes, extract_bytes_sync, extract_file, extract_file_sync,
@@ -453,6 +456,109 @@ pub async fn start_mcp_server_with_config(
     Ok(())
 }
 
+/// Start MCP server with HTTP Stream transport.
+///
+/// Uses rmcp's built-in StreamableHttpService for HTTP/SSE support per MCP spec.
+///
+/// # Arguments
+///
+/// * `host` - Host to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number (e.g., 8001)
+///
+/// # Example
+///
+/// ```no_run
+/// use kreuzberg::mcp::start_mcp_server_http;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+///     start_mcp_server_http("127.0.0.1", 8001).await?;
+///     Ok(())
+/// }
+/// ```
+#[cfg(feature = "mcp-http")]
+pub async fn start_mcp_server_http(
+    host: impl AsRef<str>,
+    port: u16,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    use axum::Router;
+    use std::net::SocketAddr;
+
+    let http_service = StreamableHttpService::new(
+        || KreuzbergMcp::new().map_err(|e| std::io::Error::other(e.to_string())),
+        LocalSessionManager::default().into(),
+        Default::default(),
+    );
+
+    let router = Router::new().nest_service("/mcp", http_service);
+
+    let addr: SocketAddr = format!("{}:{}", host.as_ref(), port)
+        .parse()
+        .map_err(|e| format!("Invalid address: {}", e))?;
+
+    #[cfg(feature = "api")]
+    tracing::info!("Starting MCP HTTP server on http://{}", addr);
+
+    let listener = tokio::net::TcpListener::bind(addr).await?;
+    axum::serve(listener, router).await?;
+
+    Ok(())
+}
+
+/// Start MCP HTTP server with custom extraction config.
+///
+/// This variant allows specifying a custom extraction configuration
+/// while using HTTP Stream transport.
+///
+/// # Arguments
+///
+/// * `host` - Host to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number (e.g., 8001)
+/// * `config` - Custom extraction configuration
+///
+/// # Example
+///
+/// ```no_run
+/// use kreuzberg::mcp::start_mcp_server_http_with_config;
+/// use kreuzberg::ExtractionConfig;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+///     let config = ExtractionConfig::default();
+///     start_mcp_server_http_with_config("127.0.0.1", 8001, config).await?;
+///     Ok(())
+/// }
+/// ```
+#[cfg(feature = "mcp-http")]
+pub async fn start_mcp_server_http_with_config(
+    host: impl AsRef<str>,
+    port: u16,
+    config: ExtractionConfig,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    use axum::Router;
+    use std::net::SocketAddr;
+
+    let http_service = StreamableHttpService::new(
+        move || Ok(KreuzbergMcp::with_config(config.clone())),
+        LocalSessionManager::default().into(),
+        Default::default(),
+    );
+
+    let router = Router::new().nest_service("/mcp", http_service);
+
+    let addr: SocketAddr = format!("{}:{}", host.as_ref(), port)
+        .parse()
+        .map_err(|e| format!("Invalid address: {}", e))?;
+
+    #[cfg(feature = "api")]
+    tracing::info!("Starting MCP HTTP server on http://{}", addr);
+
+    let listener = tokio::net::TcpListener::bind(addr).await?;
+    axum::serve(listener, router).await?;
+
+    Ok(())
+}
+
 /// Build extraction config from MCP parameters.
 ///
 /// Starts with the default config and overlays OCR settings from request parameters.
@@ -750,6 +856,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -786,6 +893,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -807,6 +915,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -825,6 +934,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let formatted = format_extraction_result(&result);
@@ -1622,19 +1732,17 @@ mod tests {
 
             let result = server.batch_extract_files(Parameters(params)).await;
 
-            if result.is_ok() {
-                let call_result = result.unwrap();
-                if let Some(content) = call_result.content.first()
-                    && let RawContent::Text(text) = &content.raw
-                {
-                    assert!(text.text.contains("Document 1"));
-                    assert!(text.text.contains("Document 2"));
+            if let Ok(call_result) = result
+                && let Some(content) = call_result.content.first()
+                && let RawContent::Text(text) = &content.raw
+            {
+                assert!(text.text.contains("Document 1"));
+                assert!(text.text.contains("Document 2"));
 
-                    let doc1_pos = text.text.find("Document 1");
-                    let doc2_pos = text.text.find("Document 2");
-                    if let (Some(pos1), Some(pos2)) = (doc1_pos, doc2_pos) {
-                        assert!(pos1 < pos2, "Documents should be in order");
-                    }
+                let doc1_pos = text.text.find("Document 1");
+                let doc2_pos = text.text.find("Document 2");
+                if let (Some(pos1), Some(pos2)) = (doc1_pos, doc2_pos) {
+                    assert!(pos1 < pos2, "Documents should be in order");
                 }
             }
         }

data/vendor/kreuzberg/src/ocr/tesseract_backend.rs

@@ -161,6 +161,7 @@ impl OcrBackend for TesseractBackend {
             content: ocr_result.content,
             mime_type: ocr_result.mime_type,
             metadata,
+            pages: None,
             tables: ocr_result
                 .tables
                 .into_iter()
@@ -214,6 +215,7 @@ impl OcrBackend for TesseractBackend {
             content: ocr_result.content,
             mime_type: ocr_result.mime_type,
             metadata,
+            pages: None,
             tables: ocr_result
                 .tables
                 .into_iter()