kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.15

data/vendor/kreuzberg/src/language_detection/processor.rs

@@ -1,219 +1,219 @@
-//! Language detection post-processor.
-//!
-//! This module provides a PostProcessor plugin that detects languages in
-//! extraction results and stores them in the result.
-
-use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
-use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
-use async_trait::async_trait;
-
-/// Post-processor that detects languages in document content.
-///
-/// This processor:
-/// - Runs in the Early processing stage
-/// - Only processes when `config.language_detection` is configured
-/// - Stores detected languages in `result.detected_languages`
-/// - Uses the whatlang library for detection
-///
-/// # Example
-///
-/// ```rust,no_run
-/// use kreuzberg::plugins::{Plugin, PostProcessor};
-/// use kreuzberg::language_detection::processor::LanguageDetector;
-///
-/// let processor = LanguageDetector;
-/// assert_eq!(processor.name(), "language-detection");
-/// ```
-#[derive(Debug, Clone, Copy)]
-pub struct LanguageDetector;
-
-impl Plugin for LanguageDetector {
-    fn name(&self) -> &str {
-        "language-detection"
-    }
-
-    fn version(&self) -> String {
-        env!("CARGO_PKG_VERSION").to_string()
-    }
-
-    fn initialize(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn shutdown(&self) -> Result<()> {
-        Ok(())
-    }
-}
-
-#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
-impl PostProcessor for LanguageDetector {
-    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
-        let lang_config = match &config.language_detection {
-            Some(cfg) => cfg,
-            None => return Ok(()),
-        };
-
-        match super::detect_languages(&result.content, lang_config)
-            .map_err(|e| KreuzbergError::Other(format!("Language detection failed: {}", e)))?
-        {
-            Some(languages) => {
-                result.detected_languages = Some(languages);
-            }
-            None => {
-                result.detected_languages = None;
-            }
-        }
-
-        Ok(())
-    }
-
-    fn processing_stage(&self) -> ProcessingStage {
-        ProcessingStage::Early
-    }
-
-    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
-        config.language_detection.is_some()
-    }
-
-    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
-        let text_length = result.content.len();
-        // Language detection is relatively fast: ~1ms per 1KB
-        (text_length / 1024).max(1) as u64
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::core::config::LanguageDetectionConfig;
-    use crate::types::Metadata;
-
-    #[tokio::test]
-    async fn test_language_detector_processor() {
-        let processor = LanguageDetector;
-        let config = ExtractionConfig {
-            language_detection: Some(LanguageDetectionConfig {
-                enabled: true,
-                min_confidence: 0.8,
-                detect_multiple: false,
-            }),
-            ..Default::default()
-        };
-
-        let mut result = ExtractionResult {
-            content: "Hello world! This is a test of the language detection system.".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        processor.process(&mut result, &config).await.unwrap();
-
-        assert!(result.detected_languages.is_some());
-        let langs = result.detected_languages.unwrap();
-        assert!(!langs.is_empty());
-        assert_eq!(langs[0], "eng");
-    }
-
-    #[tokio::test]
-    async fn test_language_detector_no_config() {
-        let processor = LanguageDetector;
-        let config = ExtractionConfig::default();
-
-        let mut result = ExtractionResult {
-            content: "Hello world!".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        processor.process(&mut result, &config).await.unwrap();
-
-        assert!(result.detected_languages.is_none());
-    }
-
-    #[test]
-    fn test_language_detector_plugin_interface() {
-        let processor = LanguageDetector;
-        assert_eq!(processor.name(), "language-detection");
-        assert!(!processor.version().is_empty());
-        assert!(processor.initialize().is_ok());
-        assert!(processor.shutdown().is_ok());
-    }
-
-    #[test]
-    fn test_language_detector_stage() {
-        let processor = LanguageDetector;
-        assert_eq!(processor.processing_stage(), ProcessingStage::Early);
-    }
-
-    #[test]
-    fn test_language_detector_should_process() {
-        let processor = LanguageDetector;
-
-        let result = ExtractionResult {
-            content: "Sample text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let config_with_lang = ExtractionConfig {
-            language_detection: Some(LanguageDetectionConfig {
-                enabled: true,
-                min_confidence: 0.8,
-                detect_multiple: false,
-            }),
-            ..Default::default()
-        };
-        assert!(processor.should_process(&result, &config_with_lang));
-
-        let config_without_lang = ExtractionConfig::default();
-        assert!(!processor.should_process(&result, &config_without_lang));
-    }
-
-    #[test]
-    fn test_language_detector_estimated_duration() {
-        let processor = LanguageDetector;
-
-        let short_result = ExtractionResult {
-            content: "Short".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let long_result = ExtractionResult {
-            content: "a".repeat(10000),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let short_duration = processor.estimated_duration_ms(&short_result);
-        let long_duration = processor.estimated_duration_ms(&long_result);
-
-        assert!(long_duration > short_duration);
-    }
-}
+//! Language detection post-processor.
+//!
+//! This module provides a PostProcessor plugin that detects languages in
+//! extraction results and stores them in the result.
+
+use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
+use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
+use async_trait::async_trait;
+
+/// Post-processor that detects languages in document content.
+///
+/// This processor:
+/// - Runs in the Early processing stage
+/// - Only processes when `config.language_detection` is configured
+/// - Stores detected languages in `result.detected_languages`
+/// - Uses the whatlang library for detection
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::plugins::{Plugin, PostProcessor};
+/// use kreuzberg::language_detection::processor::LanguageDetector;
+///
+/// let processor = LanguageDetector;
+/// assert_eq!(processor.name(), "language-detection");
+/// ```
+#[derive(Debug, Clone, Copy)]
+pub struct LanguageDetector;
+
+impl Plugin for LanguageDetector {
+    fn name(&self) -> &str {
+        "language-detection"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl PostProcessor for LanguageDetector {
+    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+        let lang_config = match &config.language_detection {
+            Some(cfg) => cfg,
+            None => return Ok(()),
+        };
+
+        match super::detect_languages(&result.content, lang_config)
+            .map_err(|e| KreuzbergError::Other(format!("Language detection failed: {}", e)))?
+        {
+            Some(languages) => {
+                result.detected_languages = Some(languages);
+            }
+            None => {
+                result.detected_languages = None;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn processing_stage(&self) -> ProcessingStage {
+        ProcessingStage::Early
+    }
+
+    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
+        config.language_detection.is_some()
+    }
+
+    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
+        let text_length = result.content.len();
+        // Language detection is relatively fast: ~1ms per 1KB
+        (text_length / 1024).max(1) as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::LanguageDetectionConfig;
+    use crate::types::Metadata;
+
+    #[tokio::test]
+    async fn test_language_detector_processor() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+
+        let mut result = ExtractionResult {
+            content: "Hello world! This is a test of the language detection system.".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_some());
+        let langs = result.detected_languages.unwrap();
+        assert!(!langs.is_empty());
+        assert_eq!(langs[0], "eng");
+    }
+
+    #[tokio::test]
+    async fn test_language_detector_no_config() {
+        let processor = LanguageDetector;
+        let config = ExtractionConfig::default();
+
+        let mut result = ExtractionResult {
+            content: "Hello world!".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.detected_languages.is_none());
+    }
+
+    #[test]
+    fn test_language_detector_plugin_interface() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.name(), "language-detection");
+        assert!(!processor.version().is_empty());
+        assert!(processor.initialize().is_ok());
+        assert!(processor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_language_detector_stage() {
+        let processor = LanguageDetector;
+        assert_eq!(processor.processing_stage(), ProcessingStage::Early);
+    }
+
+    #[test]
+    fn test_language_detector_should_process() {
+        let processor = LanguageDetector;
+
+        let result = ExtractionResult {
+            content: "Sample text".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let config_with_lang = ExtractionConfig {
+            language_detection: Some(LanguageDetectionConfig {
+                enabled: true,
+                min_confidence: 0.8,
+                detect_multiple: false,
+            }),
+            ..Default::default()
+        };
+        assert!(processor.should_process(&result, &config_with_lang));
+
+        let config_without_lang = ExtractionConfig::default();
+        assert!(!processor.should_process(&result, &config_without_lang));
+    }
+
+    #[test]
+    fn test_language_detector_estimated_duration() {
+        let processor = LanguageDetector;
+
+        let short_result = ExtractionResult {
+            content: "Short".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let long_result = ExtractionResult {
+            content: "a".repeat(10000),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+        };
+
+        let short_duration = processor.estimated_duration_ms(&short_result);
+        let long_duration = processor.estimated_duration_ms(&long_result);
+
+        assert!(long_duration > short_duration);
+    }
+}

data/vendor/kreuzberg/src/lib.rs

@@ -1,113 +1,113 @@
-//! Kreuzberg - High-Performance Document Intelligence Library
-//!
-//! Kreuzberg is a Rust-first document extraction library with language-agnostic plugin support.
-//! It provides fast, accurate extraction from PDFs, images, Office documents, emails, and more.
-//!
-//! # Quick Start
-//!
-//! ```rust,no_run
-//! use kreuzberg::{extract_file_sync, ExtractionConfig};
-//!
-//! # fn main() -> kreuzberg::Result<()> {
-//! // Extract content from a file
-//! let config = ExtractionConfig::default();
-//! let result = extract_file_sync("document.pdf", None, &config)?;
-//! println!("Extracted: {}", result.content);
-//! # Ok(())
-//! # }
-//! ```
-//!
-//! # Architecture
-//!
-//! - **Core Module** (`core`): Main extraction orchestration, MIME detection, config loading
-//! - **Plugin System**: Language-agnostic plugin architecture
-//! - **Extractors**: Format-specific extraction (PDF, images, Office docs, email, etc.)
-//! - **OCR**: Multiple OCR backend support (Tesseract, EasyOCR, PaddleOCR)
-//!
-//! # Features
-//!
-//! - Fast parallel processing with async/await
-//! - Priority-based extractor selection
-//! - Comprehensive MIME type detection (118+ file extensions)
-//! - Configurable caching and quality processing
-//! - Cross-language plugin support (Python, Node.js planned)
-
-#![deny(unsafe_code)]
-
-pub mod cache;
-pub mod core;
-pub mod error;
-pub mod extraction;
-pub mod extractors;
-pub mod panic_context;
-pub mod plugins;
-pub mod text;
-pub mod types;
-
-#[cfg(feature = "quality")]
-pub mod utils;
-
-#[cfg(feature = "api")]
-pub mod api;
-
-#[cfg(feature = "mcp")]
-pub mod mcp;
-
-#[cfg(feature = "chunking")]
-pub mod chunking;
-
-#[cfg(feature = "embeddings")]
-pub mod embeddings;
-
-#[cfg(feature = "ocr")]
-pub mod image;
-
-#[cfg(feature = "language-detection")]
-pub mod language_detection;
-
-#[cfg(feature = "stopwords")]
-pub mod stopwords;
-
-#[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-pub mod keywords;
-
-#[cfg(feature = "ocr")]
-pub mod ocr;
-
-#[cfg(feature = "pdf")]
-pub mod pdf;
-
-pub use error::{KreuzbergError, Result};
-pub use types::*;
-
-#[cfg(feature = "tokio-runtime")]
-pub use core::extractor::{batch_extract_bytes, batch_extract_file};
-pub use core::extractor::{extract_bytes, extract_file};
-
-// Available in WASM (bytes-based)
-pub use core::extractor::{batch_extract_bytes_sync, extract_bytes_sync};
-
-// Only available with filesystem access
-#[cfg(feature = "tokio-runtime")]
-pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
-
-pub use core::config::{
-    ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
-    LanguageDetectionConfig, OcrConfig, PostProcessorConfig, TokenReductionConfig,
-};
-
-#[cfg(feature = "pdf")]
-pub use core::config::PdfConfig;
-
-pub use core::mime::{
-    DOCX_MIME_TYPE, EXCEL_MIME_TYPE, HTML_MIME_TYPE, JSON_MIME_TYPE, MARKDOWN_MIME_TYPE, PDF_MIME_TYPE,
-    PLAIN_TEXT_MIME_TYPE, POWER_POINT_MIME_TYPE, XML_MIME_TYPE, detect_mime_type, detect_mime_type_from_bytes,
-    detect_or_validate, get_extensions_for_mime, validate_mime_type,
-};
-
-pub use plugins::registry::{
-    get_document_extractor_registry, get_ocr_backend_registry, get_post_processor_registry, get_validator_registry,
-};
-
-#[cfg(feature = "embeddings")]
-pub use embeddings::{EMBEDDING_PRESETS, EmbeddingPreset, get_preset, list_presets};
+//! Kreuzberg - High-Performance Document Intelligence Library
+//!
+//! Kreuzberg is a Rust-first document extraction library with language-agnostic plugin support.
+//! It provides fast, accurate extraction from PDFs, images, Office documents, emails, and more.
+//!
+//! # Quick Start
+//!
+//! ```rust,no_run
+//! use kreuzberg::{extract_file_sync, ExtractionConfig};
+//!
+//! # fn main() -> kreuzberg::Result<()> {
+//! // Extract content from a file
+//! let config = ExtractionConfig::default();
+//! let result = extract_file_sync("document.pdf", None, &config)?;
+//! println!("Extracted: {}", result.content);
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Architecture
+//!
+//! - **Core Module** (`core`): Main extraction orchestration, MIME detection, config loading
+//! - **Plugin System**: Language-agnostic plugin architecture
+//! - **Extractors**: Format-specific extraction (PDF, images, Office docs, email, etc.)
+//! - **OCR**: Multiple OCR backend support (Tesseract, EasyOCR, PaddleOCR)
+//!
+//! # Features
+//!
+//! - Fast parallel processing with async/await
+//! - Priority-based extractor selection
+//! - Comprehensive MIME type detection (118+ file extensions)
+//! - Configurable caching and quality processing
+//! - Cross-language plugin support (Python, Node.js planned)
+
+#![deny(unsafe_code)]
+
+pub mod cache;
+pub mod core;
+pub mod error;
+pub mod extraction;
+pub mod extractors;
+pub mod panic_context;
+pub mod plugins;
+pub mod text;
+pub mod types;
+
+#[cfg(feature = "quality")]
+pub mod utils;
+
+#[cfg(feature = "api")]
+pub mod api;
+
+#[cfg(feature = "mcp")]
+pub mod mcp;
+
+#[cfg(feature = "chunking")]
+pub mod chunking;
+
+#[cfg(feature = "embeddings")]
+pub mod embeddings;
+
+#[cfg(feature = "ocr")]
+pub mod image;
+
+#[cfg(feature = "language-detection")]
+pub mod language_detection;
+
+#[cfg(feature = "stopwords")]
+pub mod stopwords;
+
+#[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+pub mod keywords;
+
+#[cfg(feature = "ocr")]
+pub mod ocr;
+
+#[cfg(feature = "pdf")]
+pub mod pdf;
+
+pub use error::{KreuzbergError, Result};
+pub use types::*;
+
+#[cfg(feature = "tokio-runtime")]
+pub use core::extractor::{batch_extract_bytes, batch_extract_file};
+pub use core::extractor::{extract_bytes, extract_file};
+
+// Available in WASM (bytes-based)
+pub use core::extractor::{batch_extract_bytes_sync, extract_bytes_sync};
+
+// Only available with filesystem access
+#[cfg(feature = "tokio-runtime")]
+pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
+
+pub use core::config::{
+    ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
+    LanguageDetectionConfig, OcrConfig, PostProcessorConfig, TokenReductionConfig,
+};
+
+#[cfg(feature = "pdf")]
+pub use core::config::PdfConfig;
+
+pub use core::mime::{
+    DOCX_MIME_TYPE, EXCEL_MIME_TYPE, HTML_MIME_TYPE, JSON_MIME_TYPE, MARKDOWN_MIME_TYPE, PDF_MIME_TYPE,
+    PLAIN_TEXT_MIME_TYPE, POWER_POINT_MIME_TYPE, XML_MIME_TYPE, detect_mime_type, detect_mime_type_from_bytes,
+    detect_or_validate, get_extensions_for_mime, validate_mime_type,
+};
+
+pub use plugins::registry::{
+    get_document_extractor_registry, get_ocr_backend_registry, get_post_processor_registry, get_validator_registry,
+};
+
+#[cfg(feature = "embeddings")]
+pub use embeddings::{EMBEDDING_PRESETS, EmbeddingPreset, get_preset, list_presets};