kreuzberg 4.0.8 → 4.1.1

data/vendor/kreuzberg/src/core/mime.rs

@@ -57,6 +57,11 @@ static EXT_TO_MIME: Lazy<HashMap<&'static str, &'static str>> = Lazy::new(|| {
     m.insert("ods", OPENDOC_SPREADSHEET_MIME_TYPE);
 
     m.insert("pptx", POWER_POINT_MIME_TYPE);
+    m.insert(
+        "ppsx",
+        "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+    );
+    m.insert("pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12");
     m.insert("ppt", LEGACY_POWERPOINT_MIME_TYPE);
 
     m.insert("docx", DOCX_MIME_TYPE);
@@ -180,6 +185,8 @@ static SUPPORTED_MIME_TYPES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
 
     set.insert(PDF_MIME_TYPE);
     set.insert(POWER_POINT_MIME_TYPE);
+    set.insert("application/vnd.openxmlformats-officedocument.presentationml.slideshow"); // PPSX
+    set.insert("application/vnd.ms-powerpoint.presentation.macroEnabled.12"); // PPTM
     set.insert(LEGACY_WORD_MIME_TYPE);
     set.insert(LEGACY_POWERPOINT_MIME_TYPE);
     set.insert(HTML_MIME_TYPE);
@@ -459,6 +466,14 @@ mod tests {
             ("test.xlsx", EXCEL_MIME_TYPE),
             ("test.xls", EXCEL_BINARY_MIME_TYPE),
             ("test.pptx", POWER_POINT_MIME_TYPE),
+            (
+                "test.ppsx",
+                "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+            ),
+            (
+                "test.pptm",
+                "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+            ),
             ("test.ppt", LEGACY_POWERPOINT_MIME_TYPE),
             ("test.docx", DOCX_MIME_TYPE),
             ("test.doc", LEGACY_WORD_MIME_TYPE),

data/vendor/kreuzberg/src/core/mod.rs

@@ -41,8 +41,11 @@ pub mod mime;
 pub mod pipeline;
 pub mod server_config;
 
+#[cfg(feature = "pdf")]
+pub use config::HierarchyConfig;
 pub use config::{
-    ChunkingConfig, ExtractionConfig, ImageExtractionConfig, LanguageDetectionConfig, OcrConfig, TokenReductionConfig,
+    ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
+    LanguageDetectionConfig, OcrConfig, OutputFormat, PageConfig, PostProcessorConfig, TokenReductionConfig,
 };
 pub use config_validation::{
     validate_binarization_method, validate_chunking_params, validate_confidence, validate_dpi, validate_language_code,

data/vendor/kreuzberg/src/core/pipeline/cache.rs

@@ -0,0 +1,60 @@
+//! Processor caching to reduce lock contention.
+//!
+//! This module manages the caching of post-processors by processing stage,
+//! eliminating repeated registry lock acquisitions.
+
+use crate::Result;
+use crate::plugins::{PostProcessor, ProcessingStage};
+use once_cell::sync::Lazy;
+use std::sync::Arc;
+use std::sync::RwLock as StdRwLock;
+
+/// Cached post-processors for each stage to reduce lock contention.
+///
+/// This cache is populated once during the first pipeline run and reused
+/// for all subsequent extractions, eliminating 3 of 4 registry lock acquisitions
+/// per extraction.
+pub(super) struct ProcessorCache {
+    pub(super) early: Arc<Vec<Arc<dyn PostProcessor>>>,
+    pub(super) middle: Arc<Vec<Arc<dyn PostProcessor>>>,
+    pub(super) late: Arc<Vec<Arc<dyn PostProcessor>>>,
+}
+
+impl ProcessorCache {
+    /// Create a new processor cache by fetching from the registry.
+    pub(super) fn new() -> Result<Self> {
+        let processor_registry = crate::plugins::registry::get_post_processor_registry();
+        let registry = processor_registry
+            .read()
+            .map_err(|e| crate::KreuzbergError::Other(format!("Post-processor registry lock poisoned: {}", e)))?;
+
+        Ok(Self {
+            early: Arc::new(registry.get_for_stage(ProcessingStage::Early)),
+            middle: Arc::new(registry.get_for_stage(ProcessingStage::Middle)),
+            late: Arc::new(registry.get_for_stage(ProcessingStage::Late)),
+        })
+    }
+
+    /// Get processors for a specific stage from cache.
+    #[allow(dead_code)]
+    pub(super) fn get_for_stage(&self, stage: ProcessingStage) -> Arc<Vec<Arc<dyn PostProcessor>>> {
+        match stage {
+            ProcessingStage::Early => Arc::clone(&self.early),
+            ProcessingStage::Middle => Arc::clone(&self.middle),
+            ProcessingStage::Late => Arc::clone(&self.late),
+        }
+    }
+}
+
+/// Lazy processor cache - initialized on first use, then cached.
+pub(super) static PROCESSOR_CACHE: Lazy<StdRwLock<Option<ProcessorCache>>> = Lazy::new(|| StdRwLock::new(None));
+
+/// Clear the processor cache (primarily for testing when registry changes).
+#[allow(dead_code)]
+pub fn clear_processor_cache() -> Result<()> {
+    let mut cache = PROCESSOR_CACHE
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
+    *cache = None;
+    Ok(())
+}

data/vendor/kreuzberg/src/core/pipeline/execution.rs

@@ -0,0 +1,89 @@
+//! Core processor execution logic.
+//!
+//! This module handles the execution of post-processors and validators
+//! in the correct order.
+
+use crate::core::config::ExtractionConfig;
+use crate::plugins::ProcessingStage;
+use crate::types::ExtractionResult;
+use crate::{KreuzbergError, Result};
+
+/// Execute all registered post-processors by stage.
+pub(super) async fn execute_processors(
+    result: &mut ExtractionResult,
+    config: &ExtractionConfig,
+    pp_config: &Option<&crate::core::config::PostProcessorConfig>,
+    early_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    middle_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    late_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+) -> Result<()> {
+    for (_stage, processors_arc) in [
+        (ProcessingStage::Early, early_processors),
+        (ProcessingStage::Middle, middle_processors),
+        (ProcessingStage::Late, late_processors),
+    ] {
+        for processor in processors_arc.iter() {
+            let processor_name = processor.name();
+
+            let should_run = should_processor_run(pp_config, processor_name);
+
+            if should_run && processor.should_process(result, config) {
+                match processor.process(result, config).await {
+                    Ok(_) => {}
+                    Err(err @ KreuzbergError::Io(_))
+                    | Err(err @ KreuzbergError::LockPoisoned(_))
+                    | Err(err @ KreuzbergError::Plugin { .. }) => {
+                        return Err(err);
+                    }
+                    Err(err) => {
+                        result.metadata.additional.insert(
+                            format!("processing_error_{processor_name}"),
+                            serde_json::Value::String(err.to_string()),
+                        );
+                    }
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Determine if a processor should run based on configuration.
+fn should_processor_run(pp_config: &Option<&crate::core::config::PostProcessorConfig>, processor_name: &str) -> bool {
+    if let Some(config) = pp_config {
+        if let Some(ref enabled_set) = config.enabled_set {
+            enabled_set.contains(processor_name)
+        } else if let Some(ref disabled_set) = config.disabled_set {
+            !disabled_set.contains(processor_name)
+        } else if let Some(ref enabled) = config.enabled_processors {
+            enabled.iter().any(|name| name == processor_name)
+        } else if let Some(ref disabled) = config.disabled_processors {
+            !disabled.iter().any(|name| name == processor_name)
+        } else {
+            true
+        }
+    } else {
+        true
+    }
+}
+
+/// Execute all registered validators.
+pub(super) async fn execute_validators(result: &ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    let validator_registry = crate::plugins::registry::get_validator_registry();
+    let validators = {
+        let registry = validator_registry
+            .read()
+            .map_err(|e| crate::KreuzbergError::Other(format!("Validator registry lock poisoned: {}", e)))?;
+        registry.get_all()
+    };
+
+    if !validators.is_empty() {
+        for validator in validators {
+            if validator.should_validate(result, config) {
+                validator.validate(result, config).await?;
+            }
+        }
+    }
+
+    Ok(())
+}

data/vendor/kreuzberg/src/core/pipeline/features.rs

@@ -0,0 +1,108 @@
+//! Feature processing logic.
+//!
+//! This module handles feature-specific processing like chunking,
+//! embedding generation, and language detection.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::types::ExtractionResult;
+
+/// Execute chunking if configured.
+pub(super) fn execute_chunking(result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    #[cfg(feature = "chunking")]
+    if let Some(ref chunking_config) = config.chunking {
+        let chunk_config = crate::chunking::ChunkingConfig {
+            max_characters: chunking_config.max_chars,
+            overlap: chunking_config.max_overlap,
+            trim: true,
+            chunker_type: crate::chunking::ChunkerType::Text,
+        };
+
+        let page_boundaries = result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref());
+
+        match crate::chunking::chunk_text(&result.content, &chunk_config, page_boundaries) {
+            Ok(chunking_result) => {
+                result.chunks = Some(chunking_result.chunks);
+
+                if let Some(ref chunks) = result.chunks {
+                    result.metadata.additional.insert(
+                        "chunk_count".to_string(),
+                        serde_json::Value::Number(serde_json::Number::from(chunks.len())),
+                    );
+                }
+
+                #[cfg(feature = "embeddings")]
+                if let Some(ref embedding_config) = chunking_config.embedding
+                    && let Some(ref mut chunks) = result.chunks
+                {
+                    match crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config) {
+                        Ok(()) => {
+                            result
+                                .metadata
+                                .additional
+                                .insert("embeddings_generated".to_string(), serde_json::Value::Bool(true));
+                        }
+                        Err(e) => {
+                            result
+                                .metadata
+                                .additional
+                                .insert("embedding_error".to_string(), serde_json::Value::String(e.to_string()));
+                        }
+                    }
+                }
+
+                #[cfg(not(feature = "embeddings"))]
+                if chunking_config.embedding.is_some() {
+                    result.metadata.additional.insert(
+                        "embedding_error".to_string(),
+                        serde_json::Value::String("Embeddings feature not enabled".to_string()),
+                    );
+                }
+            }
+            Err(e) => {
+                result
+                    .metadata
+                    .additional
+                    .insert("chunking_error".to_string(), serde_json::Value::String(e.to_string()));
+            }
+        }
+    }
+
+    #[cfg(not(feature = "chunking"))]
+    if config.chunking.is_some() {
+        result.metadata.additional.insert(
+            "chunking_error".to_string(),
+            serde_json::Value::String("Chunking feature not enabled".to_string()),
+        );
+    }
+
+    Ok(())
+}
+
+/// Execute language detection if configured.
+pub(super) fn execute_language_detection(result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    #[cfg(feature = "language-detection")]
+    if let Some(ref lang_config) = config.language_detection {
+        match crate::language_detection::detect_languages(&result.content, lang_config) {
+            Ok(detected) => {
+                result.detected_languages = detected;
+            }
+            Err(e) => {
+                result.metadata.additional.insert(
+                    "language_detection_error".to_string(),
+                    serde_json::Value::String(e.to_string()),
+                );
+            }
+        }
+    }
+
+    #[cfg(not(feature = "language-detection"))]
+    if config.language_detection.is_some() {
+        result.metadata.additional.insert(
+            "language_detection_error".to_string(),
+            serde_json::Value::String("Language detection feature not enabled".to_string()),
+        );
+    }
+
+    Ok(())
+}