kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/pipeline.rs

@@ -1,1223 +0,0 @@
-//! Post-processing pipeline orchestration.
-//!
-//! This module orchestrates the post-processing pipeline, executing validators,
-//! quality processing, chunking, and custom hooks in the correct order.
-
-use crate::core::config::ExtractionConfig;
-use crate::plugins::{PostProcessor, ProcessingStage};
-use crate::types::ExtractionResult;
-use crate::{KreuzbergError, Result};
-use once_cell::sync::Lazy;
-use std::sync::Arc;
-use std::sync::RwLock as StdRwLock;
-
-/// Cached post-processors for each stage to reduce lock contention.
-///
-/// This cache is populated once during the first pipeline run and reused
-/// for all subsequent extractions, eliminating 3 of 4 registry lock acquisitions
-/// per extraction.
-struct ProcessorCache {
-    early: Arc<Vec<Arc<dyn PostProcessor>>>,
-    middle: Arc<Vec<Arc<dyn PostProcessor>>>,
-    late: Arc<Vec<Arc<dyn PostProcessor>>>,
-}
-
-impl ProcessorCache {
-    /// Create a new processor cache by fetching from the registry.
-    fn new() -> Result<Self> {
-        let processor_registry = crate::plugins::registry::get_post_processor_registry();
-        let registry = processor_registry
-            .read()
-            .map_err(|e| crate::KreuzbergError::Other(format!("Post-processor registry lock poisoned: {}", e)))?;
-
-        Ok(Self {
-            early: Arc::new(registry.get_for_stage(ProcessingStage::Early)),
-            middle: Arc::new(registry.get_for_stage(ProcessingStage::Middle)),
-            late: Arc::new(registry.get_for_stage(ProcessingStage::Late)),
-        })
-    }
-
-    /// Get processors for a specific stage from cache.
-    #[allow(dead_code)]
-    fn get_for_stage(&self, stage: ProcessingStage) -> Arc<Vec<Arc<dyn PostProcessor>>> {
-        match stage {
-            ProcessingStage::Early => Arc::clone(&self.early),
-            ProcessingStage::Middle => Arc::clone(&self.middle),
-            ProcessingStage::Late => Arc::clone(&self.late),
-        }
-    }
-}
-
-/// Lazy processor cache - initialized on first use, then cached.
-static PROCESSOR_CACHE: Lazy<StdRwLock<Option<ProcessorCache>>> = Lazy::new(|| StdRwLock::new(None));
-
-/// Clear the processor cache (primarily for testing when registry changes).
-#[allow(dead_code)]
-pub fn clear_processor_cache() -> Result<()> {
-    let mut cache = PROCESSOR_CACHE
-        .write()
-        .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
-    *cache = None;
-    Ok(())
-}
-
-/// Run the post-processing pipeline on an extraction result.
-///
-/// Executes post-processing in the following order:
-/// 1. Post-Processors - Execute by stage (Early, Middle, Late) to modify/enhance the result
-/// 2. Quality Processing - Text cleaning and quality scoring
-/// 3. Chunking - Text splitting if enabled
-/// 4. Validators - Run validation hooks on the processed result (can fail fast)
-///
-/// # Arguments
-///
-/// * `result` - The extraction result to process
-/// * `config` - Extraction configuration
-///
-/// # Returns
-///
-/// The processed extraction result.
-///
-/// # Errors
-///
-/// - Validator errors bubble up immediately
-/// - Post-processor errors are caught and recorded in metadata
-/// - System errors (IO, RuntimeError equivalents) always bubble up
-#[cfg_attr(feature = "otel", tracing::instrument(
-    skip(result, config),
-    fields(
-        pipeline.stage = "post_processing",
-        content.length = result.content.len(),
-    )
-))]
-pub async fn run_pipeline(mut result: ExtractionResult, config: &ExtractionConfig) -> Result<ExtractionResult> {
-    let pp_config = config.postprocessor.as_ref();
-    let postprocessing_enabled = pp_config.is_none_or(|c| c.enabled);
-
-    if postprocessing_enabled {
-        #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-        {
-            let _ = crate::keywords::ensure_initialized();
-        }
-
-        #[cfg(feature = "language-detection")]
-        {
-            let _ = crate::language_detection::ensure_initialized();
-        }
-
-        #[cfg(feature = "chunking")]
-        {
-            let _ = crate::chunking::ensure_initialized();
-        }
-
-        #[cfg(feature = "quality")]
-        {
-            let registry = crate::plugins::registry::get_post_processor_registry();
-            if let Ok(mut reg) = registry.write() {
-                let _ = reg.register(std::sync::Arc::new(crate::text::QualityProcessor), 30);
-            }
-        }
-
-        {
-            let mut cache_lock = PROCESSOR_CACHE
-                .write()
-                .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
-            if cache_lock.is_none() {
-                *cache_lock = Some(ProcessorCache::new()?);
-            }
-        }
-
-        let (early_processors, middle_processors, late_processors) = {
-            let cache_lock = PROCESSOR_CACHE
-                .read()
-                .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
-            let cache = cache_lock
-                .as_ref()
-                .ok_or_else(|| crate::KreuzbergError::Other("Processor cache not initialized".to_string()))?;
-            (
-                Arc::clone(&cache.early),
-                Arc::clone(&cache.middle),
-                Arc::clone(&cache.late),
-            )
-        };
-
-        for (_stage, processors_arc) in [
-            (ProcessingStage::Early, early_processors),
-            (ProcessingStage::Middle, middle_processors),
-            (ProcessingStage::Late, late_processors),
-        ] {
-            for processor in processors_arc.iter() {
-                let processor_name = processor.name();
-
-                let should_run = if let Some(config) = pp_config {
-                    if let Some(ref enabled_set) = config.enabled_set {
-                        enabled_set.contains(processor_name)
-                    } else if let Some(ref disabled_set) = config.disabled_set {
-                        !disabled_set.contains(processor_name)
-                    } else if let Some(ref enabled) = config.enabled_processors {
-                        enabled.iter().any(|name| name == processor_name)
-                    } else if let Some(ref disabled) = config.disabled_processors {
-                        !disabled.iter().any(|name| name == processor_name)
-                    } else {
-                        true
-                    }
-                } else {
-                    true
-                };
-
-                if should_run && processor.should_process(&result, config) {
-                    match processor.process(&mut result, config).await {
-                        Ok(_) => {}
-                        Err(err @ KreuzbergError::Io(_))
-                        | Err(err @ KreuzbergError::LockPoisoned(_))
-                        | Err(err @ KreuzbergError::Plugin { .. }) => {
-                            return Err(err);
-                        }
-                        Err(err) => {
-                            result.metadata.additional.insert(
-                                format!("processing_error_{processor_name}"),
-                                serde_json::Value::String(err.to_string()),
-                            );
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    #[cfg(feature = "chunking")]
-    if let Some(ref chunking_config) = config.chunking {
-        let chunk_config = crate::chunking::ChunkingConfig {
-            max_characters: chunking_config.max_chars,
-            overlap: chunking_config.max_overlap,
-            trim: true,
-            chunker_type: crate::chunking::ChunkerType::Text,
-        };
-
-        let page_boundaries = result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref());
-
-        match crate::chunking::chunk_text(&result.content, &chunk_config, page_boundaries) {
-            Ok(chunking_result) => {
-                result.chunks = Some(chunking_result.chunks);
-
-                if let Some(ref chunks) = result.chunks {
-                    result.metadata.additional.insert(
-                        "chunk_count".to_string(),
-                        serde_json::Value::Number(serde_json::Number::from(chunks.len())),
-                    );
-                }
-
-                #[cfg(feature = "embeddings")]
-                if let Some(ref embedding_config) = chunking_config.embedding
-                    && let Some(ref mut chunks) = result.chunks
-                {
-                    match crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config) {
-                        Ok(()) => {
-                            result
-                                .metadata
-                                .additional
-                                .insert("embeddings_generated".to_string(), serde_json::Value::Bool(true));
-                        }
-                        Err(e) => {
-                            result
-                                .metadata
-                                .additional
-                                .insert("embedding_error".to_string(), serde_json::Value::String(e.to_string()));
-                        }
-                    }
-                }
-
-                #[cfg(not(feature = "embeddings"))]
-                if chunking_config.embedding.is_some() {
-                    result.metadata.additional.insert(
-                        "embedding_error".to_string(),
-                        serde_json::Value::String("Embeddings feature not enabled".to_string()),
-                    );
-                }
-            }
-            Err(e) => {
-                result
-                    .metadata
-                    .additional
-                    .insert("chunking_error".to_string(), serde_json::Value::String(e.to_string()));
-            }
-        }
-    }
-
-    #[cfg(not(feature = "chunking"))]
-    if config.chunking.is_some() {
-        result.metadata.additional.insert(
-            "chunking_error".to_string(),
-            serde_json::Value::String("Chunking feature not enabled".to_string()),
-        );
-    }
-
-    #[cfg(feature = "language-detection")]
-    if let Some(ref lang_config) = config.language_detection {
-        match crate::language_detection::detect_languages(&result.content, lang_config) {
-            Ok(detected) => {
-                result.detected_languages = detected;
-            }
-            Err(e) => {
-                result.metadata.additional.insert(
-                    "language_detection_error".to_string(),
-                    serde_json::Value::String(e.to_string()),
-                );
-            }
-        }
-    }
-
-    #[cfg(not(feature = "language-detection"))]
-    if config.language_detection.is_some() {
-        result.metadata.additional.insert(
-            "language_detection_error".to_string(),
-            serde_json::Value::String("Language detection feature not enabled".to_string()),
-        );
-    }
-
-    {
-        let validator_registry = crate::plugins::registry::get_validator_registry();
-        let validators = {
-            let registry = validator_registry
-                .read()
-                .map_err(|e| crate::KreuzbergError::Other(format!("Validator registry lock poisoned: {}", e)))?;
-            registry.get_all()
-        };
-
-        if !validators.is_empty() {
-            for validator in validators {
-                if validator.should_validate(&result, config) {
-                    validator.validate(&result, config).await?;
-                }
-            }
-        }
-    }
-
-    Ok(result)
-}
-
-/// Run the post-processing pipeline synchronously (WASM-compatible version).
-///
-/// This is a synchronous implementation for WASM and non-async contexts.
-/// It performs a subset of the full async pipeline, excluding async post-processors
-/// and validators.
-///
-/// # Arguments
-///
-/// * `result` - The extraction result to process
-/// * `config` - Extraction configuration
-///
-/// # Returns
-///
-/// The processed extraction result.
-///
-/// # Notes
-///
-/// This function is only available when the `tokio-runtime` feature is disabled.
-/// It handles:
-/// - Quality processing (if enabled)
-/// - Chunking (if enabled)
-/// - Language detection (if enabled)
-///
-/// It does NOT handle:
-/// - Async post-processors
-/// - Async validators
-#[cfg(not(feature = "tokio-runtime"))]
-pub fn run_pipeline_sync(mut result: ExtractionResult, config: &ExtractionConfig) -> Result<ExtractionResult> {
-    #[cfg(feature = "chunking")]
-    if let Some(ref chunking_config) = config.chunking {
-        let chunk_config = crate::chunking::ChunkingConfig {
-            max_characters: chunking_config.max_chars,
-            overlap: chunking_config.max_overlap,
-            trim: true,
-            chunker_type: crate::chunking::ChunkerType::Text,
-        };
-
-        match crate::chunking::chunk_text(&result.content, &chunk_config, None) {
-            Ok(chunking_result) => {
-                result.chunks = Some(chunking_result.chunks);
-
-                if let Some(ref chunks) = result.chunks {
-                    result.metadata.additional.insert(
-                        "chunk_count".to_string(),
-                        serde_json::Value::Number(serde_json::Number::from(chunks.len())),
-                    );
-                }
-
-                #[cfg(feature = "embeddings")]
-                if let Some(ref embedding_config) = chunking_config.embedding
-                    && let Some(ref mut chunks) = result.chunks
-                {
-                    match crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config) {
-                        Ok(()) => {
-                            result
-                                .metadata
-                                .additional
-                                .insert("embeddings_generated".to_string(), serde_json::Value::Bool(true));
-                        }
-                        Err(e) => {
-                            result
-                                .metadata
-                                .additional
-                                .insert("embedding_error".to_string(), serde_json::Value::String(e.to_string()));
-                        }
-                    }
-                }
-
-                #[cfg(not(feature = "embeddings"))]
-                if chunking_config.embedding.is_some() {
-                    result.metadata.additional.insert(
-                        "embedding_error".to_string(),
-                        serde_json::Value::String("Embeddings feature not enabled".to_string()),
-                    );
-                }
-            }
-            Err(e) => {
-                result
-                    .metadata
-                    .additional
-                    .insert("chunking_error".to_string(), serde_json::Value::String(e.to_string()));
-            }
-        }
-    }
-
-    #[cfg(not(feature = "chunking"))]
-    if config.chunking.is_some() {
-        result.metadata.additional.insert(
-            "chunking_error".to_string(),
-            serde_json::Value::String("Chunking feature not enabled".to_string()),
-        );
-    }
-
-    #[cfg(feature = "language-detection")]
-    if let Some(ref lang_config) = config.language_detection {
-        match crate::language_detection::detect_languages(&result.content, lang_config) {
-            Ok(detected) => {
-                result.detected_languages = detected;
-            }
-            Err(e) => {
-                result.metadata.additional.insert(
-                    "language_detection_error".to_string(),
-                    serde_json::Value::String(e.to_string()),
-                );
-            }
-        }
-    }
-
-    #[cfg(not(feature = "language-detection"))]
-    if config.language_detection.is_some() {
-        result.metadata.additional.insert(
-            "language_detection_error".to_string(),
-            serde_json::Value::String("Language detection feature not enabled".to_string()),
-        );
-    }
-
-    Ok(result)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::types::Metadata;
-    use lazy_static::lazy_static;
-
-    const VALIDATION_MARKER_KEY: &str = "registry_validation_marker";
-    #[cfg(feature = "quality")]
-    const QUALITY_VALIDATION_MARKER: &str = "quality_validation_test";
-    const POSTPROCESSOR_VALIDATION_MARKER: &str = "postprocessor_validation_test";
-    const ORDER_VALIDATION_MARKER: &str = "order_validation_test";
-
-    lazy_static! {
-        static ref REGISTRY_TEST_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(());
-    }
-
-    #[tokio::test]
-    async fn test_run_pipeline_basic() {
-        let mut result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        result.metadata.additional.insert(
-            VALIDATION_MARKER_KEY.to_string(),
-            serde_json::json!(ORDER_VALIDATION_MARKER),
-        );
-        let config = ExtractionConfig::default();
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert_eq!(processed.content, "test");
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "quality")]
-    async fn test_pipeline_with_quality_processing() {
-        let result = ExtractionResult {
-            content: "This is a test document with some meaningful content.".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig {
-            enable_quality_processing: true,
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert!(processed.metadata.additional.contains_key("quality_score"));
-    }
-
-    #[tokio::test]
-    async fn test_pipeline_without_quality_processing() {
-        let result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig {
-            enable_quality_processing: false,
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert!(!processed.metadata.additional.contains_key("quality_score"));
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "chunking")]
-    async fn test_pipeline_with_chunking() {
-        let result = ExtractionResult {
-            content: "This is a long text that should be chunked. ".repeat(100),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig {
-            chunking: Some(crate::ChunkingConfig {
-                max_chars: 500,
-                max_overlap: 50,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert!(processed.metadata.additional.contains_key("chunk_count"));
-        let chunk_count = processed.metadata.additional.get("chunk_count").unwrap();
-        assert!(chunk_count.as_u64().unwrap() > 1);
-    }
-
-    #[tokio::test]
-    async fn test_pipeline_without_chunking() {
-        let result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig {
-            chunking: None,
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert!(!processed.metadata.additional.contains_key("chunk_count"));
-    }
-
-    #[tokio::test]
-    async fn test_pipeline_preserves_metadata() {
-        use std::collections::HashMap;
-        let mut additional = HashMap::new();
-        additional.insert("source".to_string(), serde_json::json!("test"));
-        additional.insert("page".to_string(), serde_json::json!(1));
-
-        let result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata {
-                additional,
-                ..Default::default()
-            },
-            pages: None,
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-        };
-        let config = ExtractionConfig::default();
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert_eq!(
-            processed.metadata.additional.get("source").unwrap(),
-            &serde_json::json!("test")
-        );
-        assert_eq!(
-            processed.metadata.additional.get("page").unwrap(),
-            &serde_json::json!(1)
-        );
-    }
-
-    #[tokio::test]
-    async fn test_pipeline_preserves_tables() {
-        use crate::types::Table;
-
-        let table = Table {
-            cells: vec![vec!["A".to_string(), "B".to_string()]],
-            markdown: "| A | B |".to_string(),
-            page_number: 0,
-        };
-
-        let result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![table],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig::default();
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert_eq!(processed.tables.len(), 1);
-        assert_eq!(processed.tables[0].cells.len(), 1);
-    }
-
-    #[tokio::test]
-    async fn test_pipeline_empty_content() {
-        let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-
-        {
-            let registry = crate::plugins::registry::get_post_processor_registry();
-            registry.write().unwrap().shutdown_all().unwrap();
-        }
-        {
-            let registry = crate::plugins::registry::get_validator_registry();
-            registry.write().unwrap().shutdown_all().unwrap();
-        }
-
-        let result = ExtractionResult {
-            content: String::new(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig::default();
-
-        drop(_guard);
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert_eq!(processed.content, "");
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "chunking")]
-    async fn test_pipeline_with_all_features() {
-        let result = ExtractionResult {
-            content: "This is a comprehensive test document. ".repeat(50),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        let config = ExtractionConfig {
-            enable_quality_processing: true,
-            chunking: Some(crate::ChunkingConfig {
-                max_chars: 500,
-                max_overlap: 50,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-        assert!(processed.metadata.additional.contains_key("quality_score"));
-        assert!(processed.metadata.additional.contains_key("chunk_count"));
-    }
-
-    #[tokio::test]
-    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-    async fn test_pipeline_with_keyword_extraction() {
-        {
-            let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-            crate::plugins::registry::get_validator_registry()
-                .write()
-                .unwrap()
-                .shutdown_all()
-                .unwrap();
-            crate::plugins::registry::get_post_processor_registry()
-                .write()
-                .unwrap()
-                .shutdown_all()
-                .unwrap();
-
-            let _ = crate::keywords::register_keyword_processor();
-        }
-
-        let result = ExtractionResult {
-            content: r#"
-Machine learning is a branch of artificial intelligence that focuses on
-building systems that can learn from data. Deep learning is a subset of
-machine learning that uses neural networks with multiple layers.
-Natural language processing enables computers to understand human language.
-            "#
-            .to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        #[cfg(feature = "keywords-yake")]
-        let keyword_config = crate::keywords::KeywordConfig::yake();
-
-        #[cfg(all(feature = "keywords-rake", not(feature = "keywords-yake")))]
-        let keyword_config = crate::keywords::KeywordConfig::rake();
-
-        let config = ExtractionConfig {
-            keywords: Some(keyword_config),
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-
-        assert!(processed.metadata.additional.contains_key("keywords"));
-
-        let keywords_value = processed.metadata.additional.get("keywords").unwrap();
-        assert!(keywords_value.is_array());
-
-        let keywords = keywords_value.as_array().unwrap();
-        assert!(!keywords.is_empty(), "Should have extracted keywords");
-
-        let first_keyword = &keywords[0];
-        assert!(first_keyword.is_object());
-        assert!(first_keyword.get("text").is_some());
-        assert!(first_keyword.get("score").is_some());
-        assert!(first_keyword.get("algorithm").is_some());
-    }
-
-    #[tokio::test]
-    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-    async fn test_pipeline_without_keyword_config() {
-        {
-            let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-        }
-        let result = ExtractionResult {
-            content: "Machine learning and artificial intelligence.".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let config = ExtractionConfig {
-            keywords: None,
-            ..Default::default()
-        };
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-
-        assert!(!processed.metadata.additional.contains_key("keywords"));
-    }
-
-    #[tokio::test]
-    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-    async fn test_pipeline_keyword_extraction_short_content() {
-        let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-        crate::plugins::registry::get_validator_registry()
-            .write()
-            .unwrap()
-            .shutdown_all()
-            .unwrap();
-        crate::plugins::registry::get_post_processor_registry()
-            .write()
-            .unwrap()
-            .shutdown_all()
-            .unwrap();
-
-        let result = ExtractionResult {
-            content: "Short text".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        #[cfg(feature = "keywords-yake")]
-        let keyword_config = crate::keywords::KeywordConfig::yake();
-
-        #[cfg(all(feature = "keywords-rake", not(feature = "keywords-yake")))]
-        let keyword_config = crate::keywords::KeywordConfig::rake();
-
-        let config = ExtractionConfig {
-            keywords: Some(keyword_config),
-            ..Default::default()
-        };
-
-        drop(_guard);
-
-        let processed = run_pipeline(result, &config).await.unwrap();
-
-        assert!(!processed.metadata.additional.contains_key("keywords"));
-    }
-
-    #[tokio::test]
-    async fn test_postprocessor_runs_before_validator() {
-        use crate::plugins::{Plugin, PostProcessor, ProcessingStage, Validator};
-        use async_trait::async_trait;
-        use std::sync::Arc;
-
-        struct TestPostProcessor;
-        impl Plugin for TestPostProcessor {
-            fn name(&self) -> &str {
-                "test-processor"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl PostProcessor for TestPostProcessor {
-            async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                result
-                    .metadata
-                    .additional
-                    .insert("processed".to_string(), serde_json::json!(true));
-                Ok(())
-            }
-
-            fn processing_stage(&self) -> ProcessingStage {
-                ProcessingStage::Middle
-            }
-        }
-
-        struct TestValidator;
-        impl Plugin for TestValidator {
-            fn name(&self) -> &str {
-                "test-validator"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl Validator for TestValidator {
-            async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                let should_validate = result
-                    .metadata
-                    .additional
-                    .get(VALIDATION_MARKER_KEY)
-                    .and_then(|v| v.as_str())
-                    == Some(POSTPROCESSOR_VALIDATION_MARKER);
-
-                if !should_validate {
-                    return Ok(());
-                }
-
-                let processed = result
-                    .metadata
-                    .additional
-                    .get("processed")
-                    .and_then(|v| v.as_bool())
-                    .unwrap_or(false);
-
-                if !processed {
-                    return Err(crate::KreuzbergError::Validation {
-                        message: "Post-processor did not run before validator".to_string(),
-                        source: None,
-                    });
-                }
-                Ok(())
-            }
-        }
-
-        let pp_registry = crate::plugins::registry::get_post_processor_registry();
-        let val_registry = crate::plugins::registry::get_validator_registry();
-
-        let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-        clear_processor_cache().unwrap();
-        pp_registry.write().unwrap().shutdown_all().unwrap();
-        val_registry.write().unwrap().shutdown_all().unwrap();
-        clear_processor_cache().unwrap();
-
-        {
-            let mut registry = pp_registry.write().unwrap();
-            registry.register(Arc::new(TestPostProcessor), 0).unwrap();
-        }
-
-        {
-            let mut registry = val_registry.write().unwrap();
-            registry.register(Arc::new(TestValidator)).unwrap();
-        }
-
-        // Clear the cache after registering new processors so it rebuilds with the test processors
-        clear_processor_cache().unwrap();
-
-        let mut result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        result.metadata.additional.insert(
-            VALIDATION_MARKER_KEY.to_string(),
-            serde_json::json!(POSTPROCESSOR_VALIDATION_MARKER),
-        );
-
-        let config = ExtractionConfig {
-            postprocessor: Some(crate::core::config::PostProcessorConfig {
-                enabled: true,
-                enabled_set: None,
-                disabled_set: None,
-                enabled_processors: None,
-                disabled_processors: None,
-            }),
-            ..Default::default()
-        };
-        drop(_guard);
-
-        let processed = run_pipeline(result, &config).await;
-
-        pp_registry.write().unwrap().shutdown_all().unwrap();
-        val_registry.write().unwrap().shutdown_all().unwrap();
-
-        assert!(processed.is_ok(), "Validator should have seen post-processor metadata");
-        let processed = processed.unwrap();
-        assert_eq!(
-            processed.metadata.additional.get("processed"),
-            Some(&serde_json::json!(true)),
-            "Post-processor metadata should be present"
-        );
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "quality")]
-    async fn test_quality_processing_runs_before_validator() {
-        let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-        use crate::plugins::{Plugin, Validator};
-        use async_trait::async_trait;
-        use std::sync::Arc;
-
-        struct QualityValidator;
-        impl Plugin for QualityValidator {
-            fn name(&self) -> &str {
-                "quality-validator"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl Validator for QualityValidator {
-            async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                let should_validate = result
-                    .metadata
-                    .additional
-                    .get(VALIDATION_MARKER_KEY)
-                    .and_then(|v| v.as_str())
-                    == Some(QUALITY_VALIDATION_MARKER);
-
-                if !should_validate {
-                    return Ok(());
-                }
-
-                if !result.metadata.additional.contains_key("quality_score") {
-                    return Err(crate::KreuzbergError::Validation {
-                        message: "Quality processing did not run before validator".to_string(),
-                        source: None,
-                    });
-                }
-                Ok(())
-            }
-        }
-
-        let val_registry = crate::plugins::registry::get_validator_registry();
-        {
-            let mut registry = val_registry.write().unwrap();
-            registry.register(Arc::new(QualityValidator)).unwrap();
-        }
-
-        let mut result = ExtractionResult {
-            content: "This is meaningful test content for quality scoring.".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-        result.metadata.additional.insert(
-            VALIDATION_MARKER_KEY.to_string(),
-            serde_json::json!(QUALITY_VALIDATION_MARKER),
-        );
-
-        let config = ExtractionConfig {
-            enable_quality_processing: true,
-            ..Default::default()
-        };
-
-        drop(_guard);
-
-        let processed = run_pipeline(result, &config).await;
-
-        {
-            let mut registry = val_registry.write().unwrap();
-            registry.remove("quality-validator").unwrap();
-        }
-
-        assert!(processed.is_ok(), "Validator should have seen quality_score");
-    }
-
-    #[tokio::test]
-    async fn test_multiple_postprocessors_run_before_validator() {
-        use crate::plugins::{Plugin, PostProcessor, ProcessingStage, Validator};
-        use async_trait::async_trait;
-        use std::sync::Arc;
-
-        struct EarlyProcessor;
-        impl Plugin for EarlyProcessor {
-            fn name(&self) -> &str {
-                "early-proc"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl PostProcessor for EarlyProcessor {
-            async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                let mut order = result
-                    .metadata
-                    .additional
-                    .get("execution_order")
-                    .and_then(|v| v.as_array())
-                    .cloned()
-                    .unwrap_or_default();
-                order.push(serde_json::json!("early"));
-                result
-                    .metadata
-                    .additional
-                    .insert("execution_order".to_string(), serde_json::json!(order));
-                Ok(())
-            }
-
-            fn processing_stage(&self) -> ProcessingStage {
-                ProcessingStage::Early
-            }
-        }
-
-        struct LateProcessor;
-        impl Plugin for LateProcessor {
-            fn name(&self) -> &str {
-                "late-proc"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl PostProcessor for LateProcessor {
-            async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                let mut order = result
-                    .metadata
-                    .additional
-                    .get("execution_order")
-                    .and_then(|v| v.as_array())
-                    .cloned()
-                    .unwrap_or_default();
-                order.push(serde_json::json!("late"));
-                result
-                    .metadata
-                    .additional
-                    .insert("execution_order".to_string(), serde_json::json!(order));
-                Ok(())
-            }
-
-            fn processing_stage(&self) -> ProcessingStage {
-                ProcessingStage::Late
-            }
-        }
-
-        struct OrderValidator;
-        impl Plugin for OrderValidator {
-            fn name(&self) -> &str {
-                "order-validator"
-            }
-            fn version(&self) -> String {
-                "1.0.0".to_string()
-            }
-            fn initialize(&self) -> Result<()> {
-                Ok(())
-            }
-            fn shutdown(&self) -> Result<()> {
-                Ok(())
-            }
-        }
-
-        #[async_trait]
-        impl Validator for OrderValidator {
-            async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
-                let should_validate = result
-                    .metadata
-                    .additional
-                    .get(VALIDATION_MARKER_KEY)
-                    .and_then(|v| v.as_str())
-                    == Some(ORDER_VALIDATION_MARKER);
-
-                if !should_validate {
-                    return Ok(());
-                }
-
-                let order = result
-                    .metadata
-                    .additional
-                    .get("execution_order")
-                    .and_then(|v| v.as_array())
-                    .ok_or_else(|| crate::KreuzbergError::Validation {
-                        message: "No execution order found".to_string(),
-                        source: None,
-                    })?;
-
-                if order.len() != 2 {
-                    return Err(crate::KreuzbergError::Validation {
-                        message: format!("Expected 2 processors to run, got {}", order.len()),
-                        source: None,
-                    });
-                }
-
-                if order[0] != "early" || order[1] != "late" {
-                    return Err(crate::KreuzbergError::Validation {
-                        message: format!("Wrong execution order: {:?}", order),
-                        source: None,
-                    });
-                }
-
-                Ok(())
-            }
-        }
-
-        let pp_registry = crate::plugins::registry::get_post_processor_registry();
-        let val_registry = crate::plugins::registry::get_validator_registry();
-        let _guard = REGISTRY_TEST_GUARD.lock().unwrap();
-
-        pp_registry.write().unwrap().shutdown_all().unwrap();
-        val_registry.write().unwrap().shutdown_all().unwrap();
-        clear_processor_cache().unwrap();
-
-        {
-            let mut registry = pp_registry.write().unwrap();
-            registry.register(Arc::new(EarlyProcessor), 0).unwrap();
-            registry.register(Arc::new(LateProcessor), 0).unwrap();
-        }
-
-        {
-            let mut registry = val_registry.write().unwrap();
-            registry.register(Arc::new(OrderValidator)).unwrap();
-        }
-
-        // Clear the cache after registering new processors so it rebuilds with the test processors
-        clear_processor_cache().unwrap();
-
-        let result = ExtractionResult {
-            content: "test".to_string(),
-            mime_type: "text/plain".to_string(),
-            metadata: Metadata::default(),
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-            pages: None,
-        };
-
-        let config = ExtractionConfig::default();
-        drop(_guard);
-
-        let processed = run_pipeline(result, &config).await;
-
-        pp_registry.write().unwrap().shutdown_all().unwrap();
-        val_registry.write().unwrap().shutdown_all().unwrap();
-        clear_processor_cache().unwrap();
-
-        assert!(processed.is_ok(), "All processors should run before validator");
-    }
-}