kreuzberg 4.0.8 → 4.1.1

data/vendor/kreuzberg/src/core/pipeline/format.rs

@@ -0,0 +1,392 @@
+//! Output format conversion for extraction results.
+//!
+//! This module handles conversion of extraction results to various output formats
+//! (Plain, Djot, Markdown, HTML) with proper error handling and metadata recording.
+
+use crate::core::config::OutputFormat;
+use crate::types::ExtractionResult;
+
+/// Apply output format conversion to the extraction result.
+///
+/// This function converts the result's content field based on the configured output format:
+/// - `Plain`: No conversion (default)
+/// - `Djot`: Use djot_content if available, otherwise keep plain text
+/// - `Markdown`: Convert to Markdown format (uses djot as it's similar)
+/// - `Html`: Convert to HTML format
+///
+/// Skips conversion if content was already formatted during extraction (e.g., HTML extractor
+/// already produced djot or markdown output).
+///
+/// # Arguments
+///
+/// * `result` - The extraction result to modify
+/// * `output_format` - The desired output format
+pub fn apply_output_format(result: &mut ExtractionResult, output_format: OutputFormat) {
+    // Check if content was already formatted during extraction
+    let already_formatted = match result.mime_type.as_str() {
+        "text/markdown" if output_format == OutputFormat::Markdown => true,
+        "text/djot" if output_format == OutputFormat::Djot => true,
+        _ => false,
+    };
+
+    if already_formatted {
+        return; // Skip re-conversion
+    }
+
+    match output_format {
+        OutputFormat::Plain => {
+            // Default - no conversion needed
+        }
+        OutputFormat::Djot => {
+            // Convert the extraction result to djot markup
+            match crate::extractors::djot_format::extraction_result_to_djot(result) {
+                Ok(djot_markup) => {
+                    result.content = djot_markup;
+                }
+                Err(e) => {
+                    // Keep original content on error, record error in metadata
+                    result.metadata.additional.insert(
+                        "output_format_error".to_string(),
+                        serde_json::Value::String(format!("Failed to convert to djot: {}", e)),
+                    );
+                }
+            }
+        }
+        OutputFormat::Markdown => {
+            // Djot is syntactically similar to Markdown, so we use djot output as a
+            // reasonable approximation. Full Markdown conversion would require a
+            // dedicated converter that handles the syntactic differences (e.g.,
+            // emphasis markers are swapped: djot uses _ for emphasis and * for strong,
+            // while CommonMark uses * for emphasis and ** for strong).
+            if result.djot_content.is_some() {
+                match crate::extractors::djot_format::extraction_result_to_djot(result) {
+                    Ok(djot_markup) => {
+                        result.content = djot_markup;
+                    }
+                    Err(e) => {
+                        // Keep original content on error, record error in metadata
+                        result.metadata.additional.insert(
+                            "output_format_error".to_string(),
+                            serde_json::Value::String(format!("Failed to convert to markdown: {}", e)),
+                        );
+                    }
+                }
+            }
+            // For non-djot documents, content remains as-is
+        }
+        OutputFormat::Html => {
+            // Convert to HTML format
+            if result.djot_content.is_some() {
+                // First generate djot markup, then convert to HTML
+                match crate::extractors::djot_format::extraction_result_to_djot(result) {
+                    Ok(djot_markup) => {
+                        match crate::extractors::djot_format::djot_to_html(&djot_markup) {
+                            Ok(html) => {
+                                result.content = html;
+                            }
+                            Err(e) => {
+                                // Keep original content on error, record error in metadata
+                                result.metadata.additional.insert(
+                                    "output_format_error".to_string(),
+                                    serde_json::Value::String(format!("Failed to convert djot to HTML: {}", e)),
+                                );
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        // Keep original content on error, record error in metadata
+                        result.metadata.additional.insert(
+                            "output_format_error".to_string(),
+                            serde_json::Value::String(format!("Failed to generate djot for HTML conversion: {}", e)),
+                        );
+                    }
+                }
+            } else {
+                // For non-djot documents, wrap plain text in basic HTML
+                let escaped_content = html_escape(&result.content);
+                result.content = format!("<pre>{}</pre>", escaped_content);
+            }
+        }
+    }
+}
+
+/// Escape HTML special characters in a string.
+fn html_escape(s: &str) -> String {
+    s.replace('&', "&amp;")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+        .replace('\'', "&#39;")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::Metadata;
+
+    #[test]
+    fn test_apply_output_format_plain() {
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Plain);
+
+        // Plain format should not modify content
+        assert_eq!(result.content, "Hello World");
+    }
+
+    #[test]
+    fn test_apply_output_format_djot_with_djot_content() {
+        use crate::types::{BlockType, DjotContent, FormattedBlock, InlineElement, InlineType};
+
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/djot".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            elements: None,
+            djot_content: Some(DjotContent {
+                plain_text: "Hello World".to_string(),
+                blocks: vec![FormattedBlock {
+                    block_type: BlockType::Heading,
+                    level: Some(1),
+                    inline_content: vec![InlineElement {
+                        element_type: InlineType::Text,
+                        content: "Hello World".to_string(),
+                        attributes: None,
+                        metadata: None,
+                    }],
+                    attributes: None,
+                    language: None,
+                    code: None,
+                    children: vec![],
+                }],
+                metadata: Metadata::default(),
+                tables: vec![],
+                images: vec![],
+                links: vec![],
+                footnotes: vec![],
+                attributes: std::collections::HashMap::new(),
+            }),
+        };
+
+        apply_output_format(&mut result, OutputFormat::Djot);
+
+        // The content should still be present (the function preserves content)
+        assert!(!result.content.is_empty());
+    }
+
+    #[test]
+    fn test_apply_output_format_djot_without_djot_content() {
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Djot);
+
+        // Without djot_content, content is converted to djot paragraphs
+        // extraction_result_to_djot creates paragraphs from plain text
+        assert!(result.content.contains("Hello World"));
+    }
+
+    #[test]
+    fn test_apply_output_format_html() {
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Html);
+
+        // For non-djot documents, HTML wraps content in <pre> tags
+        assert!(result.content.contains("<pre>"));
+        assert!(result.content.contains("Hello World"));
+        assert!(result.content.contains("</pre>"));
+    }
+
+    #[test]
+    fn test_apply_output_format_html_escapes_special_chars() {
+        let mut result = ExtractionResult {
+            content: "<script>alert('XSS')</script>".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Html);
+
+        // HTML special characters should be escaped
+        assert!(result.content.contains("&lt;"));
+        assert!(result.content.contains("&gt;"));
+        assert!(!result.content.contains("<script>"));
+    }
+
+    #[test]
+    fn test_apply_output_format_markdown() {
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Markdown);
+
+        // For non-djot documents without djot_content, markdown keeps content as-is
+        assert_eq!(result.content, "Hello World");
+    }
+
+    #[test]
+    fn test_apply_output_format_preserves_metadata() {
+        let mut additional = std::collections::HashMap::new();
+        additional.insert("custom_key".to_string(), serde_json::json!("custom_value"));
+        let metadata = Metadata {
+            title: Some("Test Title".to_string()),
+            additional,
+            ..Default::default()
+        };
+
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata,
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Djot);
+
+        // Metadata should be preserved
+        assert_eq!(result.metadata.title, Some("Test Title".to_string()));
+        assert_eq!(
+            result.metadata.additional.get("custom_key"),
+            Some(&serde_json::json!("custom_value"))
+        );
+    }
+
+    #[test]
+    fn test_apply_output_format_preserves_tables() {
+        use crate::types::Table;
+
+        let table = Table {
+            cells: vec![vec!["A".to_string(), "B".to_string()]],
+            markdown: "| A | B |".to_string(),
+            page_number: 1,
+        };
+
+        let mut result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: "text/plain".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![table],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            djot_content: None,
+            elements: None,
+        };
+
+        apply_output_format(&mut result, OutputFormat::Html);
+
+        // Tables should be preserved
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0].cells[0][0], "A");
+    }
+
+    #[test]
+    fn test_apply_output_format_preserves_djot_content() {
+        use crate::types::{BlockType, DjotContent, FormattedBlock, InlineElement, InlineType};
+
+        let djot_content = DjotContent {
+            plain_text: "test".to_string(),
+            blocks: vec![FormattedBlock {
+                block_type: BlockType::Paragraph,
+                level: None,
+                inline_content: vec![InlineElement {
+                    element_type: InlineType::Text,
+                    content: "test".to_string(),
+                    attributes: None,
+                    metadata: None,
+                }],
+                attributes: None,
+                language: None,
+                code: None,
+                children: vec![],
+            }],
+            metadata: Metadata::default(),
+            tables: vec![],
+            images: vec![],
+            links: vec![],
+            footnotes: vec![],
+            attributes: std::collections::HashMap::new(),
+        };
+
+        let mut result = ExtractionResult {
+            content: "test".to_string(),
+            mime_type: "text/djot".to_string(),
+            metadata: Metadata::default(),
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+            pages: None,
+            elements: None,
+            djot_content: Some(djot_content),
+        };
+
+        apply_output_format(&mut result, OutputFormat::Djot);
+
+        // djot_content should still be present after format application
+        assert!(result.djot_content.is_some());
+        assert_eq!(result.djot_content.as_ref().unwrap().blocks.len(), 1);
+    }
+}

data/vendor/kreuzberg/src/core/pipeline/initialization.rs

@@ -0,0 +1,67 @@
+//! Pipeline initialization and setup logic.
+//!
+//! This module handles the initialization of features and processor cache
+//! required for pipeline execution.
+
+use crate::Result;
+
+use super::cache::{PROCESSOR_CACHE, ProcessorCache};
+
+/// Type alias for processor stages tuple (Early, Middle, Late).
+type ProcessorStages = (
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+);
+
+/// Initialize feature-specific systems that may be needed during pipeline execution.
+pub(super) fn initialize_features() {
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    {
+        let _ = crate::keywords::ensure_initialized();
+    }
+
+    #[cfg(feature = "language-detection")]
+    {
+        let _ = crate::language_detection::ensure_initialized();
+    }
+
+    #[cfg(feature = "chunking")]
+    {
+        let _ = crate::chunking::ensure_initialized();
+    }
+
+    #[cfg(feature = "quality")]
+    {
+        let registry = crate::plugins::registry::get_post_processor_registry();
+        if let Ok(mut reg) = registry.write() {
+            let _ = reg.register(std::sync::Arc::new(crate::text::QualityProcessor), 30);
+        }
+    }
+}
+
+/// Initialize the processor cache if not already initialized.
+pub(super) fn initialize_processor_cache() -> Result<()> {
+    let mut cache_lock = PROCESSOR_CACHE
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
+    if cache_lock.is_none() {
+        *cache_lock = Some(ProcessorCache::new()?);
+    }
+    Ok(())
+}
+
+/// Get processors from the cache, organized by stage.
+pub(super) fn get_processors_from_cache() -> Result<ProcessorStages> {
+    let cache_lock = PROCESSOR_CACHE
+        .read()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Processor cache lock poisoned: {}", e)))?;
+    let cache = cache_lock
+        .as_ref()
+        .ok_or_else(|| crate::KreuzbergError::Other("Processor cache not initialized".to_string()))?;
+    Ok((
+        std::sync::Arc::clone(&cache.early),
+        std::sync::Arc::clone(&cache.middle),
+        std::sync::Arc::clone(&cache.late),
+    ))
+}

data/vendor/kreuzberg/src/core/pipeline/mod.rs

@@ -0,0 +1,135 @@
+//! Post-processing pipeline orchestration.
+//!
+//! This module orchestrates the post-processing pipeline, executing validators,
+//! quality processing, chunking, and custom hooks in the correct order.
+
+mod cache;
+mod execution;
+mod features;
+mod format;
+mod initialization;
+
+#[cfg(test)]
+mod tests;
+
+pub use cache::clear_processor_cache;
+pub use format::apply_output_format;
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::types::ExtractionResult;
+
+use execution::{execute_processors, execute_validators};
+use features::{execute_chunking, execute_language_detection};
+use initialization::{get_processors_from_cache, initialize_features, initialize_processor_cache};
+
+/// Run the post-processing pipeline on an extraction result.
+///
+/// Executes post-processing in the following order:
+/// 1. Post-Processors - Execute by stage (Early, Middle, Late) to modify/enhance the result
+/// 2. Quality Processing - Text cleaning and quality scoring
+/// 3. Chunking - Text splitting if enabled
+/// 4. Validators - Run validation hooks on the processed result (can fail fast)
+///
+/// # Arguments
+///
+/// * `result` - The extraction result to process
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// The processed extraction result.
+///
+/// # Errors
+///
+/// - Validator errors bubble up immediately
+/// - Post-processor errors are caught and recorded in metadata
+/// - System errors (IO, RuntimeError equivalents) always bubble up
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(result, config),
+    fields(
+        pipeline.stage = "post_processing",
+        content.length = result.content.len(),
+    )
+))]
+pub async fn run_pipeline(mut result: ExtractionResult, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    let pp_config = config.postprocessor.as_ref();
+    let postprocessing_enabled = pp_config.is_none_or(|c| c.enabled);
+
+    if postprocessing_enabled {
+        initialize_features();
+        initialize_processor_cache()?;
+
+        let (early_processors, middle_processors, late_processors) = get_processors_from_cache()?;
+
+        execute_processors(
+            &mut result,
+            config,
+            &pp_config,
+            early_processors,
+            middle_processors,
+            late_processors,
+        )
+        .await?;
+    }
+
+    execute_chunking(&mut result, config)?;
+    execute_language_detection(&mut result, config)?;
+    execute_validators(&result, config).await?;
+
+    // Transform to element-based output if requested
+    if config.result_format == crate::types::OutputFormat::ElementBased {
+        result.elements = Some(crate::extraction::transform::transform_extraction_result_to_elements(
+            &result,
+        ));
+    }
+
+    // Apply output format conversion as the final step
+    apply_output_format(&mut result, config.output_format);
+
+    Ok(result)
+}
+
+/// Run the post-processing pipeline synchronously (WASM-compatible version).
+///
+/// This is a synchronous implementation for WASM and non-async contexts.
+/// It performs a subset of the full async pipeline, excluding async post-processors
+/// and validators.
+///
+/// # Arguments
+///
+/// * `result` - The extraction result to process
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// The processed extraction result.
+///
+/// # Notes
+///
+/// This function is only available when the `tokio-runtime` feature is disabled.
+/// It handles:
+/// - Quality processing (if enabled)
+/// - Chunking (if enabled)
+/// - Language detection (if enabled)
+///
+/// It does NOT handle:
+/// - Async post-processors
+/// - Async validators
+#[cfg(not(feature = "tokio-runtime"))]
+pub fn run_pipeline_sync(mut result: ExtractionResult, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    execute_chunking(&mut result, config)?;
+    execute_language_detection(&mut result, config)?;
+
+    // Transform to element-based output if requested
+    if config.result_format == crate::types::OutputFormat::ElementBased {
+        result.elements = Some(crate::extraction::transform::transform_extraction_result_to_elements(
+            &result,
+        ));
+    }
+
+    // Apply output format conversion as the final step
+    apply_output_format(&mut result, config.output_format);
+
+    Ok(result)
+}