kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/config.rs

@@ -1,1914 +0,0 @@
-//! Configuration loading and management.
-//!
-//! This module provides utilities for loading extraction configuration from various
-//! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
-
-use crate::{KreuzbergError, Result};
-use dashmap::DashMap;
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, LazyLock};
-use std::time::SystemTime;
-
-static CONFIG_CACHE: LazyLock<DashMap<PathBuf, (SystemTime, Arc<ExtractionConfig>)>> = LazyLock::new(DashMap::new);
-
-/// Page extraction and tracking configuration.
-///
-/// Controls how pages are extracted, tracked, and represented in the extraction results.
-/// When `None`, page tracking is disabled.
-///
-/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
-/// when page boundaries are available and chunking is configured.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(default)]
-pub struct PageConfig {
-    /// Extract pages as separate array (ExtractionResult.pages)
-    #[serde(default)]
-    pub extract_pages: bool,
-
-    /// Insert page markers in main content string
-    #[serde(default)]
-    pub insert_page_markers: bool,
-
-    /// Page marker format (use {page_num} placeholder)
-    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
-    #[serde(default = "default_page_marker_format")]
-    pub marker_format: String,
-}
-
-impl Default for PageConfig {
-    fn default() -> Self {
-        Self {
-            extract_pages: false,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }
-    }
-}
-
-/// Main extraction configuration.
-///
-/// This struct contains all configuration options for the extraction process.
-/// It can be loaded from TOML, YAML, or JSON files, or created programmatically.
-///
-/// # Example
-///
-/// ```rust
-/// use kreuzberg::core::config::ExtractionConfig;
-///
-/// // Create with defaults
-/// let config = ExtractionConfig::default();
-///
-/// // Load from TOML file
-/// // let config = ExtractionConfig::from_toml_file("kreuzberg.toml")?;
-/// ```
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ExtractionConfig {
-    /// Enable caching of extraction results
-    #[serde(default = "default_true")]
-    pub use_cache: bool,
-
-    /// Enable quality post-processing
-    #[serde(default = "default_true")]
-    pub enable_quality_processing: bool,
-
-    /// OCR configuration (None = OCR disabled)
-    #[serde(default)]
-    pub ocr: Option<OcrConfig>,
-
-    /// Force OCR even for searchable PDFs
-    #[serde(default)]
-    pub force_ocr: bool,
-
-    /// Text chunking configuration (None = chunking disabled)
-    #[serde(default)]
-    pub chunking: Option<ChunkingConfig>,
-
-    /// Image extraction configuration (None = no image extraction)
-    #[serde(default)]
-    pub images: Option<ImageExtractionConfig>,
-
-    /// PDF-specific options (None = use defaults)
-    #[cfg(feature = "pdf")]
-    #[serde(default)]
-    pub pdf_options: Option<PdfConfig>,
-
-    /// Token reduction configuration (None = no token reduction)
-    #[serde(default)]
-    pub token_reduction: Option<TokenReductionConfig>,
-
-    /// Language detection configuration (None = no language detection)
-    #[serde(default)]
-    pub language_detection: Option<LanguageDetectionConfig>,
-
-    /// Page extraction configuration (None = no page tracking)
-    #[serde(default)]
-    pub pages: Option<PageConfig>,
-
-    /// Keyword extraction configuration (None = no keyword extraction)
-    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-    #[serde(default)]
-    pub keywords: Option<crate::keywords::KeywordConfig>,
-
-    /// Post-processor configuration (None = use defaults)
-    #[serde(default)]
-    pub postprocessor: Option<PostProcessorConfig>,
-
-    /// HTML to Markdown conversion options (None = use defaults)
-    ///
-    /// Configure how HTML documents are converted to Markdown, including heading styles,
-    /// list formatting, code block styles, and preprocessing options.
-    #[cfg(feature = "html")]
-    #[serde(default)]
-    pub html_options: Option<html_to_markdown_rs::ConversionOptions>,
-
-    /// Maximum concurrent extractions in batch operations (None = num_cpus * 2).
-    ///
-    /// Limits parallelism to prevent resource exhaustion when processing
-    /// large batches. Defaults to twice the number of CPU cores.
-    #[serde(default)]
-    pub max_concurrent_extractions: Option<usize>,
-}
-
-/// Post-processor configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PostProcessorConfig {
-    /// Enable post-processors
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Whitelist of processor names to run (None = all enabled)
-    #[serde(default)]
-    pub enabled_processors: Option<Vec<String>>,
-
-    /// Blacklist of processor names to skip (None = none disabled)
-    #[serde(default)]
-    pub disabled_processors: Option<Vec<String>>,
-
-    /// Pre-computed HashSet for O(1) enabled processor lookup
-    #[serde(skip)]
-    pub enabled_set: Option<HashSet<String>>,
-
-    /// Pre-computed HashSet for O(1) disabled processor lookup
-    #[serde(skip)]
-    pub disabled_set: Option<HashSet<String>>,
-}
-
-impl PostProcessorConfig {
-    /// Pre-compute HashSets for O(1) processor name lookups.
-    ///
-    /// This method converts the enabled/disabled processor Vec to HashSet
-    /// for constant-time lookups in the pipeline.
-    pub fn build_lookup_sets(&mut self) {
-        if let Some(ref enabled) = self.enabled_processors {
-            self.enabled_set = Some(enabled.iter().cloned().collect());
-        }
-        if let Some(ref disabled) = self.disabled_processors {
-            self.disabled_set = Some(disabled.iter().cloned().collect());
-        }
-    }
-}
-
-/// OCR configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct OcrConfig {
-    /// OCR backend: tesseract, easyocr, paddleocr
-    #[serde(default = "default_tesseract_backend")]
-    pub backend: String,
-
-    /// Language code (e.g., "eng", "deu")
-    #[serde(default = "default_eng")]
-    pub language: String,
-
-    /// Tesseract-specific configuration (optional)
-    #[serde(default)]
-    pub tesseract_config: Option<crate::types::TesseractConfig>,
-}
-
-impl Default for OcrConfig {
-    fn default() -> Self {
-        Self {
-            backend: default_tesseract_backend(),
-            language: default_eng(),
-            tesseract_config: None,
-        }
-    }
-}
-
-/// Chunking configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ChunkingConfig {
-    /// Maximum characters per chunk
-    #[serde(default = "default_chunk_size")]
-    pub max_chars: usize,
-
-    /// Overlap between chunks in characters
-    #[serde(default = "default_chunk_overlap")]
-    pub max_overlap: usize,
-
-    /// Optional embedding configuration for chunk embeddings
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub embedding: Option<EmbeddingConfig>,
-
-    /// Use a preset configuration (overrides individual settings if provided)
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub preset: Option<String>,
-}
-
-/// Embedding configuration for text chunks.
-///
-/// Configures embedding generation using ONNX models via fastembed-rs.
-/// Requires the `embeddings` feature to be enabled.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EmbeddingConfig {
-    /// The embedding model to use
-    pub model: EmbeddingModelType,
-
-    /// Whether to normalize embedding vectors (recommended for cosine similarity)
-    #[serde(default = "default_normalize")]
-    pub normalize: bool,
-
-    /// Batch size for embedding generation
-    #[serde(default = "default_batch_size")]
-    pub batch_size: usize,
-
-    /// Show model download progress
-    #[serde(default)]
-    pub show_download_progress: bool,
-
-    /// Custom cache directory for model files
-    ///
-    /// Defaults to `~/.cache/kreuzberg/embeddings/` if not specified.
-    /// Allows full customization of model download location.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_dir: Option<std::path::PathBuf>,
-}
-
-impl Default for EmbeddingConfig {
-    fn default() -> Self {
-        Self {
-            model: EmbeddingModelType::Preset {
-                name: "balanced".to_string(),
-            },
-            normalize: true,
-            batch_size: 32,
-            show_download_progress: false,
-            cache_dir: None,
-        }
-    }
-}
-
-/// Embedding model types supported by Kreuzberg.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum EmbeddingModelType {
-    /// Use a preset model configuration (recommended)
-    Preset { name: String },
-
-    /// Use a specific fastembed model by name
-    #[cfg(feature = "embeddings")]
-    FastEmbed { model: String, dimensions: usize },
-
-    /// Use a custom ONNX model from HuggingFace
-    Custom { model_id: String, dimensions: usize },
-}
-
-/// Image extraction configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ImageExtractionConfig {
-    /// Extract images from documents
-    #[serde(default = "default_true")]
-    pub extract_images: bool,
-
-    /// Target DPI for image normalization
-    #[serde(default = "default_target_dpi")]
-    pub target_dpi: i32,
-
-    /// Maximum dimension for images (width or height)
-    #[serde(default = "default_max_dimension")]
-    pub max_image_dimension: i32,
-
-    /// Automatically adjust DPI based on image content
-    #[serde(default = "default_true")]
-    pub auto_adjust_dpi: bool,
-
-    /// Minimum DPI threshold
-    #[serde(default = "default_min_dpi")]
-    pub min_dpi: i32,
-
-    /// Maximum DPI threshold
-    #[serde(default = "default_max_dpi")]
-    pub max_dpi: i32,
-}
-
-/// PDF-specific configuration.
-#[cfg(feature = "pdf")]
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PdfConfig {
-    /// Extract images from PDF
-    #[serde(default)]
-    pub extract_images: bool,
-
-    /// List of passwords to try when opening encrypted PDFs
-    #[serde(default)]
-    pub passwords: Option<Vec<String>>,
-
-    /// Extract PDF metadata
-    #[serde(default = "default_true")]
-    pub extract_metadata: bool,
-
-    /// Hierarchy extraction configuration (None = hierarchy extraction disabled)
-    #[serde(default)]
-    pub hierarchy: Option<HierarchyConfig>,
-}
-
-/// Hierarchy extraction configuration for PDF text structure analysis.
-///
-/// Enables extraction of document hierarchy levels (H1-H6) based on font size
-/// clustering and semantic analysis. When enabled, hierarchical blocks are
-/// included in page content.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct HierarchyConfig {
-    /// Enable hierarchy extraction
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Number of font size clusters to use for hierarchy levels (1-7)
-    ///
-    /// Default: 6, which provides H1-H6 heading levels with body text.
-    /// Larger values create more fine-grained hierarchy levels.
-    #[serde(default = "default_k_clusters")]
-    pub k_clusters: usize,
-
-    /// Include bounding box information in hierarchy blocks
-    #[serde(default = "default_true")]
-    pub include_bbox: bool,
-
-    /// OCR coverage threshold for smart OCR triggering (0.0-1.0)
-    ///
-    /// Determines when OCR should be triggered based on text block coverage.
-    /// OCR is triggered when text blocks cover less than this fraction of the page.
-    /// Default: 0.5 (trigger OCR if less than 50% of page has text)
-    #[serde(default = "default_ocr_coverage_threshold")]
-    pub ocr_coverage_threshold: Option<f32>,
-}
-
-impl Default for HierarchyConfig {
-    fn default() -> Self {
-        Self {
-            enabled: true,
-            k_clusters: 6,
-            include_bbox: true,
-            ocr_coverage_threshold: None,
-        }
-    }
-}
-
-fn default_k_clusters() -> usize {
-    6
-}
-
-fn default_ocr_coverage_threshold() -> Option<f32> {
-    None
-}
-
-/// Token reduction configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TokenReductionConfig {
-    /// Reduction mode: "off", "light", "moderate", "aggressive", "maximum"
-    #[serde(default = "default_reduction_mode")]
-    pub mode: String,
-
-    /// Preserve important words (capitalized, technical terms)
-    #[serde(default = "default_true")]
-    pub preserve_important_words: bool,
-}
-
-/// Language detection configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LanguageDetectionConfig {
-    /// Enable language detection
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Minimum confidence threshold (0.0-1.0)
-    #[serde(default = "default_confidence")]
-    pub min_confidence: f64,
-
-    /// Detect multiple languages in the document
-    #[serde(default)]
-    pub detect_multiple: bool,
-}
-
-fn default_true() -> bool {
-    true
-}
-fn default_eng() -> String {
-    "eng".to_string()
-}
-fn default_tesseract_backend() -> String {
-    "tesseract".to_string()
-}
-fn default_page_marker_format() -> String {
-    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
-}
-fn default_chunk_size() -> usize {
-    1000
-}
-fn default_chunk_overlap() -> usize {
-    200
-}
-fn default_normalize() -> bool {
-    true
-}
-fn default_batch_size() -> usize {
-    32
-}
-fn default_target_dpi() -> i32 {
-    300
-}
-fn default_max_dimension() -> i32 {
-    4096
-}
-fn default_min_dpi() -> i32 {
-    72
-}
-fn default_max_dpi() -> i32 {
-    600
-}
-fn default_reduction_mode() -> String {
-    "off".to_string()
-}
-fn default_confidence() -> f64 {
-    0.8
-}
-
-impl Default for ExtractionConfig {
-    fn default() -> Self {
-        Self {
-            use_cache: true,
-            enable_quality_processing: true,
-            ocr: None,
-            force_ocr: false,
-            chunking: None,
-            images: None,
-            #[cfg(feature = "pdf")]
-            pdf_options: None,
-            token_reduction: None,
-            language_detection: None,
-            pages: None,
-            #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-            keywords: None,
-            postprocessor: None,
-            #[cfg(feature = "html")]
-            html_options: None,
-            max_concurrent_extractions: None,
-        }
-    }
-}
-
-impl Default for PostProcessorConfig {
-    fn default() -> Self {
-        Self {
-            enabled: true,
-            enabled_processors: None,
-            disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
-        }
-    }
-}
-
-impl ExtractionConfig {
-    /// Check if image processing is needed by examining OCR and image extraction settings.
-    ///
-    /// Returns `true` if either OCR is enabled or image extraction is configured,
-    /// indicating that image decompression and processing should occur.
-    /// Returns `false` if both are disabled, allowing optimization to skip unnecessary
-    /// image decompression for text-only extraction workflows.
-    ///
-    /// # Optimization Impact
-    /// For text-only extractions (no OCR, no image extraction), skipping image
-    /// decompression can improve CPU utilization by 5-10% by avoiding wasteful
-    /// image I/O and processing when results won't be used.
-    pub fn needs_image_processing(&self) -> bool {
-        let ocr_enabled = self.ocr.is_some();
-
-        let image_extraction_enabled = self.images.as_ref().map(|i| i.extract_images).unwrap_or(false);
-
-        ocr_enabled || image_extraction_enabled
-    }
-
-    /// Apply environment variable overrides to configuration.
-    ///
-    /// Environment variables have the highest precedence and will override any values
-    /// loaded from configuration files. This method supports the following environment variables:
-    ///
-    /// - `KREUZBERG_OCR_LANGUAGE`: OCR language (ISO 639-1 or 639-3 code, e.g., "eng", "fra", "deu")
-    /// - `KREUZBERG_OCR_BACKEND`: OCR backend ("tesseract", "easyocr", or "paddleocr")
-    /// - `KREUZBERG_CHUNKING_MAX_CHARS`: Maximum characters per chunk (positive integer)
-    /// - `KREUZBERG_CHUNKING_MAX_OVERLAP`: Maximum overlap between chunks (non-negative integer)
-    /// - `KREUZBERG_CACHE_ENABLED`: Cache enabled flag ("true" or "false")
-    /// - `KREUZBERG_TOKEN_REDUCTION_MODE`: Token reduction mode ("off", "light", "moderate", "aggressive", or "maximum")
-    ///
-    /// # Behavior
-    ///
-    /// - If an environment variable is set and valid, it overrides the current configuration value
-    /// - If a required parent config is `None` (e.g., `self.ocr` is None), it's created with defaults before applying the override
-    /// - Invalid values return a `KreuzbergError::Validation` with helpful error messages
-    /// - Missing or unset environment variables are silently ignored
-    ///
-    /// # Example
-    ///
-    /// ```rust
-    /// # use kreuzberg::core::config::ExtractionConfig;
-    /// # fn example() -> kreuzberg::Result<()> {
-    /// let mut config = ExtractionConfig::from_file("config.toml")?;
-    /// // Set KREUZBERG_OCR_LANGUAGE=fra before calling
-    /// config.apply_env_overrides()?; // OCR language is now "fra"
-    /// # Ok(())
-    /// # }
-    /// ```
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if:
-    /// - An environment variable contains an invalid value
-    /// - A number cannot be parsed as the expected type
-    /// - A boolean is not "true" or "false"
-    pub fn apply_env_overrides(&mut self) -> Result<()> {
-        use crate::core::config_validation::{
-            validate_chunking_params, validate_language_code, validate_ocr_backend, validate_token_reduction_level,
-        };
-
-        // KREUZBERG_OCR_LANGUAGE override
-        if let Ok(lang) = std::env::var("KREUZBERG_OCR_LANGUAGE") {
-            validate_language_code(&lang)?;
-            if self.ocr.is_none() {
-                self.ocr = Some(OcrConfig::default());
-            }
-            if let Some(ref mut ocr) = self.ocr {
-                ocr.language = lang;
-            }
-        }
-
-        // KREUZBERG_OCR_BACKEND override
-        if let Ok(backend) = std::env::var("KREUZBERG_OCR_BACKEND") {
-            validate_ocr_backend(&backend)?;
-            if self.ocr.is_none() {
-                self.ocr = Some(OcrConfig::default());
-            }
-            if let Some(ref mut ocr) = self.ocr {
-                ocr.backend = backend;
-            }
-        }
-
-        // KREUZBERG_CHUNKING_MAX_CHARS override
-        if let Ok(max_chars_str) = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS") {
-            let max_chars: usize = max_chars_str.parse().map_err(|_| KreuzbergError::Validation {
-                message: format!(
-                    "Invalid value for KREUZBERG_CHUNKING_MAX_CHARS: '{}'. Must be a positive integer.",
-                    max_chars_str
-                ),
-                source: None,
-            })?;
-
-            if max_chars == 0 {
-                return Err(KreuzbergError::Validation {
-                    message: "KREUZBERG_CHUNKING_MAX_CHARS must be greater than 0".to_string(),
-                    source: None,
-                });
-            }
-
-            if self.chunking.is_none() {
-                self.chunking = Some(ChunkingConfig {
-                    max_chars: default_chunk_size(),
-                    max_overlap: default_chunk_overlap(),
-                    embedding: None,
-                    preset: None,
-                });
-            }
-
-            if let Some(ref mut chunking) = self.chunking {
-                // Validate against current overlap before updating
-                validate_chunking_params(max_chars, chunking.max_overlap)?;
-                chunking.max_chars = max_chars;
-            }
-        }
-
-        // KREUZBERG_CHUNKING_MAX_OVERLAP override
-        if let Ok(max_overlap_str) = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP") {
-            let max_overlap: usize = max_overlap_str.parse().map_err(|_| KreuzbergError::Validation {
-                message: format!(
-                    "Invalid value for KREUZBERG_CHUNKING_MAX_OVERLAP: '{}'. Must be a non-negative integer.",
-                    max_overlap_str
-                ),
-                source: None,
-            })?;
-
-            if self.chunking.is_none() {
-                self.chunking = Some(ChunkingConfig {
-                    max_chars: default_chunk_size(),
-                    max_overlap: default_chunk_overlap(),
-                    embedding: None,
-                    preset: None,
-                });
-            }
-
-            if let Some(ref mut chunking) = self.chunking {
-                // Validate against current max_chars before updating
-                validate_chunking_params(chunking.max_chars, max_overlap)?;
-                chunking.max_overlap = max_overlap;
-            }
-        }
-
-        // KREUZBERG_CACHE_ENABLED override
-        if let Ok(cache_str) = std::env::var("KREUZBERG_CACHE_ENABLED") {
-            let cache_enabled = match cache_str.to_lowercase().as_str() {
-                "true" => true,
-                "false" => false,
-                _ => {
-                    return Err(KreuzbergError::Validation {
-                        message: format!(
-                            "Invalid value for KREUZBERG_CACHE_ENABLED: '{}'. Must be 'true' or 'false'.",
-                            cache_str
-                        ),
-                        source: None,
-                    });
-                }
-            };
-            self.use_cache = cache_enabled;
-        }
-
-        // KREUZBERG_TOKEN_REDUCTION_MODE override
-        if let Ok(mode) = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE") {
-            validate_token_reduction_level(&mode)?;
-            if self.token_reduction.is_none() {
-                self.token_reduction = Some(TokenReductionConfig {
-                    mode: default_reduction_mode(),
-                    preserve_important_words: default_true(),
-                });
-            }
-            if let Some(ref mut token_reduction) = self.token_reduction {
-                token_reduction.mode = mode;
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Load configuration from a TOML file.
-    ///
-    /// # Arguments
-    ///
-    /// * `path` - Path to the TOML file
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if file doesn't exist or is invalid TOML.
-    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
-        })?;
-
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = toml::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
-    }
-
-    /// Load configuration from a YAML file.
-    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
-        })?;
-
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = serde_yaml_ng::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
-    }
-
-    /// Load configuration from a JSON file.
-    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
-        })?;
-
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = serde_json::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
-    }
-
-    /// Load configuration from a file, auto-detecting format by extension.
-    ///
-    /// Supported formats:
-    /// - `.toml` - TOML format
-    /// - `.yaml` - YAML format
-    /// - `.json` - JSON format
-    ///
-    /// # Arguments
-    ///
-    /// * `path` - Path to the configuration file
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if:
-    /// - File doesn't exist
-    /// - File extension is not supported
-    /// - File content is invalid for the detected format
-    ///
-    /// # Example
-    ///
-    /// ```rust
-    /// use kreuzberg::core::config::ExtractionConfig;
-    ///
-    /// // Auto-detects TOML format
-    /// // let config = ExtractionConfig::from_file("kreuzberg.toml")?;
-    ///
-    /// // Auto-detects YAML format
-    /// // let config = ExtractionConfig::from_file("kreuzberg.yaml")?;
-    /// ```
-    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
-        })?;
-
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
-            KreuzbergError::validation(format!(
-                "Cannot determine file format: no extension found in {}",
-                path.display()
-            ))
-        })?;
-
-        let config = match extension.to_lowercase().as_str() {
-            "toml" => Self::from_toml_file(path)?,
-            "yaml" | "yml" => Self::from_yaml_file(path)?,
-            "json" => Self::from_json_file(path)?,
-            _ => {
-                return Err(KreuzbergError::validation(format!(
-                    "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
-                    extension
-                )));
-            }
-        };
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
-    }
-
-    /// Discover configuration file in parent directories.
-    ///
-    /// Searches for `kreuzberg.toml` in current directory and parent directories.
-    ///
-    /// # Returns
-    ///
-    /// - `Some(config)` if found
-    /// - `None` if no config file found
-    pub fn discover() -> Result<Option<Self>> {
-        let mut current = std::env::current_dir().map_err(KreuzbergError::Io)?;
-
-        loop {
-            let kreuzberg_toml = current.join("kreuzberg.toml");
-            if kreuzberg_toml.exists() {
-                return Ok(Some(Self::from_toml_file(kreuzberg_toml)?));
-            }
-
-            if let Some(parent) = current.parent() {
-                current = parent.to_path_buf();
-            } else {
-                break;
-            }
-        }
-
-        Ok(None)
-    }
-}
-
-#[cfg(test)]
-#[allow(unsafe_code)]
-mod tests {
-    use super::*;
-    use std::fs;
-    use tempfile::tempdir;
-
-    fn set_env(key: &str, value: &str) {
-        unsafe {
-            std::env::set_var(key, value);
-        }
-    }
-
-    #[allow(unsafe_code)]
-    fn remove_env(key: &str) {
-        unsafe {
-            std::env::remove_var(key);
-        }
-    }
-
-    #[allow(unsafe_code)]
-    fn restore_env(key: &str, value: Option<String>) {
-        unsafe {
-            if let Some(v) = value {
-                std::env::set_var(key, v);
-            } else {
-                std::env::remove_var(key);
-            }
-        }
-    }
-
-    #[test]
-    fn test_default_config() {
-        let config = ExtractionConfig::default();
-        assert!(config.use_cache);
-        assert!(config.enable_quality_processing);
-        assert!(config.ocr.is_none());
-    }
-
-    #[test]
-    fn test_from_toml_file() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(!config.use_cache);
-        assert!(config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_auto_detects_toml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_file(&config_path).unwrap();
-        assert!(!config.use_cache);
-        assert!(config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_auto_detects_yaml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.yaml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache: true
-enable_quality_processing: false
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(!config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_unsupported_extension() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.txt");
-
-        fs::write(&config_path, "use_cache = false").unwrap();
-
-        let result = ExtractionConfig::from_file(&config_path);
-        assert!(result.is_err());
-        let err = result.unwrap_err();
-        assert!(err.to_string().contains("Unsupported config file format"));
-    }
-
-    #[test]
-    fn test_from_file_no_extension() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg");
-
-        fs::write(&config_path, "use_cache = false").unwrap();
-
-        let result = ExtractionConfig::from_file(&config_path);
-        assert!(result.is_err());
-        let err = result.unwrap_err();
-        assert!(err.to_string().contains("no extension found"));
-    }
-
-    #[test]
-    fn test_discover_kreuzberg_toml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&dir).unwrap();
-
-        let result = std::panic::catch_unwind(|| {
-            let config = ExtractionConfig::discover().unwrap();
-            assert!(config.is_some());
-            assert!(!config.unwrap().use_cache);
-        });
-
-        std::env::set_current_dir(&original_dir).unwrap();
-
-        if let Err(e) = result {
-            std::panic::resume_unwind(e);
-        }
-    }
-
-    #[test]
-    fn test_v4_config_with_ocr_and_chunking() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = false
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[chunking]
-max_chars = 2000
-max_overlap = 300
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(!config.enable_quality_processing);
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().backend, "tesseract");
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_chars, 2000);
-    }
-
-    #[test]
-    fn test_config_with_image_extraction() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[images]
-extract_images = true
-target_dpi = 300
-max_image_dimension = 4096
-auto_adjust_dpi = true
-min_dpi = 72
-max_dpi = 600
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.images.is_some());
-        let images = config.images.unwrap();
-        assert!(images.extract_images);
-        assert_eq!(images.target_dpi, 300);
-        assert_eq!(images.max_image_dimension, 4096);
-        assert!(images.auto_adjust_dpi);
-        assert_eq!(images.min_dpi, 72);
-        assert_eq!(images.max_dpi, 600);
-    }
-
-    #[test]
-    #[cfg(feature = "pdf")]
-    fn test_config_with_pdf_options() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[pdf_options]
-extract_images = true
-passwords = ["password1", "password2"]
-extract_metadata = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.pdf_options.is_some());
-        let pdf = config.pdf_options.unwrap();
-        assert!(pdf.extract_images);
-        assert!(pdf.extract_metadata);
-        assert!(pdf.passwords.is_some());
-        let passwords = pdf.passwords.unwrap();
-        assert_eq!(passwords.len(), 2);
-        assert_eq!(passwords[0], "password1");
-        assert_eq!(passwords[1], "password2");
-    }
-
-    #[test]
-    fn test_config_with_token_reduction() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[token_reduction]
-mode = "aggressive"
-preserve_important_words = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.token_reduction.is_some());
-        let token = config.token_reduction.unwrap();
-        assert_eq!(token.mode, "aggressive");
-        assert!(token.preserve_important_words);
-    }
-
-    #[test]
-    fn test_config_with_language_detection() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[language_detection]
-enabled = true
-min_confidence = 0.9
-detect_multiple = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.language_detection.is_some());
-        let lang = config.language_detection.unwrap();
-        assert!(lang.enabled);
-        assert_eq!(lang.min_confidence, 0.9);
-        assert!(lang.detect_multiple);
-    }
-
-    #[test]
-    fn test_config_with_all_optional_fields() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = true
-force_ocr = false
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[chunking]
-max_chars = 1500
-max_overlap = 250
-
-[images]
-extract_images = true
-target_dpi = 300
-
-[pdf_options]
-extract_images = false
-extract_metadata = true
-
-[token_reduction]
-mode = "moderate"
-
-[language_detection]
-enabled = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(config.enable_quality_processing);
-        assert!(!config.force_ocr);
-        assert!(config.ocr.is_some());
-        assert!(config.chunking.is_some());
-        assert!(config.images.is_some());
-        assert!(config.token_reduction.is_some());
-        assert!(config.language_detection.is_some());
-        #[cfg(feature = "pdf")]
-        assert!(config.pdf_options.is_some());
-    }
-
-    #[test]
-    fn test_image_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[images]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let images = config.images.unwrap();
-        assert!(images.extract_images);
-        assert_eq!(images.target_dpi, 300);
-        assert_eq!(images.max_image_dimension, 4096);
-        assert!(images.auto_adjust_dpi);
-        assert_eq!(images.min_dpi, 72);
-        assert_eq!(images.max_dpi, 600);
-    }
-
-    #[test]
-    fn test_token_reduction_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[token_reduction]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let token = config.token_reduction.unwrap();
-        assert_eq!(token.mode, "off");
-        assert!(token.preserve_important_words);
-    }
-
-    #[test]
-    fn test_language_detection_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[language_detection]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let lang = config.language_detection.unwrap();
-        assert!(lang.enabled);
-        assert_eq!(lang.min_confidence, 0.8);
-        assert!(!lang.detect_multiple);
-    }
-
-    #[test]
-    #[cfg(feature = "pdf")]
-    fn test_pdf_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[pdf_options]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pdf = config.pdf_options.unwrap();
-        assert!(!pdf.extract_images);
-        assert!(pdf.extract_metadata);
-        assert!(pdf.passwords.is_none());
-    }
-
-    #[test]
-    fn test_config_with_postprocessor() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[postprocessor]
-enabled = true
-enabled_processors = ["entity_extraction", "keyword_extraction"]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.postprocessor.is_some());
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.enabled_processors.is_some());
-        let enabled = pp.enabled_processors.unwrap();
-        assert_eq!(enabled.len(), 2);
-        assert_eq!(enabled[0], "entity_extraction");
-        assert_eq!(enabled[1], "keyword_extraction");
-        assert!(pp.disabled_processors.is_none());
-    }
-
-    #[test]
-    fn test_postprocessor_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.enabled_processors.is_none());
-        assert!(pp.disabled_processors.is_none());
-    }
-
-    #[test]
-    fn test_postprocessor_config_disabled() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-enabled = false
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(!pp.enabled);
-    }
-
-    #[test]
-    fn test_postprocessor_config_blacklist() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-disabled_processors = ["category_extraction"]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.disabled_processors.is_some());
-        let disabled = pp.disabled_processors.unwrap();
-        assert_eq!(disabled.len(), 1);
-        assert_eq!(disabled[0], "category_extraction");
-    }
-
-    #[test]
-    fn test_config_with_tesseract_config() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[ocr.tesseract_config]
-psm = 6
-output_format = "text"
-enable_table_detection = false
-tessedit_char_whitelist = "0123456789"
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.ocr.is_some());
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.backend, "tesseract");
-        assert_eq!(ocr.language, "eng");
-        assert!(ocr.tesseract_config.is_some());
-
-        let tess = ocr.tesseract_config.unwrap();
-        assert_eq!(tess.psm, 6);
-        assert_eq!(tess.output_format, "text");
-        assert!(!tess.enable_table_detection);
-        assert_eq!(tess.tessedit_char_whitelist, "0123456789");
-    }
-
-    #[test]
-    fn test_ocr_config_without_tesseract_config() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[ocr]
-backend = "easyocr"
-language = "eng"
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.ocr.is_some());
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.backend, "easyocr");
-        assert_eq!(ocr.language, "eng");
-        assert!(ocr.tesseract_config.is_none());
-    }
-
-    #[test]
-    fn test_tesseract_config_defaults() {
-        let tess = crate::types::TesseractConfig::default();
-        assert_eq!(tess.language, "eng");
-        assert_eq!(tess.psm, 3);
-        assert_eq!(tess.output_format, "markdown");
-        assert!(tess.enable_table_detection);
-        assert_eq!(tess.table_min_confidence, 0.0);
-        assert_eq!(tess.table_column_threshold, 50);
-        assert_eq!(tess.table_row_threshold_ratio, 0.5);
-        assert!(tess.use_cache);
-        assert!(tess.classify_use_pre_adapted_templates);
-        assert!(!tess.language_model_ngram_on);
-        assert!(tess.tessedit_dont_blkrej_good_wds);
-        assert!(tess.tessedit_dont_rowrej_good_wds);
-        assert!(tess.tessedit_enable_dict_correction);
-        assert_eq!(tess.tessedit_char_whitelist, "");
-        assert!(tess.tessedit_use_primary_params_model);
-        assert!(tess.textord_space_size_is_variable);
-        assert!(!tess.thresholding_method);
-    }
-
-    #[test]
-    fn test_config_cache_hit_same_mtime() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-
-        assert_eq!(config1.use_cache, config2.use_cache);
-        assert_eq!(config1.enable_quality_processing, config2.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_config_cache_invalidation_on_mtime_change() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(!config1.use_cache);
-
-        std::thread::sleep(std::time::Duration::from_secs(1));
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = false
-        "#,
-        )
-        .unwrap();
-
-        let config2 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config2.use_cache);
-        assert!(!config2.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_config_cache_works_with_json() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.json");
-
-        fs::write(
-            &config_path,
-            r#"{"use_cache": true, "enable_quality_processing": false}"#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_json_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_json_file(&config_path).unwrap();
-
-        assert!(config1.use_cache);
-        assert!(!config1.enable_quality_processing);
-        assert_eq!(config1.use_cache, config2.use_cache);
-    }
-
-    #[test]
-    fn test_config_cache_works_with_yaml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.yaml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache: true
-enable_quality_processing: false
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_yaml_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_yaml_file(&config_path).unwrap();
-
-        assert!(config1.use_cache);
-        assert!(!config1.enable_quality_processing);
-        assert_eq!(config1.use_cache, config2.use_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_ocr_language() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "fra");
-        let mut config = ExtractionConfig::default();
-        assert!(config.ocr.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().language, "fra");
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_ocr_backend() {
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-
-        set_env("KREUZBERG_OCR_BACKEND", "easyocr");
-        let mut config = ExtractionConfig::default();
-        assert!(config.ocr.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().backend, "easyocr");
-
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_ocr_language() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "invalid_lang");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid language code"));
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_ocr_backend() {
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-
-        set_env("KREUZBERG_OCR_BACKEND", "invalid_backend");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid OCR backend"));
-
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_chars() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "2000");
-        let mut config = ExtractionConfig::default();
-        assert!(config.chunking.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_chars, 2000);
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_overlap() {
-        let original_overlap = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_OVERLAP", "300");
-        let mut config = ExtractionConfig::default();
-        assert!(config.chunking.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_overlap, 300);
-
-        restore_env("KREUZBERG_CHUNKING_MAX_OVERLAP", original_overlap);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_chunking_max_chars() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "not_a_number");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("KREUZBERG_CHUNKING_MAX_CHARS"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_chars_zero() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "0");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("greater than 0"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_overlap_validation() {
-        let original_overlap = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_OVERLAP", "1500");
-        // Set chunking with max_chars = 1000
-        let mut config = ExtractionConfig {
-            chunking: Some(ChunkingConfig {
-                max_chars: 1000,
-                max_overlap: 200,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("overlap"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_OVERLAP", original_overlap);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_true() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "true");
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_false() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "false");
-        let mut config = ExtractionConfig {
-            use_cache: true,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_case_insensitive() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "TRUE");
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(config.use_cache);
-
-        set_env("KREUZBERG_CACHE_ENABLED", "FaLsE");
-        config.use_cache = true;
-        config.apply_env_overrides().unwrap();
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_cache_enabled() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "maybe");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("KREUZBERG_CACHE_ENABLED"));
-        assert!(err_msg.contains("'true' or 'false'"));
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_token_reduction_mode() {
-        let original_mode = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE").ok();
-
-        set_env("KREUZBERG_TOKEN_REDUCTION_MODE", "moderate");
-        let mut config = ExtractionConfig::default();
-        assert!(config.token_reduction.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.token_reduction.is_some());
-        assert_eq!(config.token_reduction.unwrap().mode, "moderate");
-
-        restore_env("KREUZBERG_TOKEN_REDUCTION_MODE", original_mode);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_token_reduction_mode() {
-        let original_mode = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE").ok();
-
-        set_env("KREUZBERG_TOKEN_REDUCTION_MODE", "extreme");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid token reduction level"));
-
-        restore_env("KREUZBERG_TOKEN_REDUCTION_MODE", original_mode);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_multiple_overrides() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "deu");
-        set_env("KREUZBERG_OCR_BACKEND", "paddleocr");
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "3000");
-        set_env("KREUZBERG_CACHE_ENABLED", "false");
-
-        let mut config = ExtractionConfig::default();
-        config.apply_env_overrides().unwrap();
-
-        let ocr = config.ocr.as_ref().unwrap();
-        assert_eq!(ocr.language, "deu");
-        assert_eq!(ocr.backend, "paddleocr");
-        let chunking = config.chunking.as_ref().unwrap();
-        assert_eq!(chunking.max_chars, 3000);
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_preserves_existing_ocr_config() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "fra");
-
-        let mut config = ExtractionConfig {
-            ocr: Some(OcrConfig {
-                backend: "tesseract".to_string(),
-                language: "eng".to_string(),
-                tesseract_config: None,
-            }),
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.language, "fra");
-        assert_eq!(ocr.backend, "tesseract"); // Not overridden
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_no_env_vars() {
-        // Ensure no env vars are set
-        remove_env("KREUZBERG_OCR_LANGUAGE");
-        remove_env("KREUZBERG_OCR_BACKEND");
-        remove_env("KREUZBERG_CHUNKING_MAX_CHARS");
-        remove_env("KREUZBERG_CHUNKING_MAX_OVERLAP");
-        remove_env("KREUZBERG_CACHE_ENABLED");
-        remove_env("KREUZBERG_TOKEN_REDUCTION_MODE");
-
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ocr: Some(OcrConfig {
-                backend: "tesseract".to_string(),
-                language: "eng".to_string(),
-                tesseract_config: None,
-            }),
-            ..Default::default()
-        };
-
-        // Should not change anything
-        config.apply_env_overrides().unwrap();
-
-        assert!(!config.use_cache);
-        assert_eq!(config.ocr.unwrap().language, "eng");
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_params_validation() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "500");
-        let mut config = ExtractionConfig {
-            chunking: Some(ChunkingConfig {
-                max_chars: 1000,
-                max_overlap: 600, // Valid: < 1000
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        // After override, max_chars = 500, max_overlap = 600 (invalid: >= 500)
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-}