kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/core/config.rs

@@ -4,48 +4,8 @@
 //! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
 
 use crate::{KreuzbergError, Result};
-use dashmap::DashMap;
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, LazyLock};
-use std::time::SystemTime;
-
-static CONFIG_CACHE: LazyLock<DashMap<PathBuf, (SystemTime, Arc<ExtractionConfig>)>> = LazyLock::new(DashMap::new);
-
-/// Page extraction and tracking configuration.
-///
-/// Controls how pages are extracted, tracked, and represented in the extraction results.
-/// When `None`, page tracking is disabled.
-///
-/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
-/// when page boundaries are available and chunking is configured.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(default)]
-pub struct PageConfig {
-    /// Extract pages as separate array (ExtractionResult.pages)
-    #[serde(default)]
-    pub extract_pages: bool,
-
-    /// Insert page markers in main content string
-    #[serde(default)]
-    pub insert_page_markers: bool,
-
-    /// Page marker format (use {page_num} placeholder)
-    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
-    #[serde(default = "default_page_marker_format")]
-    pub marker_format: String,
-}
-
-impl Default for PageConfig {
-    fn default() -> Self {
-        Self {
-            extract_pages: false,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }
-    }
-}
+use std::path::Path;
 
 /// Main extraction configuration.
 ///
@@ -90,7 +50,6 @@ pub struct ExtractionConfig {
     pub images: Option<ImageExtractionConfig>,
 
     /// PDF-specific options (None = use defaults)
-    #[cfg(feature = "pdf")]
     #[serde(default)]
     pub pdf_options: Option<PdfConfig>,
 
@@ -102,10 +61,6 @@ pub struct ExtractionConfig {
     #[serde(default)]
     pub language_detection: Option<LanguageDetectionConfig>,
 
-    /// Page extraction configuration (None = no page tracking)
-    #[serde(default)]
-    pub pages: Option<PageConfig>,
-
     /// Keyword extraction configuration (None = no keyword extraction)
     #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
     #[serde(default)]
@@ -145,29 +100,6 @@ pub struct PostProcessorConfig {
     /// Blacklist of processor names to skip (None = none disabled)
     #[serde(default)]
     pub disabled_processors: Option<Vec<String>>,
-
-    /// Pre-computed HashSet for O(1) enabled processor lookup
-    #[serde(skip)]
-    pub enabled_set: Option<HashSet<String>>,
-
-    /// Pre-computed HashSet for O(1) disabled processor lookup
-    #[serde(skip)]
-    pub disabled_set: Option<HashSet<String>>,
-}
-
-impl PostProcessorConfig {
-    /// Pre-compute HashSets for O(1) processor name lookups.
-    ///
-    /// This method converts the enabled/disabled processor Vec to HashSet
-    /// for constant-time lookups in the pipeline.
-    pub fn build_lookup_sets(&mut self) {
-        if let Some(ref enabled) = self.enabled_processors {
-            self.enabled_set = Some(enabled.iter().cloned().collect());
-        }
-        if let Some(ref disabled) = self.disabled_processors {
-            self.disabled_set = Some(disabled.iter().cloned().collect());
-        }
-    }
 }
 
 /// OCR configuration.
@@ -186,16 +118,6 @@ pub struct OcrConfig {
     pub tesseract_config: Option<crate::types::TesseractConfig>,
 }
 
-impl Default for OcrConfig {
-    fn default() -> Self {
-        Self {
-            backend: default_tesseract_backend(),
-            language: default_eng(),
-            tesseract_config: None,
-        }
-    }
-}
-
 /// Chunking configuration.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ChunkingConfig {
@@ -303,7 +225,6 @@ pub struct ImageExtractionConfig {
 }
 
 /// PDF-specific configuration.
-#[cfg(feature = "pdf")]
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PdfConfig {
     /// Extract images from PDF
@@ -317,60 +238,6 @@ pub struct PdfConfig {
     /// Extract PDF metadata
     #[serde(default = "default_true")]
     pub extract_metadata: bool,
-
-    /// Hierarchy extraction configuration (None = hierarchy extraction disabled)
-    #[serde(default)]
-    pub hierarchy: Option<HierarchyConfig>,
-}
-
-/// Hierarchy extraction configuration for PDF text structure analysis.
-///
-/// Enables extraction of document hierarchy levels (H1-H6) based on font size
-/// clustering and semantic analysis. When enabled, hierarchical blocks are
-/// included in page content.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct HierarchyConfig {
-    /// Enable hierarchy extraction
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Number of font size clusters to use for hierarchy levels (1-7)
-    ///
-    /// Default: 6, which provides H1-H6 heading levels with body text.
-    /// Larger values create more fine-grained hierarchy levels.
-    #[serde(default = "default_k_clusters")]
-    pub k_clusters: usize,
-
-    /// Include bounding box information in hierarchy blocks
-    #[serde(default = "default_true")]
-    pub include_bbox: bool,
-
-    /// OCR coverage threshold for smart OCR triggering (0.0-1.0)
-    ///
-    /// Determines when OCR should be triggered based on text block coverage.
-    /// OCR is triggered when text blocks cover less than this fraction of the page.
-    /// Default: 0.5 (trigger OCR if less than 50% of page has text)
-    #[serde(default = "default_ocr_coverage_threshold")]
-    pub ocr_coverage_threshold: Option<f32>,
-}
-
-impl Default for HierarchyConfig {
-    fn default() -> Self {
-        Self {
-            enabled: true,
-            k_clusters: 6,
-            include_bbox: true,
-            ocr_coverage_threshold: None,
-        }
-    }
-}
-
-fn default_k_clusters() -> usize {
-    6
-}
-
-fn default_ocr_coverage_threshold() -> Option<f32> {
-    None
 }
 
 /// Token reduction configuration.
@@ -410,9 +277,6 @@ fn default_eng() -> String {
 fn default_tesseract_backend() -> String {
     "tesseract".to_string()
 }
-fn default_page_marker_format() -> String {
-    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
-}
 fn default_chunk_size() -> usize {
     1000
 }
@@ -453,11 +317,9 @@ impl Default for ExtractionConfig {
             force_ocr: false,
             chunking: None,
             images: None,
-            #[cfg(feature = "pdf")]
             pdf_options: None,
             token_reduction: None,
             language_detection: None,
-            pages: None,
             #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
             keywords: None,
             postprocessor: None,
@@ -474,190 +336,11 @@ impl Default for PostProcessorConfig {
             enabled: true,
             enabled_processors: None,
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }
     }
 }
 
 impl ExtractionConfig {
-    /// Check if image processing is needed by examining OCR and image extraction settings.
-    ///
-    /// Returns `true` if either OCR is enabled or image extraction is configured,
-    /// indicating that image decompression and processing should occur.
-    /// Returns `false` if both are disabled, allowing optimization to skip unnecessary
-    /// image decompression for text-only extraction workflows.
-    ///
-    /// # Optimization Impact
-    /// For text-only extractions (no OCR, no image extraction), skipping image
-    /// decompression can improve CPU utilization by 5-10% by avoiding wasteful
-    /// image I/O and processing when results won't be used.
-    pub fn needs_image_processing(&self) -> bool {
-        let ocr_enabled = self.ocr.is_some();
-
-        let image_extraction_enabled = self.images.as_ref().map(|i| i.extract_images).unwrap_or(false);
-
-        ocr_enabled || image_extraction_enabled
-    }
-
-    /// Apply environment variable overrides to configuration.
-    ///
-    /// Environment variables have the highest precedence and will override any values
-    /// loaded from configuration files. This method supports the following environment variables:
-    ///
-    /// - `KREUZBERG_OCR_LANGUAGE`: OCR language (ISO 639-1 or 639-3 code, e.g., "eng", "fra", "deu")
-    /// - `KREUZBERG_OCR_BACKEND`: OCR backend ("tesseract", "easyocr", or "paddleocr")
-    /// - `KREUZBERG_CHUNKING_MAX_CHARS`: Maximum characters per chunk (positive integer)
-    /// - `KREUZBERG_CHUNKING_MAX_OVERLAP`: Maximum overlap between chunks (non-negative integer)
-    /// - `KREUZBERG_CACHE_ENABLED`: Cache enabled flag ("true" or "false")
-    /// - `KREUZBERG_TOKEN_REDUCTION_MODE`: Token reduction mode ("off", "light", "moderate", "aggressive", or "maximum")
-    ///
-    /// # Behavior
-    ///
-    /// - If an environment variable is set and valid, it overrides the current configuration value
-    /// - If a required parent config is `None` (e.g., `self.ocr` is None), it's created with defaults before applying the override
-    /// - Invalid values return a `KreuzbergError::Validation` with helpful error messages
-    /// - Missing or unset environment variables are silently ignored
-    ///
-    /// # Example
-    ///
-    /// ```rust
-    /// # use kreuzberg::core::config::ExtractionConfig;
-    /// # fn example() -> kreuzberg::Result<()> {
-    /// let mut config = ExtractionConfig::from_file("config.toml")?;
-    /// // Set KREUZBERG_OCR_LANGUAGE=fra before calling
-    /// config.apply_env_overrides()?; // OCR language is now "fra"
-    /// # Ok(())
-    /// # }
-    /// ```
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if:
-    /// - An environment variable contains an invalid value
-    /// - A number cannot be parsed as the expected type
-    /// - A boolean is not "true" or "false"
-    pub fn apply_env_overrides(&mut self) -> Result<()> {
-        use crate::core::config_validation::{
-            validate_chunking_params, validate_language_code, validate_ocr_backend, validate_token_reduction_level,
-        };
-
-        // KREUZBERG_OCR_LANGUAGE override
-        if let Ok(lang) = std::env::var("KREUZBERG_OCR_LANGUAGE") {
-            validate_language_code(&lang)?;
-            if self.ocr.is_none() {
-                self.ocr = Some(OcrConfig::default());
-            }
-            if let Some(ref mut ocr) = self.ocr {
-                ocr.language = lang;
-            }
-        }
-
-        // KREUZBERG_OCR_BACKEND override
-        if let Ok(backend) = std::env::var("KREUZBERG_OCR_BACKEND") {
-            validate_ocr_backend(&backend)?;
-            if self.ocr.is_none() {
-                self.ocr = Some(OcrConfig::default());
-            }
-            if let Some(ref mut ocr) = self.ocr {
-                ocr.backend = backend;
-            }
-        }
-
-        // KREUZBERG_CHUNKING_MAX_CHARS override
-        if let Ok(max_chars_str) = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS") {
-            let max_chars: usize = max_chars_str.parse().map_err(|_| KreuzbergError::Validation {
-                message: format!(
-                    "Invalid value for KREUZBERG_CHUNKING_MAX_CHARS: '{}'. Must be a positive integer.",
-                    max_chars_str
-                ),
-                source: None,
-            })?;
-
-            if max_chars == 0 {
-                return Err(KreuzbergError::Validation {
-                    message: "KREUZBERG_CHUNKING_MAX_CHARS must be greater than 0".to_string(),
-                    source: None,
-                });
-            }
-
-            if self.chunking.is_none() {
-                self.chunking = Some(ChunkingConfig {
-                    max_chars: default_chunk_size(),
-                    max_overlap: default_chunk_overlap(),
-                    embedding: None,
-                    preset: None,
-                });
-            }
-
-            if let Some(ref mut chunking) = self.chunking {
-                // Validate against current overlap before updating
-                validate_chunking_params(max_chars, chunking.max_overlap)?;
-                chunking.max_chars = max_chars;
-            }
-        }
-
-        // KREUZBERG_CHUNKING_MAX_OVERLAP override
-        if let Ok(max_overlap_str) = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP") {
-            let max_overlap: usize = max_overlap_str.parse().map_err(|_| KreuzbergError::Validation {
-                message: format!(
-                    "Invalid value for KREUZBERG_CHUNKING_MAX_OVERLAP: '{}'. Must be a non-negative integer.",
-                    max_overlap_str
-                ),
-                source: None,
-            })?;
-
-            if self.chunking.is_none() {
-                self.chunking = Some(ChunkingConfig {
-                    max_chars: default_chunk_size(),
-                    max_overlap: default_chunk_overlap(),
-                    embedding: None,
-                    preset: None,
-                });
-            }
-
-            if let Some(ref mut chunking) = self.chunking {
-                // Validate against current max_chars before updating
-                validate_chunking_params(chunking.max_chars, max_overlap)?;
-                chunking.max_overlap = max_overlap;
-            }
-        }
-
-        // KREUZBERG_CACHE_ENABLED override
-        if let Ok(cache_str) = std::env::var("KREUZBERG_CACHE_ENABLED") {
-            let cache_enabled = match cache_str.to_lowercase().as_str() {
-                "true" => true,
-                "false" => false,
-                _ => {
-                    return Err(KreuzbergError::Validation {
-                        message: format!(
-                            "Invalid value for KREUZBERG_CACHE_ENABLED: '{}'. Must be 'true' or 'false'.",
-                            cache_str
-                        ),
-                        source: None,
-                    });
-                }
-            };
-            self.use_cache = cache_enabled;
-        }
-
-        // KREUZBERG_TOKEN_REDUCTION_MODE override
-        if let Ok(mode) = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE") {
-            validate_token_reduction_level(&mode)?;
-            if self.token_reduction.is_none() {
-                self.token_reduction = Some(TokenReductionConfig {
-                    mode: default_reduction_mode(),
-                    preserve_important_words: default_true(),
-                });
-            }
-            if let Some(ref mut token_reduction) = self.token_reduction {
-                token_reduction.mode = mode;
-            }
-        }
-
-        Ok(())
-    }
-
     /// Load configuration from a TOML file.
     ///
     /// # Arguments
@@ -668,86 +351,32 @@ impl ExtractionConfig {
     ///
     /// Returns `KreuzbergError::Validation` if file doesn't exist or is invalid TOML.
     pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
         })?;
 
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = toml::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
+        toml::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.as_ref().display(), e)))
     }
 
     /// Load configuration from a YAML file.
     pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
         })?;
 
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = serde_yaml_ng::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
+        serde_yaml_ng::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.as_ref().display(), e)))
     }
 
     /// Load configuration from a JSON file.
     pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
         })?;
 
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
-        let content = std::fs::read_to_string(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-
-        let config: Self = serde_json::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))?;
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
+        serde_json::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.as_ref().display(), e)))
     }
 
     /// Load configuration from a file, auto-detecting format by extension.
@@ -781,19 +410,6 @@ impl ExtractionConfig {
     /// ```
     pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
         let path = path.as_ref();
-
-        let metadata = std::fs::metadata(path)
-            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
-        let mtime = metadata.modified().map_err(|e| {
-            KreuzbergError::validation(format!("Failed to get modification time for {}: {}", path.display(), e))
-        })?;
-
-        if let Some(entry) = CONFIG_CACHE.get(path)
-            && entry.0 == mtime
-        {
-            return Ok((*entry.1).clone());
-        }
-
         let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
             KreuzbergError::validation(format!(
                 "Cannot determine file format: no extension found in {}",
@@ -801,22 +417,15 @@ impl ExtractionConfig {
             ))
         })?;
 
-        let config = match extension.to_lowercase().as_str() {
-            "toml" => Self::from_toml_file(path)?,
-            "yaml" | "yml" => Self::from_yaml_file(path)?,
-            "json" => Self::from_json_file(path)?,
-            _ => {
-                return Err(KreuzbergError::validation(format!(
-                    "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
-                    extension
-                )));
-            }
-        };
-
-        let config_arc = Arc::new(config.clone());
-        CONFIG_CACHE.insert(path.to_path_buf(), (mtime, config_arc));
-
-        Ok(config)
+        match extension.to_lowercase().as_str() {
+            "toml" => Self::from_toml_file(path),
+            "yaml" | "yml" => Self::from_yaml_file(path),
+            "json" => Self::from_json_file(path),
+            _ => Err(KreuzbergError::validation(format!(
+                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
+                extension
+            ))),
+        }
     }
 
     /// Discover configuration file in parent directories.
@@ -848,36 +457,11 @@ impl ExtractionConfig {
 }
 
 #[cfg(test)]
-#[allow(unsafe_code)]
 mod tests {
     use super::*;
     use std::fs;
     use tempfile::tempdir;
 
-    fn set_env(key: &str, value: &str) {
-        unsafe {
-            std::env::set_var(key, value);
-        }
-    }
-
-    #[allow(unsafe_code)]
-    fn remove_env(key: &str) {
-        unsafe {
-            std::env::remove_var(key);
-        }
-    }
-
-    #[allow(unsafe_code)]
-    fn restore_env(key: &str, value: Option<String>) {
-        unsafe {
-            if let Some(v) = value {
-                std::env::set_var(key, v);
-            } else {
-                std::env::remove_var(key);
-            }
-        }
-    }
-
     #[test]
     fn test_default_config() {
         let config = ExtractionConfig::default();
@@ -1063,7 +647,6 @@ max_dpi = 600
     }
 
     #[test]
-    #[cfg(feature = "pdf")]
     fn test_config_with_pdf_options() {
         let dir = tempdir().unwrap();
         let config_path = dir.path().join("kreuzberg.toml");
@@ -1187,10 +770,9 @@ enabled = true
         assert!(config.ocr.is_some());
         assert!(config.chunking.is_some());
         assert!(config.images.is_some());
+        assert!(config.pdf_options.is_some());
         assert!(config.token_reduction.is_some());
         assert!(config.language_detection.is_some());
-        #[cfg(feature = "pdf")]
-        assert!(config.pdf_options.is_some());
     }
 
     #[test]
@@ -1256,7 +838,6 @@ enabled = true
     }
 
     #[test]
-    #[cfg(feature = "pdf")]
     fn test_pdf_config_defaults() {
         let dir = tempdir().unwrap();
         let config_path = dir.path().join("kreuzberg.toml");
@@ -1448,467 +1029,4 @@ language = "eng"
         assert!(tess.textord_space_size_is_variable);
         assert!(!tess.thresholding_method);
     }
-
-    #[test]
-    fn test_config_cache_hit_same_mtime() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-
-        assert_eq!(config1.use_cache, config2.use_cache);
-        assert_eq!(config1.enable_quality_processing, config2.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_config_cache_invalidation_on_mtime_change() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(!config1.use_cache);
-
-        std::thread::sleep(std::time::Duration::from_secs(1));
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = false
-        "#,
-        )
-        .unwrap();
-
-        let config2 = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config2.use_cache);
-        assert!(!config2.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_config_cache_works_with_json() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.json");
-
-        fs::write(
-            &config_path,
-            r#"{"use_cache": true, "enable_quality_processing": false}"#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_json_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_json_file(&config_path).unwrap();
-
-        assert!(config1.use_cache);
-        assert!(!config1.enable_quality_processing);
-        assert_eq!(config1.use_cache, config2.use_cache);
-    }
-
-    #[test]
-    fn test_config_cache_works_with_yaml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.yaml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache: true
-enable_quality_processing: false
-        "#,
-        )
-        .unwrap();
-
-        let config1 = ExtractionConfig::from_yaml_file(&config_path).unwrap();
-        let config2 = ExtractionConfig::from_yaml_file(&config_path).unwrap();
-
-        assert!(config1.use_cache);
-        assert!(!config1.enable_quality_processing);
-        assert_eq!(config1.use_cache, config2.use_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_ocr_language() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "fra");
-        let mut config = ExtractionConfig::default();
-        assert!(config.ocr.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().language, "fra");
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_ocr_backend() {
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-
-        set_env("KREUZBERG_OCR_BACKEND", "easyocr");
-        let mut config = ExtractionConfig::default();
-        assert!(config.ocr.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().backend, "easyocr");
-
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_ocr_language() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "invalid_lang");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid language code"));
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_ocr_backend() {
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-
-        set_env("KREUZBERG_OCR_BACKEND", "invalid_backend");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid OCR backend"));
-
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_chars() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "2000");
-        let mut config = ExtractionConfig::default();
-        assert!(config.chunking.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_chars, 2000);
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_overlap() {
-        let original_overlap = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_OVERLAP", "300");
-        let mut config = ExtractionConfig::default();
-        assert!(config.chunking.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_overlap, 300);
-
-        restore_env("KREUZBERG_CHUNKING_MAX_OVERLAP", original_overlap);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_chunking_max_chars() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "not_a_number");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("KREUZBERG_CHUNKING_MAX_CHARS"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_max_chars_zero() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "0");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("greater than 0"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_overlap_validation() {
-        let original_overlap = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_OVERLAP", "1500");
-        // Set chunking with max_chars = 1000
-        let mut config = ExtractionConfig {
-            chunking: Some(ChunkingConfig {
-                max_chars: 1000,
-                max_overlap: 200,
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("overlap"));
-
-        restore_env("KREUZBERG_CHUNKING_MAX_OVERLAP", original_overlap);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_true() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "true");
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_false() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "false");
-        let mut config = ExtractionConfig {
-            use_cache: true,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_cache_enabled_case_insensitive() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "TRUE");
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-        assert!(config.use_cache);
-
-        set_env("KREUZBERG_CACHE_ENABLED", "FaLsE");
-        config.use_cache = true;
-        config.apply_env_overrides().unwrap();
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_cache_enabled() {
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_CACHE_ENABLED", "maybe");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("KREUZBERG_CACHE_ENABLED"));
-        assert!(err_msg.contains("'true' or 'false'"));
-
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_token_reduction_mode() {
-        let original_mode = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE").ok();
-
-        set_env("KREUZBERG_TOKEN_REDUCTION_MODE", "moderate");
-        let mut config = ExtractionConfig::default();
-        assert!(config.token_reduction.is_none());
-
-        config.apply_env_overrides().unwrap();
-
-        assert!(config.token_reduction.is_some());
-        assert_eq!(config.token_reduction.unwrap().mode, "moderate");
-
-        restore_env("KREUZBERG_TOKEN_REDUCTION_MODE", original_mode);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_invalid_token_reduction_mode() {
-        let original_mode = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE").ok();
-
-        set_env("KREUZBERG_TOKEN_REDUCTION_MODE", "extreme");
-        let mut config = ExtractionConfig::default();
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        let err_msg = result.unwrap_err().to_string();
-        assert!(err_msg.contains("Invalid token reduction level"));
-
-        restore_env("KREUZBERG_TOKEN_REDUCTION_MODE", original_mode);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_multiple_overrides() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-        let original_backend = std::env::var("KREUZBERG_OCR_BACKEND").ok();
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-        let original_cache = std::env::var("KREUZBERG_CACHE_ENABLED").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "deu");
-        set_env("KREUZBERG_OCR_BACKEND", "paddleocr");
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "3000");
-        set_env("KREUZBERG_CACHE_ENABLED", "false");
-
-        let mut config = ExtractionConfig::default();
-        config.apply_env_overrides().unwrap();
-
-        let ocr = config.ocr.as_ref().unwrap();
-        assert_eq!(ocr.language, "deu");
-        assert_eq!(ocr.backend, "paddleocr");
-        let chunking = config.chunking.as_ref().unwrap();
-        assert_eq!(chunking.max_chars, 3000);
-        assert!(!config.use_cache);
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-        restore_env("KREUZBERG_OCR_BACKEND", original_backend);
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-        restore_env("KREUZBERG_CACHE_ENABLED", original_cache);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_preserves_existing_ocr_config() {
-        let original_lang = std::env::var("KREUZBERG_OCR_LANGUAGE").ok();
-
-        set_env("KREUZBERG_OCR_LANGUAGE", "fra");
-
-        let mut config = ExtractionConfig {
-            ocr: Some(OcrConfig {
-                backend: "tesseract".to_string(),
-                language: "eng".to_string(),
-                tesseract_config: None,
-            }),
-            ..Default::default()
-        };
-
-        config.apply_env_overrides().unwrap();
-
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.language, "fra");
-        assert_eq!(ocr.backend, "tesseract"); // Not overridden
-
-        restore_env("KREUZBERG_OCR_LANGUAGE", original_lang);
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_no_env_vars() {
-        // Ensure no env vars are set
-        remove_env("KREUZBERG_OCR_LANGUAGE");
-        remove_env("KREUZBERG_OCR_BACKEND");
-        remove_env("KREUZBERG_CHUNKING_MAX_CHARS");
-        remove_env("KREUZBERG_CHUNKING_MAX_OVERLAP");
-        remove_env("KREUZBERG_CACHE_ENABLED");
-        remove_env("KREUZBERG_TOKEN_REDUCTION_MODE");
-
-        let mut config = ExtractionConfig {
-            use_cache: false,
-            ocr: Some(OcrConfig {
-                backend: "tesseract".to_string(),
-                language: "eng".to_string(),
-                tesseract_config: None,
-            }),
-            ..Default::default()
-        };
-
-        // Should not change anything
-        config.apply_env_overrides().unwrap();
-
-        assert!(!config.use_cache);
-        assert_eq!(config.ocr.unwrap().language, "eng");
-    }
-
-    #[test]
-    #[serial_test::serial]
-    fn test_apply_env_overrides_chunking_params_validation() {
-        let original_max_chars = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS").ok();
-
-        set_env("KREUZBERG_CHUNKING_MAX_CHARS", "500");
-        let mut config = ExtractionConfig {
-            chunking: Some(ChunkingConfig {
-                max_chars: 1000,
-                max_overlap: 600, // Valid: < 1000
-                embedding: None,
-                preset: None,
-            }),
-            ..Default::default()
-        };
-
-        let result = config.apply_env_overrides();
-        assert!(result.is_err());
-        // After override, max_chars = 500, max_overlap = 600 (invalid: >= 500)
-
-        restore_env("KREUZBERG_CHUNKING_MAX_CHARS", original_max_chars);
-    }
 }