kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14

data/vendor/kreuzberg/src/core/config.rs

@@ -1,1080 +1,1080 @@
-//! Configuration loading and management.
-//!
-//! This module provides utilities for loading extraction configuration from various
-//! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
-
-use crate::{KreuzbergError, Result};
-use serde::{Deserialize, Serialize};
-use std::path::Path;
-
-/// Page extraction and tracking configuration.
-///
-/// Controls how pages are extracted, tracked, and represented in the extraction results.
-/// When `None`, page tracking is disabled.
-///
-/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
-/// when page boundaries are available and chunking is configured.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(default)]
-pub struct PageConfig {
-    /// Extract pages as separate array (ExtractionResult.pages)
-    #[serde(default)]
-    pub extract_pages: bool,
-
-    /// Insert page markers in main content string
-    #[serde(default)]
-    pub insert_page_markers: bool,
-
-    /// Page marker format (use {page_num} placeholder)
-    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
-    #[serde(default = "default_page_marker_format")]
-    pub marker_format: String,
-}
-
-impl Default for PageConfig {
-    fn default() -> Self {
-        Self {
-            extract_pages: false,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }
-    }
-}
-
-/// Main extraction configuration.
-///
-/// This struct contains all configuration options for the extraction process.
-/// It can be loaded from TOML, YAML, or JSON files, or created programmatically.
-///
-/// # Example
-///
-/// ```rust
-/// use kreuzberg::core::config::ExtractionConfig;
-///
-/// // Create with defaults
-/// let config = ExtractionConfig::default();
-///
-/// // Load from TOML file
-/// // let config = ExtractionConfig::from_toml_file("kreuzberg.toml")?;
-/// ```
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ExtractionConfig {
-    /// Enable caching of extraction results
-    #[serde(default = "default_true")]
-    pub use_cache: bool,
-
-    /// Enable quality post-processing
-    #[serde(default = "default_true")]
-    pub enable_quality_processing: bool,
-
-    /// OCR configuration (None = OCR disabled)
-    #[serde(default)]
-    pub ocr: Option<OcrConfig>,
-
-    /// Force OCR even for searchable PDFs
-    #[serde(default)]
-    pub force_ocr: bool,
-
-    /// Text chunking configuration (None = chunking disabled)
-    #[serde(default)]
-    pub chunking: Option<ChunkingConfig>,
-
-    /// Image extraction configuration (None = no image extraction)
-    #[serde(default)]
-    pub images: Option<ImageExtractionConfig>,
-
-    /// PDF-specific options (None = use defaults)
-    #[cfg(feature = "pdf")]
-    #[serde(default)]
-    pub pdf_options: Option<PdfConfig>,
-
-    /// Token reduction configuration (None = no token reduction)
-    #[serde(default)]
-    pub token_reduction: Option<TokenReductionConfig>,
-
-    /// Language detection configuration (None = no language detection)
-    #[serde(default)]
-    pub language_detection: Option<LanguageDetectionConfig>,
-
-    /// Page extraction configuration (None = no page tracking)
-    #[serde(default)]
-    pub pages: Option<PageConfig>,
-
-    /// Keyword extraction configuration (None = no keyword extraction)
-    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-    #[serde(default)]
-    pub keywords: Option<crate::keywords::KeywordConfig>,
-
-    /// Post-processor configuration (None = use defaults)
-    #[serde(default)]
-    pub postprocessor: Option<PostProcessorConfig>,
-
-    /// HTML conversion options (None = use defaults)
-    ///
-    /// Note: This field cannot be deserialized from TOML/YAML/JSON files.
-    /// Set it programmatically after loading config.
-    #[cfg(feature = "html")]
-    #[serde(skip)]
-    pub html_options: Option<html_to_markdown_rs::ConversionOptions>,
-
-    /// Maximum concurrent extractions in batch operations (None = num_cpus * 2).
-    ///
-    /// Limits parallelism to prevent resource exhaustion when processing
-    /// large batches. Defaults to twice the number of CPU cores.
-    #[serde(default)]
-    pub max_concurrent_extractions: Option<usize>,
-}
-
-/// Post-processor configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PostProcessorConfig {
-    /// Enable post-processors
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Whitelist of processor names to run (None = all enabled)
-    #[serde(default)]
-    pub enabled_processors: Option<Vec<String>>,
-
-    /// Blacklist of processor names to skip (None = none disabled)
-    #[serde(default)]
-    pub disabled_processors: Option<Vec<String>>,
-}
-
-/// OCR configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct OcrConfig {
-    /// OCR backend: tesseract, easyocr, paddleocr
-    #[serde(default = "default_tesseract_backend")]
-    pub backend: String,
-
-    /// Language code (e.g., "eng", "deu")
-    #[serde(default = "default_eng")]
-    pub language: String,
-
-    /// Tesseract-specific configuration (optional)
-    #[serde(default)]
-    pub tesseract_config: Option<crate::types::TesseractConfig>,
-}
-
-/// Chunking configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ChunkingConfig {
-    /// Maximum characters per chunk
-    #[serde(default = "default_chunk_size")]
-    pub max_chars: usize,
-
-    /// Overlap between chunks in characters
-    #[serde(default = "default_chunk_overlap")]
-    pub max_overlap: usize,
-
-    /// Optional embedding configuration for chunk embeddings
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub embedding: Option<EmbeddingConfig>,
-
-    /// Use a preset configuration (overrides individual settings if provided)
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub preset: Option<String>,
-}
-
-/// Embedding configuration for text chunks.
-///
-/// Configures embedding generation using ONNX models via fastembed-rs.
-/// Requires the `embeddings` feature to be enabled.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EmbeddingConfig {
-    /// The embedding model to use
-    pub model: EmbeddingModelType,
-
-    /// Whether to normalize embedding vectors (recommended for cosine similarity)
-    #[serde(default = "default_normalize")]
-    pub normalize: bool,
-
-    /// Batch size for embedding generation
-    #[serde(default = "default_batch_size")]
-    pub batch_size: usize,
-
-    /// Show model download progress
-    #[serde(default)]
-    pub show_download_progress: bool,
-
-    /// Custom cache directory for model files
-    ///
-    /// Defaults to `~/.cache/kreuzberg/embeddings/` if not specified.
-    /// Allows full customization of model download location.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub cache_dir: Option<std::path::PathBuf>,
-}
-
-impl Default for EmbeddingConfig {
-    fn default() -> Self {
-        Self {
-            model: EmbeddingModelType::Preset {
-                name: "balanced".to_string(),
-            },
-            normalize: true,
-            batch_size: 32,
-            show_download_progress: false,
-            cache_dir: None,
-        }
-    }
-}
-
-/// Embedding model types supported by Kreuzberg.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum EmbeddingModelType {
-    /// Use a preset model configuration (recommended)
-    Preset { name: String },
-
-    /// Use a specific fastembed model by name
-    #[cfg(feature = "embeddings")]
-    FastEmbed { model: String, dimensions: usize },
-
-    /// Use a custom ONNX model from HuggingFace
-    Custom { model_id: String, dimensions: usize },
-}
-
-/// Image extraction configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ImageExtractionConfig {
-    /// Extract images from documents
-    #[serde(default = "default_true")]
-    pub extract_images: bool,
-
-    /// Target DPI for image normalization
-    #[serde(default = "default_target_dpi")]
-    pub target_dpi: i32,
-
-    /// Maximum dimension for images (width or height)
-    #[serde(default = "default_max_dimension")]
-    pub max_image_dimension: i32,
-
-    /// Automatically adjust DPI based on image content
-    #[serde(default = "default_true")]
-    pub auto_adjust_dpi: bool,
-
-    /// Minimum DPI threshold
-    #[serde(default = "default_min_dpi")]
-    pub min_dpi: i32,
-
-    /// Maximum DPI threshold
-    #[serde(default = "default_max_dpi")]
-    pub max_dpi: i32,
-}
-
-/// PDF-specific configuration.
-#[cfg(feature = "pdf")]
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct PdfConfig {
-    /// Extract images from PDF
-    #[serde(default)]
-    pub extract_images: bool,
-
-    /// List of passwords to try when opening encrypted PDFs
-    #[serde(default)]
-    pub passwords: Option<Vec<String>>,
-
-    /// Extract PDF metadata
-    #[serde(default = "default_true")]
-    pub extract_metadata: bool,
-}
-
-/// Token reduction configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TokenReductionConfig {
-    /// Reduction mode: "off", "light", "moderate", "aggressive", "maximum"
-    #[serde(default = "default_reduction_mode")]
-    pub mode: String,
-
-    /// Preserve important words (capitalized, technical terms)
-    #[serde(default = "default_true")]
-    pub preserve_important_words: bool,
-}
-
-/// Language detection configuration.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LanguageDetectionConfig {
-    /// Enable language detection
-    #[serde(default = "default_true")]
-    pub enabled: bool,
-
-    /// Minimum confidence threshold (0.0-1.0)
-    #[serde(default = "default_confidence")]
-    pub min_confidence: f64,
-
-    /// Detect multiple languages in the document
-    #[serde(default)]
-    pub detect_multiple: bool,
-}
-
-fn default_true() -> bool {
-    true
-}
-fn default_eng() -> String {
-    "eng".to_string()
-}
-fn default_tesseract_backend() -> String {
-    "tesseract".to_string()
-}
-fn default_page_marker_format() -> String {
-    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
-}
-fn default_chunk_size() -> usize {
-    1000
-}
-fn default_chunk_overlap() -> usize {
-    200
-}
-fn default_normalize() -> bool {
-    true
-}
-fn default_batch_size() -> usize {
-    32
-}
-fn default_target_dpi() -> i32 {
-    300
-}
-fn default_max_dimension() -> i32 {
-    4096
-}
-fn default_min_dpi() -> i32 {
-    72
-}
-fn default_max_dpi() -> i32 {
-    600
-}
-fn default_reduction_mode() -> String {
-    "off".to_string()
-}
-fn default_confidence() -> f64 {
-    0.8
-}
-
-impl Default for ExtractionConfig {
-    fn default() -> Self {
-        Self {
-            use_cache: true,
-            enable_quality_processing: true,
-            ocr: None,
-            force_ocr: false,
-            chunking: None,
-            images: None,
-            #[cfg(feature = "pdf")]
-            pdf_options: None,
-            token_reduction: None,
-            language_detection: None,
-            pages: None,
-            #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
-            keywords: None,
-            postprocessor: None,
-            #[cfg(feature = "html")]
-            html_options: None,
-            max_concurrent_extractions: None,
-        }
-    }
-}
-
-impl Default for PostProcessorConfig {
-    fn default() -> Self {
-        Self {
-            enabled: true,
-            enabled_processors: None,
-            disabled_processors: None,
-        }
-    }
-}
-
-impl ExtractionConfig {
-    /// Load configuration from a TOML file.
-    ///
-    /// # Arguments
-    ///
-    /// * `path` - Path to the TOML file
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if file doesn't exist or is invalid TOML.
-    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
-            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
-        })?;
-
-        toml::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.as_ref().display(), e)))
-    }
-
-    /// Load configuration from a YAML file.
-    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
-        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
-            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
-        })?;
-
-        serde_yaml_ng::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.as_ref().display(), e)))
-    }
-
-    /// Load configuration from a JSON file.
-    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
-        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
-            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
-        })?;
-
-        serde_json::from_str(&content)
-            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.as_ref().display(), e)))
-    }
-
-    /// Load configuration from a file, auto-detecting format by extension.
-    ///
-    /// Supported formats:
-    /// - `.toml` - TOML format
-    /// - `.yaml` - YAML format
-    /// - `.json` - JSON format
-    ///
-    /// # Arguments
-    ///
-    /// * `path` - Path to the configuration file
-    ///
-    /// # Errors
-    ///
-    /// Returns `KreuzbergError::Validation` if:
-    /// - File doesn't exist
-    /// - File extension is not supported
-    /// - File content is invalid for the detected format
-    ///
-    /// # Example
-    ///
-    /// ```rust
-    /// use kreuzberg::core::config::ExtractionConfig;
-    ///
-    /// // Auto-detects TOML format
-    /// // let config = ExtractionConfig::from_file("kreuzberg.toml")?;
-    ///
-    /// // Auto-detects YAML format
-    /// // let config = ExtractionConfig::from_file("kreuzberg.yaml")?;
-    /// ```
-    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref();
-        let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
-            KreuzbergError::validation(format!(
-                "Cannot determine file format: no extension found in {}",
-                path.display()
-            ))
-        })?;
-
-        match extension.to_lowercase().as_str() {
-            "toml" => Self::from_toml_file(path),
-            "yaml" | "yml" => Self::from_yaml_file(path),
-            "json" => Self::from_json_file(path),
-            _ => Err(KreuzbergError::validation(format!(
-                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
-                extension
-            ))),
-        }
-    }
-
-    /// Discover configuration file in parent directories.
-    ///
-    /// Searches for `kreuzberg.toml` in current directory and parent directories.
-    ///
-    /// # Returns
-    ///
-    /// - `Some(config)` if found
-    /// - `None` if no config file found
-    pub fn discover() -> Result<Option<Self>> {
-        let mut current = std::env::current_dir().map_err(KreuzbergError::Io)?;
-
-        loop {
-            let kreuzberg_toml = current.join("kreuzberg.toml");
-            if kreuzberg_toml.exists() {
-                return Ok(Some(Self::from_toml_file(kreuzberg_toml)?));
-            }
-
-            if let Some(parent) = current.parent() {
-                current = parent.to_path_buf();
-            } else {
-                break;
-            }
-        }
-
-        Ok(None)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs;
-    use tempfile::tempdir;
-
-    #[test]
-    fn test_default_config() {
-        let config = ExtractionConfig::default();
-        assert!(config.use_cache);
-        assert!(config.enable_quality_processing);
-        assert!(config.ocr.is_none());
-    }
-
-    #[test]
-    fn test_from_toml_file() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(!config.use_cache);
-        assert!(config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_auto_detects_toml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_file(&config_path).unwrap();
-        assert!(!config.use_cache);
-        assert!(config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_auto_detects_yaml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.yaml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache: true
-enable_quality_processing: false
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(!config.enable_quality_processing);
-    }
-
-    #[test]
-    fn test_from_file_unsupported_extension() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.txt");
-
-        fs::write(&config_path, "use_cache = false").unwrap();
-
-        let result = ExtractionConfig::from_file(&config_path);
-        assert!(result.is_err());
-        let err = result.unwrap_err();
-        assert!(err.to_string().contains("Unsupported config file format"));
-    }
-
-    #[test]
-    fn test_from_file_no_extension() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg");
-
-        fs::write(&config_path, "use_cache = false").unwrap();
-
-        let result = ExtractionConfig::from_file(&config_path);
-        assert!(result.is_err());
-        let err = result.unwrap_err();
-        assert!(err.to_string().contains("no extension found"));
-    }
-
-    #[test]
-    fn test_discover_kreuzberg_toml() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = false
-enable_quality_processing = true
-        "#,
-        )
-        .unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&dir).unwrap();
-
-        let result = std::panic::catch_unwind(|| {
-            let config = ExtractionConfig::discover().unwrap();
-            assert!(config.is_some());
-            assert!(!config.unwrap().use_cache);
-        });
-
-        std::env::set_current_dir(&original_dir).unwrap();
-
-        if let Err(e) = result {
-            std::panic::resume_unwind(e);
-        }
-    }
-
-    #[test]
-    fn test_v4_config_with_ocr_and_chunking() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = false
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[chunking]
-max_chars = 2000
-max_overlap = 300
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(!config.enable_quality_processing);
-        assert!(config.ocr.is_some());
-        assert_eq!(config.ocr.unwrap().backend, "tesseract");
-        assert!(config.chunking.is_some());
-        assert_eq!(config.chunking.unwrap().max_chars, 2000);
-    }
-
-    #[test]
-    fn test_config_with_image_extraction() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[images]
-extract_images = true
-target_dpi = 300
-max_image_dimension = 4096
-auto_adjust_dpi = true
-min_dpi = 72
-max_dpi = 600
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.images.is_some());
-        let images = config.images.unwrap();
-        assert!(images.extract_images);
-        assert_eq!(images.target_dpi, 300);
-        assert_eq!(images.max_image_dimension, 4096);
-        assert!(images.auto_adjust_dpi);
-        assert_eq!(images.min_dpi, 72);
-        assert_eq!(images.max_dpi, 600);
-    }
-
-    #[test]
-    #[cfg(feature = "pdf")]
-    fn test_config_with_pdf_options() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[pdf_options]
-extract_images = true
-passwords = ["password1", "password2"]
-extract_metadata = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.pdf_options.is_some());
-        let pdf = config.pdf_options.unwrap();
-        assert!(pdf.extract_images);
-        assert!(pdf.extract_metadata);
-        assert!(pdf.passwords.is_some());
-        let passwords = pdf.passwords.unwrap();
-        assert_eq!(passwords.len(), 2);
-        assert_eq!(passwords[0], "password1");
-        assert_eq!(passwords[1], "password2");
-    }
-
-    #[test]
-    fn test_config_with_token_reduction() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[token_reduction]
-mode = "aggressive"
-preserve_important_words = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.token_reduction.is_some());
-        let token = config.token_reduction.unwrap();
-        assert_eq!(token.mode, "aggressive");
-        assert!(token.preserve_important_words);
-    }
-
-    #[test]
-    fn test_config_with_language_detection() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[language_detection]
-enabled = true
-min_confidence = 0.9
-detect_multiple = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.language_detection.is_some());
-        let lang = config.language_detection.unwrap();
-        assert!(lang.enabled);
-        assert_eq!(lang.min_confidence, 0.9);
-        assert!(lang.detect_multiple);
-    }
-
-    #[test]
-    fn test_config_with_all_optional_fields() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-enable_quality_processing = true
-force_ocr = false
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[chunking]
-max_chars = 1500
-max_overlap = 250
-
-[images]
-extract_images = true
-target_dpi = 300
-
-[pdf_options]
-extract_images = false
-extract_metadata = true
-
-[token_reduction]
-mode = "moderate"
-
-[language_detection]
-enabled = true
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.use_cache);
-        assert!(config.enable_quality_processing);
-        assert!(!config.force_ocr);
-        assert!(config.ocr.is_some());
-        assert!(config.chunking.is_some());
-        assert!(config.images.is_some());
-        assert!(config.token_reduction.is_some());
-        assert!(config.language_detection.is_some());
-        #[cfg(feature = "pdf")]
-        assert!(config.pdf_options.is_some());
-    }
-
-    #[test]
-    fn test_image_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[images]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let images = config.images.unwrap();
-        assert!(images.extract_images);
-        assert_eq!(images.target_dpi, 300);
-        assert_eq!(images.max_image_dimension, 4096);
-        assert!(images.auto_adjust_dpi);
-        assert_eq!(images.min_dpi, 72);
-        assert_eq!(images.max_dpi, 600);
-    }
-
-    #[test]
-    fn test_token_reduction_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[token_reduction]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let token = config.token_reduction.unwrap();
-        assert_eq!(token.mode, "off");
-        assert!(token.preserve_important_words);
-    }
-
-    #[test]
-    fn test_language_detection_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[language_detection]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let lang = config.language_detection.unwrap();
-        assert!(lang.enabled);
-        assert_eq!(lang.min_confidence, 0.8);
-        assert!(!lang.detect_multiple);
-    }
-
-    #[test]
-    #[cfg(feature = "pdf")]
-    fn test_pdf_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[pdf_options]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pdf = config.pdf_options.unwrap();
-        assert!(!pdf.extract_images);
-        assert!(pdf.extract_metadata);
-        assert!(pdf.passwords.is_none());
-    }
-
-    #[test]
-    fn test_config_with_postprocessor() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[postprocessor]
-enabled = true
-enabled_processors = ["entity_extraction", "keyword_extraction"]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.postprocessor.is_some());
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.enabled_processors.is_some());
-        let enabled = pp.enabled_processors.unwrap();
-        assert_eq!(enabled.len(), 2);
-        assert_eq!(enabled[0], "entity_extraction");
-        assert_eq!(enabled[1], "keyword_extraction");
-        assert!(pp.disabled_processors.is_none());
-    }
-
-    #[test]
-    fn test_postprocessor_config_defaults() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.enabled_processors.is_none());
-        assert!(pp.disabled_processors.is_none());
-    }
-
-    #[test]
-    fn test_postprocessor_config_disabled() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-enabled = false
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(!pp.enabled);
-    }
-
-    #[test]
-    fn test_postprocessor_config_blacklist() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[postprocessor]
-disabled_processors = ["category_extraction"]
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        let pp = config.postprocessor.unwrap();
-        assert!(pp.enabled);
-        assert!(pp.disabled_processors.is_some());
-        let disabled = pp.disabled_processors.unwrap();
-        assert_eq!(disabled.len(), 1);
-        assert_eq!(disabled[0], "category_extraction");
-    }
-
-    #[test]
-    fn test_config_with_tesseract_config() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-use_cache = true
-
-[ocr]
-backend = "tesseract"
-language = "eng"
-
-[ocr.tesseract_config]
-psm = 6
-output_format = "text"
-enable_table_detection = false
-tessedit_char_whitelist = "0123456789"
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.ocr.is_some());
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.backend, "tesseract");
-        assert_eq!(ocr.language, "eng");
-        assert!(ocr.tesseract_config.is_some());
-
-        let tess = ocr.tesseract_config.unwrap();
-        assert_eq!(tess.psm, 6);
-        assert_eq!(tess.output_format, "text");
-        assert!(!tess.enable_table_detection);
-        assert_eq!(tess.tessedit_char_whitelist, "0123456789");
-    }
-
-    #[test]
-    fn test_ocr_config_without_tesseract_config() {
-        let dir = tempdir().unwrap();
-        let config_path = dir.path().join("kreuzberg.toml");
-
-        fs::write(
-            &config_path,
-            r#"
-[ocr]
-backend = "easyocr"
-language = "eng"
-        "#,
-        )
-        .unwrap();
-
-        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
-        assert!(config.ocr.is_some());
-        let ocr = config.ocr.unwrap();
-        assert_eq!(ocr.backend, "easyocr");
-        assert_eq!(ocr.language, "eng");
-        assert!(ocr.tesseract_config.is_none());
-    }
-
-    #[test]
-    fn test_tesseract_config_defaults() {
-        let tess = crate::types::TesseractConfig::default();
-        assert_eq!(tess.language, "eng");
-        assert_eq!(tess.psm, 3);
-        assert_eq!(tess.output_format, "markdown");
-        assert!(tess.enable_table_detection);
-        assert_eq!(tess.table_min_confidence, 0.0);
-        assert_eq!(tess.table_column_threshold, 50);
-        assert_eq!(tess.table_row_threshold_ratio, 0.5);
-        assert!(tess.use_cache);
-        assert!(tess.classify_use_pre_adapted_templates);
-        assert!(!tess.language_model_ngram_on);
-        assert!(tess.tessedit_dont_blkrej_good_wds);
-        assert!(tess.tessedit_dont_rowrej_good_wds);
-        assert!(tess.tessedit_enable_dict_correction);
-        assert_eq!(tess.tessedit_char_whitelist, "");
-        assert!(tess.tessedit_use_primary_params_model);
-        assert!(tess.textord_space_size_is_variable);
-        assert!(!tess.thresholding_method);
-    }
-}
+//! Configuration loading and management.
+//!
+//! This module provides utilities for loading extraction configuration from various
+//! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
+
+use crate::{KreuzbergError, Result};
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+
+/// Page extraction and tracking configuration.
+///
+/// Controls how pages are extracted, tracked, and represented in the extraction results.
+/// When `None`, page tracking is disabled.
+///
+/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
+/// when page boundaries are available and chunking is configured.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct PageConfig {
+    /// Extract pages as separate array (ExtractionResult.pages)
+    #[serde(default)]
+    pub extract_pages: bool,
+
+    /// Insert page markers in main content string
+    #[serde(default)]
+    pub insert_page_markers: bool,
+
+    /// Page marker format (use {page_num} placeholder)
+    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
+    #[serde(default = "default_page_marker_format")]
+    pub marker_format: String,
+}
+
+impl Default for PageConfig {
+    fn default() -> Self {
+        Self {
+            extract_pages: false,
+            insert_page_markers: false,
+            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
+        }
+    }
+}
+
+/// Main extraction configuration.
+///
+/// This struct contains all configuration options for the extraction process.
+/// It can be loaded from TOML, YAML, or JSON files, or created programmatically.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// // Create with defaults
+/// let config = ExtractionConfig::default();
+///
+/// // Load from TOML file
+/// // let config = ExtractionConfig::from_toml_file("kreuzberg.toml")?;
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExtractionConfig {
+    /// Enable caching of extraction results
+    #[serde(default = "default_true")]
+    pub use_cache: bool,
+
+    /// Enable quality post-processing
+    #[serde(default = "default_true")]
+    pub enable_quality_processing: bool,
+
+    /// OCR configuration (None = OCR disabled)
+    #[serde(default)]
+    pub ocr: Option<OcrConfig>,
+
+    /// Force OCR even for searchable PDFs
+    #[serde(default)]
+    pub force_ocr: bool,
+
+    /// Text chunking configuration (None = chunking disabled)
+    #[serde(default)]
+    pub chunking: Option<ChunkingConfig>,
+
+    /// Image extraction configuration (None = no image extraction)
+    #[serde(default)]
+    pub images: Option<ImageExtractionConfig>,
+
+    /// PDF-specific options (None = use defaults)
+    #[cfg(feature = "pdf")]
+    #[serde(default)]
+    pub pdf_options: Option<PdfConfig>,
+
+    /// Token reduction configuration (None = no token reduction)
+    #[serde(default)]
+    pub token_reduction: Option<TokenReductionConfig>,
+
+    /// Language detection configuration (None = no language detection)
+    #[serde(default)]
+    pub language_detection: Option<LanguageDetectionConfig>,
+
+    /// Page extraction configuration (None = no page tracking)
+    #[serde(default)]
+    pub pages: Option<PageConfig>,
+
+    /// Keyword extraction configuration (None = no keyword extraction)
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    #[serde(default)]
+    pub keywords: Option<crate::keywords::KeywordConfig>,
+
+    /// Post-processor configuration (None = use defaults)
+    #[serde(default)]
+    pub postprocessor: Option<PostProcessorConfig>,
+
+    /// HTML conversion options (None = use defaults)
+    ///
+    /// Note: This field cannot be deserialized from TOML/YAML/JSON files.
+    /// Set it programmatically after loading config.
+    #[cfg(feature = "html")]
+    #[serde(skip)]
+    pub html_options: Option<html_to_markdown_rs::ConversionOptions>,
+
+    /// Maximum concurrent extractions in batch operations (None = num_cpus * 2).
+    ///
+    /// Limits parallelism to prevent resource exhaustion when processing
+    /// large batches. Defaults to twice the number of CPU cores.
+    #[serde(default)]
+    pub max_concurrent_extractions: Option<usize>,
+}
+
+/// Post-processor configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PostProcessorConfig {
+    /// Enable post-processors
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Whitelist of processor names to run (None = all enabled)
+    #[serde(default)]
+    pub enabled_processors: Option<Vec<String>>,
+
+    /// Blacklist of processor names to skip (None = none disabled)
+    #[serde(default)]
+    pub disabled_processors: Option<Vec<String>>,
+}
+
+/// OCR configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrConfig {
+    /// OCR backend: tesseract, easyocr, paddleocr
+    #[serde(default = "default_tesseract_backend")]
+    pub backend: String,
+
+    /// Language code (e.g., "eng", "deu")
+    #[serde(default = "default_eng")]
+    pub language: String,
+
+    /// Tesseract-specific configuration (optional)
+    #[serde(default)]
+    pub tesseract_config: Option<crate::types::TesseractConfig>,
+}
+
+/// Chunking configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChunkingConfig {
+    /// Maximum characters per chunk
+    #[serde(default = "default_chunk_size")]
+    pub max_chars: usize,
+
+    /// Overlap between chunks in characters
+    #[serde(default = "default_chunk_overlap")]
+    pub max_overlap: usize,
+
+    /// Optional embedding configuration for chunk embeddings
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub embedding: Option<EmbeddingConfig>,
+
+    /// Use a preset configuration (overrides individual settings if provided)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub preset: Option<String>,
+}
+
+/// Embedding configuration for text chunks.
+///
+/// Configures embedding generation using ONNX models via fastembed-rs.
+/// Requires the `embeddings` feature to be enabled.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EmbeddingConfig {
+    /// The embedding model to use
+    pub model: EmbeddingModelType,
+
+    /// Whether to normalize embedding vectors (recommended for cosine similarity)
+    #[serde(default = "default_normalize")]
+    pub normalize: bool,
+
+    /// Batch size for embedding generation
+    #[serde(default = "default_batch_size")]
+    pub batch_size: usize,
+
+    /// Show model download progress
+    #[serde(default)]
+    pub show_download_progress: bool,
+
+    /// Custom cache directory for model files
+    ///
+    /// Defaults to `~/.cache/kreuzberg/embeddings/` if not specified.
+    /// Allows full customization of model download location.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_dir: Option<std::path::PathBuf>,
+}
+
+impl Default for EmbeddingConfig {
+    fn default() -> Self {
+        Self {
+            model: EmbeddingModelType::Preset {
+                name: "balanced".to_string(),
+            },
+            normalize: true,
+            batch_size: 32,
+            show_download_progress: false,
+            cache_dir: None,
+        }
+    }
+}
+
+/// Embedding model types supported by Kreuzberg.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum EmbeddingModelType {
+    /// Use a preset model configuration (recommended)
+    Preset { name: String },
+
+    /// Use a specific fastembed model by name
+    #[cfg(feature = "embeddings")]
+    FastEmbed { model: String, dimensions: usize },
+
+    /// Use a custom ONNX model from HuggingFace
+    Custom { model_id: String, dimensions: usize },
+}
+
+/// Image extraction configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ImageExtractionConfig {
+    /// Extract images from documents
+    #[serde(default = "default_true")]
+    pub extract_images: bool,
+
+    /// Target DPI for image normalization
+    #[serde(default = "default_target_dpi")]
+    pub target_dpi: i32,
+
+    /// Maximum dimension for images (width or height)
+    #[serde(default = "default_max_dimension")]
+    pub max_image_dimension: i32,
+
+    /// Automatically adjust DPI based on image content
+    #[serde(default = "default_true")]
+    pub auto_adjust_dpi: bool,
+
+    /// Minimum DPI threshold
+    #[serde(default = "default_min_dpi")]
+    pub min_dpi: i32,
+
+    /// Maximum DPI threshold
+    #[serde(default = "default_max_dpi")]
+    pub max_dpi: i32,
+}
+
+/// PDF-specific configuration.
+#[cfg(feature = "pdf")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PdfConfig {
+    /// Extract images from PDF
+    #[serde(default)]
+    pub extract_images: bool,
+
+    /// List of passwords to try when opening encrypted PDFs
+    #[serde(default)]
+    pub passwords: Option<Vec<String>>,
+
+    /// Extract PDF metadata
+    #[serde(default = "default_true")]
+    pub extract_metadata: bool,
+}
+
+/// Token reduction configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TokenReductionConfig {
+    /// Reduction mode: "off", "light", "moderate", "aggressive", "maximum"
+    #[serde(default = "default_reduction_mode")]
+    pub mode: String,
+
+    /// Preserve important words (capitalized, technical terms)
+    #[serde(default = "default_true")]
+    pub preserve_important_words: bool,
+}
+
+/// Language detection configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LanguageDetectionConfig {
+    /// Enable language detection
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Minimum confidence threshold (0.0-1.0)
+    #[serde(default = "default_confidence")]
+    pub min_confidence: f64,
+
+    /// Detect multiple languages in the document
+    #[serde(default)]
+    pub detect_multiple: bool,
+}
+
+fn default_true() -> bool {
+    true
+}
+fn default_eng() -> String {
+    "eng".to_string()
+}
+fn default_tesseract_backend() -> String {
+    "tesseract".to_string()
+}
+fn default_page_marker_format() -> String {
+    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
+}
+fn default_chunk_size() -> usize {
+    1000
+}
+fn default_chunk_overlap() -> usize {
+    200
+}
+fn default_normalize() -> bool {
+    true
+}
+fn default_batch_size() -> usize {
+    32
+}
+fn default_target_dpi() -> i32 {
+    300
+}
+fn default_max_dimension() -> i32 {
+    4096
+}
+fn default_min_dpi() -> i32 {
+    72
+}
+fn default_max_dpi() -> i32 {
+    600
+}
+fn default_reduction_mode() -> String {
+    "off".to_string()
+}
+fn default_confidence() -> f64 {
+    0.8
+}
+
+impl Default for ExtractionConfig {
+    fn default() -> Self {
+        Self {
+            use_cache: true,
+            enable_quality_processing: true,
+            ocr: None,
+            force_ocr: false,
+            chunking: None,
+            images: None,
+            #[cfg(feature = "pdf")]
+            pdf_options: None,
+            token_reduction: None,
+            language_detection: None,
+            pages: None,
+            #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+            keywords: None,
+            postprocessor: None,
+            #[cfg(feature = "html")]
+            html_options: None,
+            max_concurrent_extractions: None,
+        }
+    }
+}
+
+impl Default for PostProcessorConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            enabled_processors: None,
+            disabled_processors: None,
+        }
+    }
+}
+
+impl ExtractionConfig {
+    /// Load configuration from a TOML file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the TOML file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if file doesn't exist or is invalid TOML.
+    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
+        })?;
+
+        toml::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.as_ref().display(), e)))
+    }
+
+    /// Load configuration from a YAML file.
+    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
+        })?;
+
+        serde_yaml_ng::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.as_ref().display(), e)))
+    }
+
+    /// Load configuration from a JSON file.
+    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
+        let content = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+            KreuzbergError::validation(format!("Failed to read config file {}: {}", path.as_ref().display(), e))
+        })?;
+
+        serde_json::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.as_ref().display(), e)))
+    }
+
+    /// Load configuration from a file, auto-detecting format by extension.
+    ///
+    /// Supported formats:
+    /// - `.toml` - TOML format
+    /// - `.yaml` - YAML format
+    /// - `.json` - JSON format
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the configuration file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - File doesn't exist
+    /// - File extension is not supported
+    /// - File content is invalid for the detected format
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::config::ExtractionConfig;
+    ///
+    /// // Auto-detects TOML format
+    /// // let config = ExtractionConfig::from_file("kreuzberg.toml")?;
+    ///
+    /// // Auto-detects YAML format
+    /// // let config = ExtractionConfig::from_file("kreuzberg.yaml")?;
+    /// ```
+    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref();
+        let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
+            KreuzbergError::validation(format!(
+                "Cannot determine file format: no extension found in {}",
+                path.display()
+            ))
+        })?;
+
+        match extension.to_lowercase().as_str() {
+            "toml" => Self::from_toml_file(path),
+            "yaml" | "yml" => Self::from_yaml_file(path),
+            "json" => Self::from_json_file(path),
+            _ => Err(KreuzbergError::validation(format!(
+                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
+                extension
+            ))),
+        }
+    }
+
+    /// Discover configuration file in parent directories.
+    ///
+    /// Searches for `kreuzberg.toml` in current directory and parent directories.
+    ///
+    /// # Returns
+    ///
+    /// - `Some(config)` if found
+    /// - `None` if no config file found
+    pub fn discover() -> Result<Option<Self>> {
+        let mut current = std::env::current_dir().map_err(KreuzbergError::Io)?;
+
+        loop {
+            let kreuzberg_toml = current.join("kreuzberg.toml");
+            if kreuzberg_toml.exists() {
+                return Ok(Some(Self::from_toml_file(kreuzberg_toml)?));
+            }
+
+            if let Some(parent) = current.parent() {
+                current = parent.to_path_buf();
+            } else {
+                break;
+            }
+        }
+
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_default_config() {
+        let config = ExtractionConfig::default();
+        assert!(config.use_cache);
+        assert!(config.enable_quality_processing);
+        assert!(config.ocr.is_none());
+    }
+
+    #[test]
+    fn test_from_toml_file() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = false
+enable_quality_processing = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(!config.use_cache);
+        assert!(config.enable_quality_processing);
+    }
+
+    #[test]
+    fn test_from_file_auto_detects_toml() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = false
+enable_quality_processing = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_file(&config_path).unwrap();
+        assert!(!config.use_cache);
+        assert!(config.enable_quality_processing);
+    }
+
+    #[test]
+    fn test_from_file_auto_detects_yaml() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.yaml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache: true
+enable_quality_processing: false
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_file(&config_path).unwrap();
+        assert!(config.use_cache);
+        assert!(!config.enable_quality_processing);
+    }
+
+    #[test]
+    fn test_from_file_unsupported_extension() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.txt");
+
+        fs::write(&config_path, "use_cache = false").unwrap();
+
+        let result = ExtractionConfig::from_file(&config_path);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("Unsupported config file format"));
+    }
+
+    #[test]
+    fn test_from_file_no_extension() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg");
+
+        fs::write(&config_path, "use_cache = false").unwrap();
+
+        let result = ExtractionConfig::from_file(&config_path);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("no extension found"));
+    }
+
+    #[test]
+    fn test_discover_kreuzberg_toml() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = false
+enable_quality_processing = true
+        "#,
+        )
+        .unwrap();
+
+        let original_dir = std::env::current_dir().unwrap();
+        std::env::set_current_dir(&dir).unwrap();
+
+        let result = std::panic::catch_unwind(|| {
+            let config = ExtractionConfig::discover().unwrap();
+            assert!(config.is_some());
+            assert!(!config.unwrap().use_cache);
+        });
+
+        std::env::set_current_dir(&original_dir).unwrap();
+
+        if let Err(e) = result {
+            std::panic::resume_unwind(e);
+        }
+    }
+
+    #[test]
+    fn test_v4_config_with_ocr_and_chunking() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+enable_quality_processing = false
+
+[ocr]
+backend = "tesseract"
+language = "eng"
+
+[chunking]
+max_chars = 2000
+max_overlap = 300
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.use_cache);
+        assert!(!config.enable_quality_processing);
+        assert!(config.ocr.is_some());
+        assert_eq!(config.ocr.unwrap().backend, "tesseract");
+        assert!(config.chunking.is_some());
+        assert_eq!(config.chunking.unwrap().max_chars, 2000);
+    }
+
+    #[test]
+    fn test_config_with_image_extraction() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[images]
+extract_images = true
+target_dpi = 300
+max_image_dimension = 4096
+auto_adjust_dpi = true
+min_dpi = 72
+max_dpi = 600
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.images.is_some());
+        let images = config.images.unwrap();
+        assert!(images.extract_images);
+        assert_eq!(images.target_dpi, 300);
+        assert_eq!(images.max_image_dimension, 4096);
+        assert!(images.auto_adjust_dpi);
+        assert_eq!(images.min_dpi, 72);
+        assert_eq!(images.max_dpi, 600);
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_config_with_pdf_options() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[pdf_options]
+extract_images = true
+passwords = ["password1", "password2"]
+extract_metadata = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.pdf_options.is_some());
+        let pdf = config.pdf_options.unwrap();
+        assert!(pdf.extract_images);
+        assert!(pdf.extract_metadata);
+        assert!(pdf.passwords.is_some());
+        let passwords = pdf.passwords.unwrap();
+        assert_eq!(passwords.len(), 2);
+        assert_eq!(passwords[0], "password1");
+        assert_eq!(passwords[1], "password2");
+    }
+
+    #[test]
+    fn test_config_with_token_reduction() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[token_reduction]
+mode = "aggressive"
+preserve_important_words = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.token_reduction.is_some());
+        let token = config.token_reduction.unwrap();
+        assert_eq!(token.mode, "aggressive");
+        assert!(token.preserve_important_words);
+    }
+
+    #[test]
+    fn test_config_with_language_detection() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[language_detection]
+enabled = true
+min_confidence = 0.9
+detect_multiple = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.language_detection.is_some());
+        let lang = config.language_detection.unwrap();
+        assert!(lang.enabled);
+        assert_eq!(lang.min_confidence, 0.9);
+        assert!(lang.detect_multiple);
+    }
+
+    #[test]
+    fn test_config_with_all_optional_fields() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+enable_quality_processing = true
+force_ocr = false
+
+[ocr]
+backend = "tesseract"
+language = "eng"
+
+[chunking]
+max_chars = 1500
+max_overlap = 250
+
+[images]
+extract_images = true
+target_dpi = 300
+
+[pdf_options]
+extract_images = false
+extract_metadata = true
+
+[token_reduction]
+mode = "moderate"
+
+[language_detection]
+enabled = true
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.use_cache);
+        assert!(config.enable_quality_processing);
+        assert!(!config.force_ocr);
+        assert!(config.ocr.is_some());
+        assert!(config.chunking.is_some());
+        assert!(config.images.is_some());
+        assert!(config.token_reduction.is_some());
+        assert!(config.language_detection.is_some());
+        #[cfg(feature = "pdf")]
+        assert!(config.pdf_options.is_some());
+    }
+
+    #[test]
+    fn test_image_config_defaults() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[images]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let images = config.images.unwrap();
+        assert!(images.extract_images);
+        assert_eq!(images.target_dpi, 300);
+        assert_eq!(images.max_image_dimension, 4096);
+        assert!(images.auto_adjust_dpi);
+        assert_eq!(images.min_dpi, 72);
+        assert_eq!(images.max_dpi, 600);
+    }
+
+    #[test]
+    fn test_token_reduction_defaults() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[token_reduction]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let token = config.token_reduction.unwrap();
+        assert_eq!(token.mode, "off");
+        assert!(token.preserve_important_words);
+    }
+
+    #[test]
+    fn test_language_detection_defaults() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[language_detection]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let lang = config.language_detection.unwrap();
+        assert!(lang.enabled);
+        assert_eq!(lang.min_confidence, 0.8);
+        assert!(!lang.detect_multiple);
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_pdf_config_defaults() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[pdf_options]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let pdf = config.pdf_options.unwrap();
+        assert!(!pdf.extract_images);
+        assert!(pdf.extract_metadata);
+        assert!(pdf.passwords.is_none());
+    }
+
+    #[test]
+    fn test_config_with_postprocessor() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[postprocessor]
+enabled = true
+enabled_processors = ["entity_extraction", "keyword_extraction"]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.postprocessor.is_some());
+        let pp = config.postprocessor.unwrap();
+        assert!(pp.enabled);
+        assert!(pp.enabled_processors.is_some());
+        let enabled = pp.enabled_processors.unwrap();
+        assert_eq!(enabled.len(), 2);
+        assert_eq!(enabled[0], "entity_extraction");
+        assert_eq!(enabled[1], "keyword_extraction");
+        assert!(pp.disabled_processors.is_none());
+    }
+
+    #[test]
+    fn test_postprocessor_config_defaults() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[postprocessor]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let pp = config.postprocessor.unwrap();
+        assert!(pp.enabled);
+        assert!(pp.enabled_processors.is_none());
+        assert!(pp.disabled_processors.is_none());
+    }
+
+    #[test]
+    fn test_postprocessor_config_disabled() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[postprocessor]
+enabled = false
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let pp = config.postprocessor.unwrap();
+        assert!(!pp.enabled);
+    }
+
+    #[test]
+    fn test_postprocessor_config_blacklist() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[postprocessor]
+disabled_processors = ["category_extraction"]
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        let pp = config.postprocessor.unwrap();
+        assert!(pp.enabled);
+        assert!(pp.disabled_processors.is_some());
+        let disabled = pp.disabled_processors.unwrap();
+        assert_eq!(disabled.len(), 1);
+        assert_eq!(disabled[0], "category_extraction");
+    }
+
+    #[test]
+    fn test_config_with_tesseract_config() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+use_cache = true
+
+[ocr]
+backend = "tesseract"
+language = "eng"
+
+[ocr.tesseract_config]
+psm = 6
+output_format = "text"
+enable_table_detection = false
+tessedit_char_whitelist = "0123456789"
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.ocr.is_some());
+        let ocr = config.ocr.unwrap();
+        assert_eq!(ocr.backend, "tesseract");
+        assert_eq!(ocr.language, "eng");
+        assert!(ocr.tesseract_config.is_some());
+
+        let tess = ocr.tesseract_config.unwrap();
+        assert_eq!(tess.psm, 6);
+        assert_eq!(tess.output_format, "text");
+        assert!(!tess.enable_table_detection);
+        assert_eq!(tess.tessedit_char_whitelist, "0123456789");
+    }
+
+    #[test]
+    fn test_ocr_config_without_tesseract_config() {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("kreuzberg.toml");
+
+        fs::write(
+            &config_path,
+            r#"
+[ocr]
+backend = "easyocr"
+language = "eng"
+        "#,
+        )
+        .unwrap();
+
+        let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+        assert!(config.ocr.is_some());
+        let ocr = config.ocr.unwrap();
+        assert_eq!(ocr.backend, "easyocr");
+        assert_eq!(ocr.language, "eng");
+        assert!(ocr.tesseract_config.is_none());
+    }
+
+    #[test]
+    fn test_tesseract_config_defaults() {
+        let tess = crate::types::TesseractConfig::default();
+        assert_eq!(tess.language, "eng");
+        assert_eq!(tess.psm, 3);
+        assert_eq!(tess.output_format, "markdown");
+        assert!(tess.enable_table_detection);
+        assert_eq!(tess.table_min_confidence, 0.0);
+        assert_eq!(tess.table_column_threshold, 50);
+        assert_eq!(tess.table_row_threshold_ratio, 0.5);
+        assert!(tess.use_cache);
+        assert!(tess.classify_use_pre_adapted_templates);
+        assert!(!tess.language_model_ngram_on);
+        assert!(tess.tessedit_dont_blkrej_good_wds);
+        assert!(tess.tessedit_dont_rowrej_good_wds);
+        assert!(tess.tessedit_enable_dict_correction);
+        assert_eq!(tess.tessedit_char_whitelist, "");
+        assert!(tess.tessedit_use_primary_params_model);
+        assert!(tess.textord_space_size_is_variable);
+        assert!(!tess.thresholding_method);
+    }
+}