kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/config_validation/sections.rs

@@ -0,0 +1,401 @@
+//! Per-section validation functions.
+//!
+//! This module contains validation functions for individual configuration sections
+//! and their specific parameters. Each function validates a specific aspect of
+//! the configuration and returns detailed error messages when validation fails.
+
+use crate::{KreuzbergError, Result};
+
+/// Valid binarization methods for image preprocessing.
+const VALID_BINARIZATION_METHODS: &[&str] = &["otsu", "adaptive", "sauvola"];
+
+/// Valid token reduction levels.
+const VALID_TOKEN_REDUCTION_LEVELS: &[&str] = &["off", "light", "moderate", "aggressive", "maximum"];
+
+/// Valid OCR backends.
+const VALID_OCR_BACKENDS: &[&str] = &["tesseract", "easyocr", "paddleocr"];
+
+/// Common ISO 639-1 language codes (extended list).
+/// Covers most major languages and variants used in document processing.
+const VALID_LANGUAGE_CODES: &[&str] = &[
+    "en", "de", "fr", "es", "it", "pt", "nl", "pl", "ru", "zh", "ja", "ko", "bg", "cs", "da", "el", "et", "fi", "hu",
+    "lt", "lv", "ro", "sk", "sl", "sv", "uk", "ar", "hi", "th", "tr", "vi", "eng", "deu", "fra", "spa", "ita", "por",
+    "nld", "pol", "rus", "zho", "jpn", "kor", "ces", "dan", "ell", "est", "fin", "hun", "lit", "lav", "ron", "slk",
+    "slv", "swe", "tur",
+];
+
+/// Valid tesseract PSM (Page Segmentation Mode) values.
+const VALID_TESSERACT_PSM: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
+
+/// Valid tesseract OEM (OCR Engine Mode) values.
+const VALID_TESSERACT_OEM: &[i32] = &[0, 1, 2, 3];
+
+/// Valid output formats for tesseract.
+const VALID_OUTPUT_FORMATS: &[&str] = &["text", "markdown"];
+
+/// Validate a binarization method string.
+///
+/// # Arguments
+///
+/// * `method` - The binarization method to validate (e.g., "otsu", "adaptive", "sauvola")
+///
+/// # Returns
+///
+/// `Ok(())` if the method is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_binarization_method;
+///
+/// assert!(validate_binarization_method("otsu").is_ok());
+/// assert!(validate_binarization_method("adaptive").is_ok());
+/// assert!(validate_binarization_method("invalid").is_err());
+/// ```
+pub fn validate_binarization_method(method: &str) -> Result<()> {
+    let method = method.to_lowercase();
+    if VALID_BINARIZATION_METHODS.contains(&method.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid binarization method '{}'. Valid options are: {}",
+                method,
+                VALID_BINARIZATION_METHODS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a token reduction level string.
+///
+/// # Arguments
+///
+/// * `level` - The token reduction level to validate (e.g., "off", "light", "moderate")
+///
+/// # Returns
+///
+/// `Ok(())` if the level is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_token_reduction_level;
+///
+/// assert!(validate_token_reduction_level("off").is_ok());
+/// assert!(validate_token_reduction_level("moderate").is_ok());
+/// assert!(validate_token_reduction_level("extreme").is_err());
+/// ```
+pub fn validate_token_reduction_level(level: &str) -> Result<()> {
+    let level = level.to_lowercase();
+    if VALID_TOKEN_REDUCTION_LEVELS.contains(&level.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid token reduction level '{}'. Valid options are: {}",
+                level,
+                VALID_TOKEN_REDUCTION_LEVELS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate an OCR backend string.
+///
+/// # Arguments
+///
+/// * `backend` - The OCR backend to validate (e.g., "tesseract", "easyocr", "paddleocr")
+///
+/// # Returns
+///
+/// `Ok(())` if the backend is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_ocr_backend;
+///
+/// assert!(validate_ocr_backend("tesseract").is_ok());
+/// assert!(validate_ocr_backend("easyocr").is_ok());
+/// assert!(validate_ocr_backend("invalid").is_err());
+/// ```
+pub fn validate_ocr_backend(backend: &str) -> Result<()> {
+    let backend = backend.to_lowercase();
+    if VALID_OCR_BACKENDS.contains(&backend.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid OCR backend '{}'. Valid options are: {}",
+                backend,
+                VALID_OCR_BACKENDS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a language code (ISO 639-1 or 639-3 format).
+///
+/// Accepts both 2-letter ISO 639-1 codes (e.g., "en", "de") and
+/// 3-letter ISO 639-3 codes (e.g., "eng", "deu") for broader compatibility.
+///
+/// # Arguments
+///
+/// * `code` - The language code to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the code is valid, or a `ValidationError` indicating an invalid language code.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_language_code;
+///
+/// assert!(validate_language_code("en").is_ok());
+/// assert!(validate_language_code("eng").is_ok());
+/// assert!(validate_language_code("de").is_ok());
+/// assert!(validate_language_code("deu").is_ok());
+/// assert!(validate_language_code("invalid").is_err());
+/// ```
+pub fn validate_language_code(code: &str) -> Result<()> {
+    let code_lower = code.to_lowercase();
+
+    if VALID_LANGUAGE_CODES.contains(&code_lower.as_str()) {
+        return Ok(());
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!(
+            "Invalid language code '{}'. Use ISO 639-1 (2-letter, e.g., 'en', 'de') \
+             or ISO 639-3 (3-letter, e.g., 'eng', 'deu') codes. \
+             Common codes: en, de, fr, es, it, pt, nl, pl, ru, zh, ja, ko, ar, hi, th.",
+            code
+        ),
+        source: None,
+    })
+}
+
+/// Validate a tesseract Page Segmentation Mode (PSM).
+///
+/// # Arguments
+///
+/// * `psm` - The PSM value to validate (0-13)
+///
+/// # Returns
+///
+/// `Ok(())` if the PSM is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_tesseract_psm;
+///
+/// assert!(validate_tesseract_psm(3).is_ok());  // Fully automatic
+/// assert!(validate_tesseract_psm(6).is_ok());  // Single block of text
+/// assert!(validate_tesseract_psm(14).is_err()); // Out of range
+/// ```
+pub fn validate_tesseract_psm(psm: i32) -> Result<()> {
+    if VALID_TESSERACT_PSM.contains(&psm) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid tesseract PSM value '{}'. Valid range is 0-13. \
+                 Common values: 3 (auto), 6 (single block), 11 (sparse text).",
+                psm
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a tesseract OCR Engine Mode (OEM).
+///
+/// # Arguments
+///
+/// * `oem` - The OEM value to validate (0-3)
+///
+/// # Returns
+///
+/// `Ok(())` if the OEM is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_tesseract_oem;
+///
+/// assert!(validate_tesseract_oem(1).is_ok());  // Neural nets (LSTM)
+/// assert!(validate_tesseract_oem(2).is_ok());  // Legacy + LSTM
+/// assert!(validate_tesseract_oem(4).is_err()); // Out of range
+/// ```
+pub fn validate_tesseract_oem(oem: i32) -> Result<()> {
+    if VALID_TESSERACT_OEM.contains(&oem) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid tesseract OEM value '{}'. Valid range is 0-3. \
+                 0=Legacy, 1=LSTM, 2=Legacy+LSTM, 3=Default",
+                oem
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a tesseract output format.
+///
+/// # Arguments
+///
+/// * `format` - The output format to validate (e.g., "text", "markdown")
+///
+/// # Returns
+///
+/// `Ok(())` if the format is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_output_format;
+///
+/// assert!(validate_output_format("text").is_ok());
+/// assert!(validate_output_format("markdown").is_ok());
+/// assert!(validate_output_format("json").is_err());
+/// ```
+pub fn validate_output_format(format: &str) -> Result<()> {
+    let format = format.to_lowercase();
+    if VALID_OUTPUT_FORMATS.contains(&format.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid output format '{}'. Valid options are: {}",
+                format,
+                VALID_OUTPUT_FORMATS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a confidence threshold value.
+///
+/// Confidence thresholds should be between 0.0 and 1.0 inclusive.
+///
+/// # Arguments
+///
+/// * `confidence` - The confidence threshold to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the confidence is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_confidence;
+///
+/// assert!(validate_confidence(0.5).is_ok());
+/// assert!(validate_confidence(0.0).is_ok());
+/// assert!(validate_confidence(1.0).is_ok());
+/// assert!(validate_confidence(1.5).is_err());
+/// assert!(validate_confidence(-0.1).is_err());
+/// ```
+pub fn validate_confidence(confidence: f64) -> Result<()> {
+    if (0.0..=1.0).contains(&confidence) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid confidence threshold '{}'. Must be between 0.0 and 1.0.",
+                confidence
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a DPI (dots per inch) value.
+///
+/// DPI should be a positive integer, typically 72-600.
+///
+/// # Arguments
+///
+/// * `dpi` - The DPI value to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the DPI is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_dpi;
+///
+/// assert!(validate_dpi(96).is_ok());
+/// assert!(validate_dpi(300).is_ok());
+/// assert!(validate_dpi(0).is_err());
+/// assert!(validate_dpi(-1).is_err());
+/// ```
+pub fn validate_dpi(dpi: i32) -> Result<()> {
+    if dpi > 0 && dpi <= 2400 {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid DPI value '{}'. Must be a positive integer, typically 72-600.",
+                dpi
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate chunk size parameters.
+///
+/// Checks that max_chars > 0 and max_overlap < max_chars.
+///
+/// # Arguments
+///
+/// * `max_chars` - The maximum characters per chunk
+/// * `max_overlap` - The maximum overlap between chunks
+///
+/// # Returns
+///
+/// `Ok(())` if the parameters are valid, or a `ValidationError` with details about constraints.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_chunking_params;
+///
+/// assert!(validate_chunking_params(1000, 200).is_ok());
+/// assert!(validate_chunking_params(500, 50).is_ok());
+/// assert!(validate_chunking_params(0, 100).is_err()); // max_chars must be > 0
+/// assert!(validate_chunking_params(100, 150).is_err()); // overlap >= max_chars
+/// ```
+pub fn validate_chunking_params(max_chars: usize, max_overlap: usize) -> Result<()> {
+    if max_chars == 0 {
+        return Err(KreuzbergError::Validation {
+            message: "max_chars must be greater than 0".to_string(),
+            source: None,
+        });
+    }
+
+    if max_overlap >= max_chars {
+        return Err(KreuzbergError::Validation {
+            message: format!(
+                "max_overlap ({}) must be less than max_chars ({})",
+                max_overlap, max_chars
+            ),
+            source: None,
+        });
+    }
+
+    Ok(())
+}

data/vendor/kreuzberg/src/core/extractor/batch.rs

@@ -0,0 +1,246 @@
+//! Batch extraction operations for concurrent processing.
+//!
+//! This module provides parallel extraction capabilities for processing
+//! multiple files or byte arrays concurrently with automatic resource management.
+
+use crate::core::config::ExtractionConfig;
+use crate::types::{ErrorMetadata, ExtractionResult, Metadata};
+use crate::{KreuzbergError, Result};
+use std::path::Path;
+use std::sync::Arc;
+
+use super::bytes::extract_bytes;
+use super::file::extract_file;
+
+/// Extract content from multiple files concurrently.
+///
+/// This function processes multiple files in parallel, automatically managing
+/// concurrency to prevent resource exhaustion. The concurrency limit can be
+/// configured via `ExtractionConfig::max_concurrent_extractions` or defaults
+/// to `num_cpus * 2`.
+///
+/// # Arguments
+///
+/// * `paths` - Vector of file paths to extract
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// A vector of `ExtractionResult` in the same order as the input paths.
+///
+/// # Errors
+///
+/// Individual file errors are captured in the result metadata. System errors
+/// (IO, RuntimeError equivalents) will bubble up and fail the entire batch.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_file;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let paths = vec!["doc1.pdf", "doc2.pdf"];
+/// let results = batch_extract_file(paths, &config).await?;
+/// println!("Processed {} files", results.len());
+/// # Ok(())
+/// # }
+/// ```
+#[cfg(feature = "tokio-runtime")]
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, paths),
+    fields(
+        extraction.batch_size = paths.len(),
+    )
+))]
+pub async fn batch_extract_file(
+    paths: Vec<impl AsRef<Path>>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    use tokio::sync::Semaphore;
+    use tokio::task::JoinSet;
+
+    if paths.is_empty() {
+        return Ok(vec![]);
+    }
+
+    let config = Arc::new(config.clone());
+
+    let max_concurrent = config
+        .max_concurrent_extractions
+        .unwrap_or_else(|| (num_cpus::get() as f64 * 1.5).ceil() as usize);
+    let semaphore = Arc::new(Semaphore::new(max_concurrent));
+
+    let mut tasks = JoinSet::new();
+
+    for (index, path) in paths.into_iter().enumerate() {
+        let path_buf = path.as_ref().to_path_buf();
+        let config_clone = Arc::clone(&config);
+        let semaphore_clone = Arc::clone(&semaphore);
+
+        tasks.spawn(async move {
+            let _permit = semaphore_clone.acquire().await.unwrap();
+            let result =
+                crate::core::batch_mode::with_batch_mode(async { extract_file(&path_buf, None, &config_clone).await })
+                    .await;
+            (index, result)
+        });
+    }
+
+    let mut results: Vec<Option<ExtractionResult>> = vec![None; tasks.len()];
+
+    while let Some(task_result) = tasks.join_next().await {
+        match task_result {
+            Ok((index, Ok(result))) => {
+                results[index] = Some(result);
+            }
+            Ok((index, Err(e))) => {
+                // All errors (including Io) should create error results
+                // instead of causing early return that abandons running tasks
+                let metadata = Metadata {
+                    error: Some(ErrorMetadata {
+                        error_type: format!("{:?}", e),
+                        message: e.to_string(),
+                    }),
+                    ..Default::default()
+                };
+
+                results[index] = Some(ExtractionResult {
+                    content: format!("Error: {}", e),
+                    mime_type: "text/plain".to_string(),
+                    metadata,
+                    tables: vec![],
+                    detected_languages: None,
+                    chunks: None,
+                    images: None,
+                    djot_content: None,
+                    pages: None,
+                    elements: None,
+                });
+            }
+            Err(join_err) => {
+                return Err(KreuzbergError::Other(format!("Task panicked: {}", join_err)));
+            }
+        }
+    }
+
+    #[allow(clippy::unwrap_used)]
+    Ok(results.into_iter().map(|r| r.unwrap()).collect())
+}
+
+/// Extract content from multiple byte arrays concurrently.
+///
+/// This function processes multiple byte arrays in parallel, automatically managing
+/// concurrency to prevent resource exhaustion. The concurrency limit can be
+/// configured via `ExtractionConfig::max_concurrent_extractions` or defaults
+/// to `num_cpus * 2`.
+///
+/// # Arguments
+///
+/// * `contents` - Vector of (bytes, mime_type) tuples
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// A vector of `ExtractionResult` in the same order as the input.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_bytes;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let contents = vec![
+///     (b"content 1".to_vec(), "text/plain".to_string()),
+///     (b"content 2".to_vec(), "text/plain".to_string()),
+/// ];
+/// let results = batch_extract_bytes(contents, &config).await?;
+/// println!("Processed {} items", results.len());
+/// # Ok(())
+/// # }
+/// ```
+#[cfg(feature = "tokio-runtime")]
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, contents),
+    fields(
+        extraction.batch_size = contents.len(),
+    )
+))]
+pub async fn batch_extract_bytes(
+    contents: Vec<(Vec<u8>, String)>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    use tokio::sync::Semaphore;
+    use tokio::task::JoinSet;
+
+    if contents.is_empty() {
+        return Ok(vec![]);
+    }
+
+    let batch_config = config.clone();
+    let config = Arc::new(batch_config);
+
+    let max_concurrent = config
+        .max_concurrent_extractions
+        .unwrap_or_else(|| (num_cpus::get() as f64 * 1.5).ceil() as usize);
+    let semaphore = Arc::new(Semaphore::new(max_concurrent));
+
+    let mut tasks = JoinSet::new();
+
+    for (index, (bytes, mime_type)) in contents.into_iter().enumerate() {
+        let config_clone = Arc::clone(&config);
+        let semaphore_clone = Arc::clone(&semaphore);
+
+        tasks.spawn(async move {
+            let _permit = semaphore_clone.acquire().await.unwrap();
+            let result = crate::core::batch_mode::with_batch_mode(async {
+                extract_bytes(&bytes, &mime_type, &config_clone).await
+            })
+            .await;
+            (index, result)
+        });
+    }
+
+    let mut results: Vec<Option<ExtractionResult>> = vec![None; tasks.len()];
+
+    while let Some(task_result) = tasks.join_next().await {
+        match task_result {
+            Ok((index, Ok(result))) => {
+                results[index] = Some(result);
+            }
+            Ok((index, Err(e))) => {
+                // All errors (including Io) should create error results
+                // instead of causing early return that abandons running tasks
+                let metadata = Metadata {
+                    error: Some(ErrorMetadata {
+                        error_type: format!("{:?}", e),
+                        message: e.to_string(),
+                    }),
+                    ..Default::default()
+                };
+
+                results[index] = Some(ExtractionResult {
+                    content: format!("Error: {}", e),
+                    mime_type: "text/plain".to_string(),
+                    metadata,
+                    tables: vec![],
+                    detected_languages: None,
+                    chunks: None,
+                    images: None,
+                    djot_content: None,
+                    pages: None,
+                    elements: None,
+                });
+            }
+            Err(join_err) => {
+                return Err(KreuzbergError::Other(format!("Task panicked: {}", join_err)));
+            }
+        }
+    }
+
+    #[allow(clippy::unwrap_used)]
+    Ok(results.into_iter().map(|r| r.unwrap()).collect())
+}