kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/config/formats.rs

@@ -0,0 +1,135 @@
+//! Output format configuration and validation.
+//!
+//! This module defines the `OutputFormat` enum for controlling how extraction
+//! results are formatted (plain text, markdown, HTML, etc.) and provides
+//! serialization/deserialization support.
+
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+/// Output format for extraction results.
+///
+/// Controls the format of the `content` field in `ExtractionResult`.
+/// When set to `Markdown`, `Djot`, or `Html`, the output will be formatted
+/// accordingly. `Plain` returns the raw extracted text.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum OutputFormat {
+    /// Plain text content only (default)
+    #[default]
+    Plain,
+    /// Markdown format
+    Markdown,
+    /// Djot markup format (requires djot feature)
+    Djot,
+    /// HTML format
+    Html,
+}
+
+impl std::fmt::Display for OutputFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            OutputFormat::Plain => write!(f, "plain"),
+            OutputFormat::Markdown => write!(f, "markdown"),
+            OutputFormat::Djot => write!(f, "djot"),
+            OutputFormat::Html => write!(f, "html"),
+        }
+    }
+}
+
+impl FromStr for OutputFormat {
+    type Err = String;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "plain" | "text" => Ok(OutputFormat::Plain),
+            "markdown" | "md" => Ok(OutputFormat::Markdown),
+            "djot" => Ok(OutputFormat::Djot),
+            "html" => Ok(OutputFormat::Html),
+            _ => Err(format!(
+                "Invalid output format: '{}'. Valid formats: plain, text, markdown, md, djot, html",
+                s
+            )),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_output_format_from_str_plain() {
+        assert_eq!("plain".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("PLAIN".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("text".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("TEXT".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+    }
+
+    #[test]
+    fn test_output_format_from_str_markdown() {
+        assert_eq!("markdown".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("MARKDOWN".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("md".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("MD".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+    }
+
+    #[test]
+    fn test_output_format_from_str_djot() {
+        assert_eq!("djot".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+        assert_eq!("DJOT".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+        assert_eq!("Djot".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+    }
+
+    #[test]
+    fn test_output_format_from_str_html() {
+        assert_eq!("html".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+        assert_eq!("HTML".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+        assert_eq!("Html".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+    }
+
+    #[test]
+    fn test_output_format_from_str_invalid() {
+        let result = "invalid".parse::<OutputFormat>();
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.contains("Invalid output format"));
+        assert!(err.contains("invalid"));
+    }
+
+    #[test]
+    fn test_output_format_to_string() {
+        assert_eq!(OutputFormat::Plain.to_string(), "plain");
+        assert_eq!(OutputFormat::Markdown.to_string(), "markdown");
+        assert_eq!(OutputFormat::Djot.to_string(), "djot");
+        assert_eq!(OutputFormat::Html.to_string(), "html");
+    }
+
+    #[test]
+    fn test_output_format_default() {
+        let format = OutputFormat::default();
+        assert_eq!(format, OutputFormat::Plain);
+    }
+
+    #[test]
+    fn test_output_format_serde_roundtrip() {
+        for format in [
+            OutputFormat::Plain,
+            OutputFormat::Markdown,
+            OutputFormat::Djot,
+            OutputFormat::Html,
+        ] {
+            let json = serde_json::to_string(&format).unwrap();
+            let deserialized: OutputFormat = serde_json::from_str(&json).unwrap();
+            assert_eq!(format, deserialized);
+        }
+    }
+
+    #[test]
+    fn test_output_format_serde_values() {
+        assert_eq!(serde_json::to_string(&OutputFormat::Plain).unwrap(), "\"plain\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Markdown).unwrap(), "\"markdown\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Djot).unwrap(), "\"djot\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Html).unwrap(), "\"html\"");
+    }
+}

data/vendor/kreuzberg/src/core/config/mod.rs

@@ -0,0 +1,20 @@
+//! Configuration loading and management.
+//!
+//! This module provides utilities for loading extraction configuration from various
+//! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
+
+pub mod extraction;
+pub mod formats;
+pub mod ocr;
+pub mod page;
+pub mod pdf;
+pub mod processing;
+
+// Re-export main types for backward compatibility
+pub use extraction::{ExtractionConfig, ImageExtractionConfig, LanguageDetectionConfig, TokenReductionConfig};
+pub use formats::OutputFormat;
+pub use ocr::OcrConfig;
+pub use page::PageConfig;
+#[cfg(feature = "pdf")]
+pub use pdf::{HierarchyConfig, PdfConfig};
+pub use processing::{ChunkingConfig, EmbeddingConfig, EmbeddingModelType, PostProcessorConfig};

data/vendor/kreuzberg/src/core/config/ocr.rs

@@ -0,0 +1,73 @@
+//! OCR configuration.
+//!
+//! Defines OCR-specific configuration including backend selection, language settings,
+//! and Tesseract-specific parameters.
+
+use serde::{Deserialize, Serialize};
+
+use super::formats::OutputFormat;
+
+/// OCR configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrConfig {
+    /// OCR backend: tesseract, easyocr, paddleocr
+    #[serde(default = "default_tesseract_backend")]
+    pub backend: String,
+
+    /// Language code (e.g., "eng", "deu")
+    #[serde(default = "default_eng")]
+    pub language: String,
+
+    /// Tesseract-specific configuration (optional)
+    #[serde(default)]
+    pub tesseract_config: Option<crate::types::TesseractConfig>,
+
+    /// Output format for OCR results (optional, for format conversion)
+    #[serde(default)]
+    pub output_format: Option<OutputFormat>,
+}
+
+impl Default for OcrConfig {
+    fn default() -> Self {
+        Self {
+            backend: default_tesseract_backend(),
+            language: default_eng(),
+            tesseract_config: None,
+            output_format: None,
+        }
+    }
+}
+
+fn default_tesseract_backend() -> String {
+    "tesseract".to_string()
+}
+
+fn default_eng() -> String {
+    "eng".to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ocr_config_default() {
+        let config = OcrConfig::default();
+        assert_eq!(config.backend, "tesseract");
+        assert_eq!(config.language, "eng");
+        assert!(config.tesseract_config.is_none());
+        assert!(config.output_format.is_none());
+    }
+
+    #[test]
+    fn test_ocr_config_with_tesseract() {
+        let config = OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "fra".to_string(),
+            tesseract_config: None,
+            output_format: None,
+        };
+        assert_eq!(config.backend, "tesseract");
+        assert_eq!(config.language, "fra");
+    }
+}

data/vendor/kreuzberg/src/core/config/page.rs

@@ -0,0 +1,57 @@
+//! Page extraction and tracking configuration.
+//!
+//! Controls how pages are extracted, tracked, and represented in extraction results.
+//! When `None`, page tracking is disabled.
+
+use serde::{Deserialize, Serialize};
+
+/// Page extraction and tracking configuration.
+///
+/// Controls how pages are extracted, tracked, and represented in the extraction results.
+/// When `None`, page tracking is disabled.
+///
+/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
+/// when page boundaries are available and chunking is configured.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct PageConfig {
+    /// Extract pages as separate array (ExtractionResult.pages)
+    #[serde(default)]
+    pub extract_pages: bool,
+
+    /// Insert page markers in main content string
+    #[serde(default)]
+    pub insert_page_markers: bool,
+
+    /// Page marker format (use {page_num} placeholder)
+    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
+    #[serde(default = "default_page_marker_format")]
+    pub marker_format: String,
+}
+
+impl Default for PageConfig {
+    fn default() -> Self {
+        Self {
+            extract_pages: false,
+            insert_page_markers: false,
+            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
+        }
+    }
+}
+
+fn default_page_marker_format() -> String {
+    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_page_config_default() {
+        let config = PageConfig::default();
+        assert!(!config.extract_pages);
+        assert!(!config.insert_page_markers);
+        assert_eq!(config.marker_format, "\n\n<!-- PAGE {page_num} -->\n\n");
+    }
+}

data/vendor/kreuzberg/src/core/config/pdf.rs

@@ -0,0 +1,111 @@
+//! PDF-specific configuration.
+//!
+//! Defines PDF extraction options including metadata handling, image extraction,
+//! password management, and hierarchy extraction for document structure analysis.
+
+use serde::{Deserialize, Serialize};
+
+/// PDF-specific configuration.
+#[cfg(feature = "pdf")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PdfConfig {
+    /// Extract images from PDF
+    #[serde(default)]
+    pub extract_images: bool,
+
+    /// List of passwords to try when opening encrypted PDFs
+    #[serde(default)]
+    pub passwords: Option<Vec<String>>,
+
+    /// Extract PDF metadata
+    #[serde(default = "default_true")]
+    pub extract_metadata: bool,
+
+    /// Hierarchy extraction configuration (None = hierarchy extraction disabled)
+    #[serde(default)]
+    pub hierarchy: Option<HierarchyConfig>,
+}
+
+/// Hierarchy extraction configuration for PDF text structure analysis.
+///
+/// Enables extraction of document hierarchy levels (H1-H6) based on font size
+/// clustering and semantic analysis. When enabled, hierarchical blocks are
+/// included in page content.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HierarchyConfig {
+    /// Enable hierarchy extraction
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Number of font size clusters to use for hierarchy levels (1-7)
+    ///
+    /// Default: 6, which provides H1-H6 heading levels with body text.
+    /// Larger values create more fine-grained hierarchy levels.
+    #[serde(default = "default_k_clusters")]
+    pub k_clusters: usize,
+
+    /// Include bounding box information in hierarchy blocks
+    #[serde(default = "default_true")]
+    pub include_bbox: bool,
+
+    /// OCR coverage threshold for smart OCR triggering (0.0-1.0)
+    ///
+    /// Determines when OCR should be triggered based on text block coverage.
+    /// OCR is triggered when text blocks cover less than this fraction of the page.
+    /// Default: 0.5 (trigger OCR if less than 50% of page has text)
+    #[serde(default = "default_ocr_coverage_threshold")]
+    pub ocr_coverage_threshold: Option<f32>,
+}
+
+impl Default for HierarchyConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            k_clusters: 6,
+            include_bbox: true,
+            ocr_coverage_threshold: None,
+        }
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_k_clusters() -> usize {
+    6
+}
+
+fn default_ocr_coverage_threshold() -> Option<f32> {
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_hierarchy_config_default() {
+        let config = HierarchyConfig::default();
+        assert!(config.enabled);
+        assert_eq!(config.k_clusters, 6);
+        assert!(config.include_bbox);
+        assert!(config.ocr_coverage_threshold.is_none());
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_hierarchy_config_disabled() {
+        let config = HierarchyConfig {
+            enabled: false,
+            k_clusters: 3,
+            include_bbox: false,
+            ocr_coverage_threshold: Some(0.7),
+        };
+        assert!(!config.enabled);
+        assert_eq!(config.k_clusters, 3);
+        assert!(!config.include_bbox);
+        assert_eq!(config.ocr_coverage_threshold, Some(0.7));
+    }
+}