kreuzberg 4.0.8 → 4.1.1

data/vendor/kreuzberg-ffi/src/config/html.rs

@@ -0,0 +1,318 @@
+//! HTML options parsing from JSON
+//!
+//! Handles the complex nested structure of HTML conversion options.
+
+use html_to_markdown_rs::options::{
+    CodeBlockStyle, ConversionOptions, HeadingStyle, HighlightStyle, ListIndentType, NewlineStyle, PreprocessingPreset,
+    WhitespaceMode,
+};
+
+type FfiResult<T> = std::result::Result<T, String>;
+
+/// Parse enum value from optional JSON value
+fn parse_enum<T, F>(value: Option<&serde_json::Value>, parse_fn: F) -> FfiResult<Option<T>>
+where
+    F: Fn(&str) -> FfiResult<T>,
+{
+    if let Some(raw) = value {
+        let text = raw
+            .as_str()
+            .ok_or_else(|| "Expected string for enum field".to_string())?;
+        return parse_fn(text).map(Some);
+    }
+    Ok(None)
+}
+
+/// Parse HeadingStyle from string
+fn parse_heading_style(value: &str) -> FfiResult<HeadingStyle> {
+    match value.to_lowercase().as_str() {
+        "atx" => Ok(HeadingStyle::Atx),
+        "underlined" => Ok(HeadingStyle::Underlined),
+        "atx_closed" => Ok(HeadingStyle::AtxClosed),
+        other => Err(format!(
+            "Invalid heading_style '{}'. Expected one of: atx, underlined, atx_closed",
+            other
+        )),
+    }
+}
+
+/// Parse ListIndentType from string
+fn parse_list_indent_type(value: &str) -> FfiResult<ListIndentType> {
+    match value.to_lowercase().as_str() {
+        "spaces" => Ok(ListIndentType::Spaces),
+        "tabs" => Ok(ListIndentType::Tabs),
+        other => Err(format!(
+            "Invalid list_indent_type '{}'. Expected 'spaces' or 'tabs'",
+            other
+        )),
+    }
+}
+
+/// Parse HighlightStyle from string
+fn parse_highlight_style(value: &str) -> FfiResult<HighlightStyle> {
+    match value.to_lowercase().as_str() {
+        "double_equal" | "==" | "highlight" => Ok(HighlightStyle::DoubleEqual),
+        "html" => Ok(HighlightStyle::Html),
+        "bold" => Ok(HighlightStyle::Bold),
+        "none" => Ok(HighlightStyle::None),
+        other => Err(format!(
+            "Invalid highlight_style '{}'. Expected one of: double_equal, html, bold, none",
+            other
+        )),
+    }
+}
+
+/// Parse WhitespaceMode from string
+fn parse_whitespace_mode(value: &str) -> FfiResult<WhitespaceMode> {
+    match value.to_lowercase().as_str() {
+        "normalized" => Ok(WhitespaceMode::Normalized),
+        "strict" => Ok(WhitespaceMode::Strict),
+        other => Err(format!(
+            "Invalid whitespace_mode '{}'. Expected 'normalized' or 'strict'",
+            other
+        )),
+    }
+}
+
+/// Parse NewlineStyle from string
+fn parse_newline_style(value: &str) -> FfiResult<NewlineStyle> {
+    match value.to_lowercase().as_str() {
+        "spaces" => Ok(NewlineStyle::Spaces),
+        "backslash" => Ok(NewlineStyle::Backslash),
+        other => Err(format!(
+            "Invalid newline_style '{}'. Expected 'spaces' or 'backslash'",
+            other
+        )),
+    }
+}
+
+/// Parse CodeBlockStyle from string
+fn parse_code_block_style(value: &str) -> FfiResult<CodeBlockStyle> {
+    match value.to_lowercase().as_str() {
+        "indented" => Ok(CodeBlockStyle::Indented),
+        "backticks" => Ok(CodeBlockStyle::Backticks),
+        "tildes" => Ok(CodeBlockStyle::Tildes),
+        other => Err(format!(
+            "Invalid code_block_style '{}'. Expected 'indented', 'backticks', or 'tildes'",
+            other
+        )),
+    }
+}
+
+/// Parse PreprocessingPreset from string
+#[allow(dead_code)]
+fn parse_preprocessing_preset(value: &str) -> FfiResult<PreprocessingPreset> {
+    match value.to_lowercase().as_str() {
+        "minimal" => Ok(PreprocessingPreset::Minimal),
+        "standard" => Ok(PreprocessingPreset::Standard),
+        "aggressive" => Ok(PreprocessingPreset::Aggressive),
+        other => Err(format!(
+            "Invalid preprocessing.preset '{}'. Expected one of: minimal, standard, aggressive",
+            other
+        )),
+    }
+}
+
+/// Parse HTML conversion options from JSON value
+pub fn parse_html_options(value: &serde_json::Value) -> FfiResult<ConversionOptions> {
+    let mut opts = ConversionOptions::default();
+    let obj = value
+        .as_object()
+        .ok_or_else(|| "html_options must be an object".to_string())?;
+
+    if let Some(val) = obj.get("heading_style") {
+        opts.heading_style = parse_enum(Some(val), parse_heading_style)?.unwrap_or(opts.heading_style);
+    }
+
+    if let Some(val) = obj.get("list_indent_type") {
+        opts.list_indent_type = parse_enum(Some(val), parse_list_indent_type)?.unwrap_or(opts.list_indent_type);
+    }
+
+    if let Some(val) = obj.get("list_indent_width") {
+        opts.list_indent_width = val
+            .as_u64()
+            .map(|v| v as usize)
+            .ok_or_else(|| "list_indent_width must be an integer".to_string())?;
+    }
+
+    if let Some(val) = obj.get("bullets") {
+        opts.bullets = val
+            .as_str()
+            .map(str::to_string)
+            .ok_or_else(|| "bullets must be a string".to_string())?;
+    }
+
+    if let Some(val) = obj.get("strong_em_symbol") {
+        let symbol = val
+            .as_str()
+            .ok_or_else(|| "strong_em_symbol must be a string".to_string())?;
+        let mut chars = symbol.chars();
+        opts.strong_em_symbol = chars
+            .next()
+            .ok_or_else(|| "strong_em_symbol must not be empty".to_string())?;
+    }
+
+    if let Some(val) = obj.get("escape_asterisks") {
+        opts.escape_asterisks = val
+            .as_bool()
+            .ok_or_else(|| "escape_asterisks must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("escape_underscores") {
+        opts.escape_underscores = val
+            .as_bool()
+            .ok_or_else(|| "escape_underscores must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("escape_misc") {
+        opts.escape_misc = val
+            .as_bool()
+            .ok_or_else(|| "escape_misc must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("escape_ascii") {
+        opts.escape_ascii = val
+            .as_bool()
+            .ok_or_else(|| "escape_ascii must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("code_language") {
+        opts.code_language = val
+            .as_str()
+            .map(str::to_string)
+            .ok_or_else(|| "code_language must be a string".to_string())?;
+    }
+
+    if let Some(val) = obj.get("autolinks") {
+        opts.autolinks = val.as_bool().ok_or_else(|| "autolinks must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("default_title") {
+        opts.default_title = val
+            .as_bool()
+            .ok_or_else(|| "default_title must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("br_in_tables") {
+        opts.br_in_tables = val
+            .as_bool()
+            .ok_or_else(|| "br_in_tables must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("hocr_spatial_tables") {
+        opts.hocr_spatial_tables = val
+            .as_bool()
+            .ok_or_else(|| "hocr_spatial_tables must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("highlight_style") {
+        opts.highlight_style = parse_enum(Some(val), parse_highlight_style)?.unwrap_or(opts.highlight_style);
+    }
+
+    if let Some(val) = obj.get("extract_metadata") {
+        opts.extract_metadata = val
+            .as_bool()
+            .ok_or_else(|| "extract_metadata must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("whitespace_mode") {
+        opts.whitespace_mode = parse_enum(Some(val), parse_whitespace_mode)?.unwrap_or(opts.whitespace_mode);
+    }
+
+    if let Some(val) = obj.get("strip_newlines") {
+        opts.strip_newlines = val
+            .as_bool()
+            .ok_or_else(|| "strip_newlines must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("wrap") {
+        opts.wrap = val.as_bool().ok_or_else(|| "wrap must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("wrap_width") {
+        opts.wrap_width = val
+            .as_u64()
+            .map(|v| v as usize)
+            .ok_or_else(|| "wrap_width must be an integer".to_string())?;
+    }
+
+    if let Some(val) = obj.get("convert_as_inline") {
+        opts.convert_as_inline = val
+            .as_bool()
+            .ok_or_else(|| "convert_as_inline must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("sub_symbol") {
+        opts.sub_symbol = val
+            .as_str()
+            .map(str::to_string)
+            .ok_or_else(|| "sub_symbol must be a string".to_string())?;
+    }
+
+    if let Some(val) = obj.get("sup_symbol") {
+        opts.sup_symbol = val
+            .as_str()
+            .map(str::to_string)
+            .ok_or_else(|| "sup_symbol must be a string".to_string())?;
+    }
+
+    if let Some(val) = obj.get("newline_style") {
+        opts.newline_style = parse_enum(Some(val), parse_newline_style)?.unwrap_or(opts.newline_style);
+    }
+
+    if let Some(val) = obj.get("code_block_style") {
+        opts.code_block_style = parse_enum(Some(val), parse_code_block_style)?.unwrap_or(opts.code_block_style);
+    }
+
+    if let Some(val) = obj.get("keep_inline_images_in") {
+        opts.keep_inline_images_in = val
+            .as_array()
+            .ok_or_else(|| "keep_inline_images_in must be an array".to_string())?
+            .iter()
+            .map(|v| {
+                v.as_str()
+                    .map(str::to_string)
+                    .ok_or_else(|| "keep_inline_images_in entries must be strings".to_string())
+            })
+            .collect::<FfiResult<Vec<_>>>()?;
+    }
+
+    if let Some(val) = obj.get("encoding") {
+        opts.encoding = val
+            .as_str()
+            .map(str::to_string)
+            .ok_or_else(|| "encoding must be a string".to_string())?;
+    }
+
+    if let Some(val) = obj.get("debug") {
+        opts.debug = val.as_bool().ok_or_else(|| "debug must be a boolean".to_string())?;
+    }
+
+    if let Some(val) = obj.get("strip_tags") {
+        opts.strip_tags = val
+            .as_array()
+            .ok_or_else(|| "strip_tags must be an array".to_string())?
+            .iter()
+            .map(|v| {
+                v.as_str()
+                    .map(str::to_string)
+                    .ok_or_else(|| "strip_tags entries must be strings".to_string())
+            })
+            .collect::<FfiResult<Vec<_>>>()?;
+    }
+
+    if let Some(val) = obj.get("preserve_tags") {
+        opts.preserve_tags = val
+            .as_array()
+            .ok_or_else(|| "preserve_tags must be an array".to_string())?
+            .iter()
+            .map(|v| {
+                v.as_str()
+                    .map(str::to_string)
+                    .ok_or_else(|| "preserve_tags entries must be strings".to_string())
+            })
+            .collect::<FfiResult<Vec<_>>>()?;
+    }
+
+    Ok(opts)
+}

data/vendor/kreuzberg-ffi/src/config/loader.rs

@@ -0,0 +1,154 @@
+//! Configuration loading from files
+//!
+//! Handles loading ExtractionConfig from TOML/JSON/YAML files and discovery.
+
+use crate::helpers::set_last_error;
+use kreuzberg::KreuzbergError;
+use kreuzberg::core::config::ExtractionConfig;
+use std::path::Path;
+
+/// Load an ExtractionConfig from a file (returns JSON string).
+///
+/// # Arguments
+///
+/// * `file_path` - Path to the configuration file
+///
+/// # Returns
+///
+/// JSON string representation of the config, or error message.
+pub fn load_config_as_json(file_path: &str) -> Result<String, String> {
+    match ExtractionConfig::from_file(file_path) {
+        Ok(config) => match serde_json::to_string(&config) {
+            Ok(json) => Ok(json),
+            Err(e) => Err(format!("Failed to serialize config to JSON: {}", e)),
+        },
+        Err(e) => Err(e.to_string()),
+    }
+}
+
+/// Load an ExtractionConfig from a file (returns config struct).
+///
+/// # Arguments
+///
+/// * `path` - Path to the configuration file
+///
+/// # Returns
+///
+/// ExtractionConfig on success, or error message.
+pub fn load_config_from_file(path: &Path) -> Result<ExtractionConfig, String> {
+    match ExtractionConfig::from_file(path) {
+        Ok(config) => Ok(config),
+        Err(e) => match &e {
+            KreuzbergError::Io(io_err) => Err(format!("IO error loading config: {}", io_err)),
+            _ => Err(format!("Failed to load config from file: {}", e)),
+        },
+    }
+}
+
+/// Discover and load an ExtractionConfig (returns JSON string).
+///
+/// Searches the current directory and all parent directories for:
+/// - `kreuzberg.toml`
+/// - `kreuzberg.json`
+///
+/// # Returns
+///
+/// JSON string of the first config file found, or None if not found.
+pub fn discover_config_as_json() -> Option<String> {
+    match ExtractionConfig::discover() {
+        Ok(Some(config)) => match serde_json::to_string(&config) {
+            Ok(json) => Some(json),
+            Err(e) => {
+                set_last_error(format!("Failed to serialize config: {}", e));
+                None
+            }
+        },
+        Ok(None) => None,
+        Err(e) => {
+            match &e {
+                KreuzbergError::Io(io_err) => {
+                    set_last_error(format!("IO error discovering config: {}", io_err));
+                }
+                _ => {
+                    set_last_error(format!("Failed to discover config: {}", e));
+                }
+            }
+            None
+        }
+    }
+}
+
+/// List available embedding preset names.
+///
+/// # Returns
+///
+/// JSON array of preset names, or error message.
+pub fn list_embedding_presets() -> Result<String, String> {
+    let presets = kreuzberg::embeddings::list_presets();
+    match serde_json::to_string(&presets) {
+        Ok(json) => Ok(json),
+        Err(e) => Err(format!("Failed to serialize presets: {}", e)),
+    }
+}
+
+/// Get a specific embedding preset by name.
+///
+/// # Arguments
+///
+/// * `preset_name` - Name of the preset to retrieve
+///
+/// # Returns
+///
+/// JSON representation of the preset, or error message.
+pub fn get_embedding_preset(preset_name: &str) -> Result<String, String> {
+    let preset = match kreuzberg::embeddings::get_preset(preset_name) {
+        Some(preset) => preset,
+        None => {
+            return Err(format!("Unknown embedding preset: {}", preset_name));
+        }
+    };
+
+    let model_name = format!("{:?}", preset.model);
+    let serializable = super::serialize::SerializableEmbeddingPreset {
+        name: preset.name,
+        chunk_size: preset.chunk_size,
+        overlap: preset.overlap,
+        model_name,
+        dimensions: preset.dimensions,
+        description: preset.description,
+    };
+
+    match serde_json::to_string(&serializable) {
+        Ok(json) => Ok(json),
+        Err(e) => Err(format!("Failed to serialize embedding preset: {}", e)),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_list_embedding_presets() {
+        let result = list_embedding_presets();
+        assert!(result.is_ok());
+        let json = result.unwrap();
+        assert!(json.starts_with('['));
+        assert!(json.ends_with(']'));
+    }
+
+    #[test]
+    fn test_get_embedding_preset_unknown() {
+        let result = get_embedding_preset("nonexistent_preset");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_get_embedding_preset_valid() {
+        let result = get_embedding_preset("fast");
+        assert!(result.is_ok());
+        let json = result.unwrap();
+        assert!(json.contains("name"));
+        assert!(json.contains("chunk_size"));
+    }
+}

data/vendor/kreuzberg-ffi/src/config/merge.rs

@@ -0,0 +1,104 @@
+//! Configuration merging logic
+//!
+//! Provides functionality to merge two ExtractionConfig instances.
+
+use kreuzberg::core::config::ExtractionConfig;
+
+/// Merge two configs (override takes precedence over base).
+///
+/// Performs a shallow merge where fields from `override_config` take
+/// precedence over fields in `base`. The `base` config is modified in-place.
+///
+/// # Arguments
+///
+/// * `base` - Mutable reference to the base config (will be modified)
+/// * `override_config` - Reference to the override config (read-only)
+pub fn merge_configs(base: &mut ExtractionConfig, override_config: &ExtractionConfig) {
+    base.use_cache = override_config.use_cache;
+    base.enable_quality_processing = override_config.enable_quality_processing;
+    base.force_ocr = override_config.force_ocr;
+    base.max_concurrent_extractions = override_config.max_concurrent_extractions;
+
+    if override_config.ocr.is_some() {
+        base.ocr = override_config.ocr.clone();
+    }
+
+    if override_config.chunking.is_some() {
+        base.chunking = override_config.chunking.clone();
+    }
+
+    if override_config.images.is_some() {
+        base.images = override_config.images.clone();
+    }
+
+    #[cfg(feature = "pdf")]
+    if override_config.pdf_options.is_some() {
+        base.pdf_options = override_config.pdf_options.clone();
+    }
+
+    if override_config.token_reduction.is_some() {
+        base.token_reduction = override_config.token_reduction.clone();
+    }
+
+    if override_config.language_detection.is_some() {
+        base.language_detection = override_config.language_detection.clone();
+    }
+
+    if override_config.pages.is_some() {
+        base.pages = override_config.pages.clone();
+    }
+
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    if override_config.keywords.is_some() {
+        base.keywords = override_config.keywords.clone();
+    }
+
+    if override_config.postprocessor.is_some() {
+        base.postprocessor = override_config.postprocessor.clone();
+    }
+
+    if override_config.html_options.is_some() {
+        base.html_options = override_config.html_options.clone();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_merge_configs_simple() {
+        let mut base = ExtractionConfig {
+            use_cache: true,
+            force_ocr: false,
+            ..Default::default()
+        };
+
+        let override_config = ExtractionConfig {
+            force_ocr: true,
+            ..Default::default()
+        };
+
+        merge_configs(&mut base, &override_config);
+
+        assert!(base.use_cache);
+        assert!(base.force_ocr);
+    }
+
+    #[test]
+    fn test_merge_configs_override_to_default() {
+        let mut base = ExtractionConfig {
+            use_cache: false,
+            ..Default::default()
+        };
+
+        let override_config = ExtractionConfig {
+            use_cache: true,
+            ..Default::default()
+        };
+
+        merge_configs(&mut base, &override_config);
+
+        assert!(base.use_cache, "override to default value should be applied");
+    }
+}