kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/config/processing.rs

@@ -0,0 +1,312 @@
+//! Post-processing and chunking configuration.
+//!
+//! Defines configuration for post-processing pipelines, text chunking,
+//! and embedding generation.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+/// Post-processor configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PostProcessorConfig {
+    /// Enable post-processors
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Whitelist of processor names to run (None = all enabled)
+    #[serde(default)]
+    pub enabled_processors: Option<Vec<String>>,
+
+    /// Blacklist of processor names to skip (None = none disabled)
+    #[serde(default)]
+    pub disabled_processors: Option<Vec<String>>,
+
+    /// Pre-computed HashSet for O(1) enabled processor lookup
+    #[serde(skip)]
+    pub enabled_set: Option<HashSet<String>>,
+
+    /// Pre-computed HashSet for O(1) disabled processor lookup
+    #[serde(skip)]
+    pub disabled_set: Option<HashSet<String>>,
+}
+
+impl PostProcessorConfig {
+    /// Pre-compute HashSets for O(1) processor name lookups.
+    ///
+    /// This method converts the enabled/disabled processor Vec to HashSet
+    /// for constant-time lookups in the pipeline.
+    pub fn build_lookup_sets(&mut self) {
+        if let Some(ref enabled) = self.enabled_processors {
+            self.enabled_set = Some(enabled.iter().cloned().collect());
+        }
+        if let Some(ref disabled) = self.disabled_processors {
+            self.disabled_set = Some(disabled.iter().cloned().collect());
+        }
+    }
+}
+
+impl Default for PostProcessorConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            enabled_processors: None,
+            disabled_processors: None,
+            enabled_set: None,
+            disabled_set: None,
+        }
+    }
+}
+
+/// Chunking configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChunkingConfig {
+    /// Maximum characters per chunk
+    #[serde(default = "default_chunk_size")]
+    pub max_chars: usize,
+
+    /// Overlap between chunks in characters
+    #[serde(default = "default_chunk_overlap")]
+    pub max_overlap: usize,
+
+    /// Optional embedding configuration for chunk embeddings
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub embedding: Option<EmbeddingConfig>,
+
+    /// Use a preset configuration (overrides individual settings if provided)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub preset: Option<String>,
+}
+
+/// Embedding configuration for text chunks.
+///
+/// Configures embedding generation using ONNX models via fastembed-rs.
+/// Requires the `embeddings` feature to be enabled.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EmbeddingConfig {
+    /// The embedding model to use
+    pub model: EmbeddingModelType,
+
+    /// Whether to normalize embedding vectors (recommended for cosine similarity)
+    #[serde(default = "default_normalize")]
+    pub normalize: bool,
+
+    /// Batch size for embedding generation
+    #[serde(default = "default_batch_size")]
+    pub batch_size: usize,
+
+    /// Show model download progress
+    #[serde(default)]
+    pub show_download_progress: bool,
+
+    /// Custom cache directory for model files
+    ///
+    /// Defaults to `~/.cache/kreuzberg/embeddings/` if not specified.
+    /// Allows full customization of model download location.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_dir: Option<PathBuf>,
+}
+
+impl Default for EmbeddingConfig {
+    fn default() -> Self {
+        Self {
+            model: EmbeddingModelType::Preset {
+                name: "balanced".to_string(),
+            },
+            normalize: true,
+            batch_size: 32,
+            show_download_progress: false,
+            cache_dir: None,
+        }
+    }
+}
+
+/// Embedding model types supported by Kreuzberg.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum EmbeddingModelType {
+    /// Use a preset model configuration (recommended)
+    Preset { name: String },
+
+    /// Use a specific fastembed model by name
+    #[cfg(feature = "embeddings")]
+    FastEmbed { model: String, dimensions: usize },
+
+    /// Use a custom ONNX model from HuggingFace
+    Custom { model_id: String, dimensions: usize },
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_chunk_size() -> usize {
+    1000
+}
+
+fn default_chunk_overlap() -> usize {
+    200
+}
+
+fn default_normalize() -> bool {
+    true
+}
+
+fn default_batch_size() -> usize {
+    32
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_postprocessor_config_default() {
+        let config = PostProcessorConfig::default();
+        assert!(config.enabled);
+        assert!(config.enabled_processors.is_none());
+        assert!(config.disabled_processors.is_none());
+    }
+
+    #[test]
+    fn test_postprocessor_config_build_lookup_sets() {
+        let mut config = PostProcessorConfig {
+            enabled: true,
+            enabled_processors: Some(vec!["a".to_string(), "b".to_string()]),
+            disabled_processors: Some(vec!["c".to_string()]),
+            enabled_set: None,
+            disabled_set: None,
+        };
+
+        config.build_lookup_sets();
+
+        assert!(config.enabled_set.is_some());
+        assert!(config.disabled_set.is_some());
+        assert!(config.enabled_set.unwrap().contains("a"));
+        assert!(config.disabled_set.unwrap().contains("c"));
+    }
+
+    #[test]
+    fn test_chunking_config_defaults() {
+        let config = ChunkingConfig {
+            max_chars: 1000,
+            max_overlap: 200,
+            embedding: None,
+            preset: None,
+        };
+        assert_eq!(config.max_chars, 1000);
+        assert_eq!(config.max_overlap, 200);
+    }
+
+    #[test]
+    fn test_embedding_config_default() {
+        let config = EmbeddingConfig::default();
+        assert!(config.normalize);
+        assert_eq!(config.batch_size, 32);
+        assert!(config.cache_dir.is_none());
+    }
+
+    /// Tests that EmbeddingModelType::Preset serializes with "type" field (internally-tagged).
+    /// This validates the API schema matches the documented format:
+    /// `{"type": "preset", "name": "fast"}` NOT `{"preset": {"name": "fast"}}`
+    #[test]
+    fn test_embedding_model_type_preset_serialization() {
+        let model = EmbeddingModelType::Preset {
+            name: "fast".to_string(),
+        };
+        let json = serde_json::to_string(&model).unwrap();
+
+        // Should use internally-tagged format with "type" discriminator
+        assert!(json.contains(r#""type":"preset""#), "Should contain type:preset field");
+        assert!(json.contains(r#""name":"fast""#), "Should contain name:fast field");
+
+        // Should NOT use adjacently-tagged format
+        assert!(
+            !json.contains(r#"{"preset":"#),
+            "Should NOT use adjacently-tagged format"
+        );
+    }
+
+    /// Tests that EmbeddingModelType::Preset deserializes from the documented API format.
+    /// API documentation shows: `{"type": "preset", "name": "fast"}`
+    #[test]
+    fn test_embedding_model_type_preset_deserialization() {
+        // This is the documented API format that users should send
+        let json = r#"{"type": "preset", "name": "fast"}"#;
+        let model: EmbeddingModelType = serde_json::from_str(json).unwrap();
+
+        match model {
+            EmbeddingModelType::Preset { name } => {
+                assert_eq!(name, "fast");
+            }
+            _ => panic!("Expected Preset variant"),
+        }
+    }
+
+    /// Tests that the wrong format (adjacently-tagged) is rejected.
+    /// This ensures the API doesn't accept the old/wrong documentation format.
+    #[test]
+    fn test_embedding_model_type_rejects_wrong_format() {
+        // This is the WRONG format that was in the old documentation
+        let wrong_json = r#"{"preset": {"name": "fast"}}"#;
+        let result: Result<EmbeddingModelType, _> = serde_json::from_str(wrong_json);
+
+        // Should fail to parse - the wrong format should be rejected
+        assert!(result.is_err(), "Should reject adjacently-tagged format");
+    }
+
+    /// Tests round-trip serialization/deserialization of EmbeddingConfig.
+    #[test]
+    fn test_embedding_config_roundtrip() {
+        let config = EmbeddingConfig {
+            model: EmbeddingModelType::Preset {
+                name: "balanced".to_string(),
+            },
+            normalize: true,
+            batch_size: 64,
+            show_download_progress: false,
+            cache_dir: None,
+        };
+
+        let json = serde_json::to_string(&config).unwrap();
+        let deserialized: EmbeddingConfig = serde_json::from_str(&json).unwrap();
+
+        match deserialized.model {
+            EmbeddingModelType::Preset { name } => {
+                assert_eq!(name, "balanced");
+            }
+            _ => panic!("Expected Preset variant"),
+        }
+        assert!(deserialized.normalize);
+        assert_eq!(deserialized.batch_size, 64);
+    }
+
+    /// Tests Custom model type serialization format.
+    #[test]
+    fn test_embedding_model_type_custom_serialization() {
+        let model = EmbeddingModelType::Custom {
+            model_id: "sentence-transformers/all-MiniLM-L6-v2".to_string(),
+            dimensions: 384,
+        };
+        let json = serde_json::to_string(&model).unwrap();
+
+        assert!(json.contains(r#""type":"custom""#), "Should contain type:custom field");
+        assert!(json.contains(r#""model_id":"#), "Should contain model_id field");
+        assert!(json.contains(r#""dimensions":384"#), "Should contain dimensions field");
+    }
+
+    /// Tests Custom model type deserialization.
+    #[test]
+    fn test_embedding_model_type_custom_deserialization() {
+        let json = r#"{"type": "custom", "model_id": "test/model", "dimensions": 512}"#;
+        let model: EmbeddingModelType = serde_json::from_str(json).unwrap();
+
+        match model {
+            EmbeddingModelType::Custom { model_id, dimensions } => {
+                assert_eq!(model_id, "test/model");
+                assert_eq!(dimensions, 512);
+            }
+            _ => panic!("Expected Custom variant"),
+        }
+    }
+}

data/vendor/kreuzberg/src/core/config_validation/dependencies.rs

@@ -0,0 +1,187 @@
+//! Cross-section dependency validation.
+//!
+//! This module contains validation functions that check dependencies and relationships
+//! between different configuration sections. These validators ensure that related
+//! configuration values are consistent and compatible with each other.
+
+use crate::{KreuzbergError, Result};
+
+/// Validate a port number for server configuration.
+///
+/// Port must be in the range 1-65535. While ports 1-1023 are privileged and may require
+/// special permissions on some systems, they are still valid port numbers.
+///
+/// # Arguments
+///
+/// * `port` - The port number to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the port is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_port;
+///
+/// assert!(validate_port(8000).is_ok());
+/// assert!(validate_port(80).is_ok());
+/// assert!(validate_port(1).is_ok());
+/// assert!(validate_port(65535).is_ok());
+/// assert!(validate_port(0).is_err());
+/// ```
+pub fn validate_port(port: u16) -> Result<()> {
+    if port > 0 {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!("Port must be 1-65535, got {}", port),
+            source: None,
+        })
+    }
+}
+
+/// Validate a host/IP address string for server configuration.
+///
+/// Accepts valid IPv4 addresses (e.g., "127.0.0.1", "0.0.0.0"), valid IPv6 addresses
+/// (e.g., "::1", "::"), and hostnames (e.g., "localhost", "example.com").
+///
+/// # Arguments
+///
+/// * `host` - The host/IP address string to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the host is valid, or a `ValidationError` with details about valid formats.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_host;
+///
+/// assert!(validate_host("127.0.0.1").is_ok());
+/// assert!(validate_host("0.0.0.0").is_ok());
+/// assert!(validate_host("::1").is_ok());
+/// assert!(validate_host("::").is_ok());
+/// assert!(validate_host("localhost").is_ok());
+/// assert!(validate_host("example.com").is_ok());
+/// assert!(validate_host("").is_err());
+/// ```
+pub fn validate_host(host: &str) -> Result<()> {
+    let host = host.trim();
+
+    if host.is_empty() {
+        return Err(KreuzbergError::Validation {
+            message: "Invalid host '': must be a valid IP address or hostname".to_string(),
+            source: None,
+        });
+    }
+
+    // Check if it's a valid IPv4 address
+    if host.parse::<std::net::Ipv4Addr>().is_ok() {
+        return Ok(());
+    }
+
+    // Check if it's a valid IPv6 address
+    if host.parse::<std::net::Ipv6Addr>().is_ok() {
+        return Ok(());
+    }
+
+    // Check if it's a valid hostname (basic validation)
+    // Hostnames must contain only alphanumeric characters, dots, and hyphens
+    // Must not look like an invalid IPv4 address (all numeric with dots)
+    let looks_like_ipv4 = host
+        .split('.')
+        .all(|part| !part.is_empty() && part.chars().all(|c| c.is_numeric()));
+    if !looks_like_ipv4
+        && host.chars().all(|c| c.is_alphanumeric() || c == '.' || c == '-')
+        && !host.starts_with('-')
+        && !host.ends_with('-')
+    {
+        return Ok(());
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!("Invalid host '{}': must be a valid IP address or hostname", host),
+        source: None,
+    })
+}
+
+/// Validate a CORS (Cross-Origin Resource Sharing) origin URL.
+///
+/// Accepts valid HTTP/HTTPS URLs (e.g., "https://example.com") or the wildcard "*"
+/// to allow all origins. URLs must start with "http://" or "https://", or be exactly "*".
+///
+/// # Arguments
+///
+/// * `origin` - The CORS origin URL to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the origin is valid, or a `ValidationError` with details about valid formats.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_cors_origin;
+///
+/// assert!(validate_cors_origin("https://example.com").is_ok());
+/// assert!(validate_cors_origin("http://localhost:3000").is_ok());
+/// assert!(validate_cors_origin("*").is_ok());
+/// assert!(validate_cors_origin("not-a-url").is_err());
+/// assert!(validate_cors_origin("ftp://example.com").is_err());
+/// ```
+pub fn validate_cors_origin(origin: &str) -> Result<()> {
+    let origin = origin.trim();
+
+    if origin == "*" {
+        return Ok(());
+    }
+
+    if origin.starts_with("http://") || origin.starts_with("https://") {
+        // Basic validation: ensure there's something after the protocol
+        if origin.len() > 8 && (origin.starts_with("http://") && origin.len() > 7 || origin.starts_with("https://")) {
+            return Ok(());
+        }
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!(
+            "Invalid CORS origin '{}': must be a valid HTTP/HTTPS URL or '*'",
+            origin
+        ),
+        source: None,
+    })
+}
+
+/// Validate an upload size limit for server configuration.
+///
+/// Upload size must be greater than 0 (measured in bytes).
+///
+/// # Arguments
+///
+/// * `size` - The maximum upload size in bytes to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the size is valid, or a `ValidationError` with details about constraints.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_upload_size;
+///
+/// assert!(validate_upload_size(1024).is_ok());
+/// assert!(validate_upload_size(1_000_000).is_ok());
+/// assert!(validate_upload_size(0).is_err());
+/// ```
+pub fn validate_upload_size(size: usize) -> Result<()> {
+    if size > 0 {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!("Upload size must be greater than 0, got {}", size),
+            source: None,
+        })
+    }
+}