kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/server_config/env.rs

@@ -0,0 +1,90 @@
+//! Environment variable overrides for server configuration.
+//!
+//! This module provides functionality to override server configuration values
+//! using environment variables. All settings can be overridden at runtime.
+
+use crate::{KreuzbergError, Result};
+
+/// Apply environment variable overrides to a ServerConfig.
+///
+/// Reads the following environment variables and overrides config values if set:
+///
+/// - `KREUZBERG_HOST` - Server host address
+/// - `KREUZBERG_PORT` - Server port number (parsed as u16)
+/// - `KREUZBERG_CORS_ORIGINS` - Comma-separated list of allowed origins
+/// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` - Max request body size in bytes
+/// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` - Max multipart field size in bytes
+/// - `KREUZBERG_MAX_UPLOAD_SIZE_MB` - Max upload size in MB (legacy)
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if:
+/// - `KREUZBERG_PORT` cannot be parsed as u16
+/// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` cannot be parsed as usize
+/// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` cannot be parsed as usize
+/// - `KREUZBERG_MAX_UPLOAD_SIZE_MB` cannot be parsed as usize
+pub fn apply_env_overrides(
+    host: &mut String,
+    port: &mut u16,
+    cors_origins: &mut Vec<String>,
+    max_request_body_bytes: &mut usize,
+    max_multipart_field_bytes: &mut usize,
+    max_upload_mb: &mut Option<usize>,
+) -> Result<()> {
+    // Host override
+    if let Ok(env_host) = std::env::var("KREUZBERG_HOST") {
+        *host = env_host;
+    }
+
+    // Port override
+    if let Ok(port_str) = std::env::var("KREUZBERG_PORT") {
+        *port = port_str.parse::<u16>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_PORT must be a valid u16 number, got '{}': {}",
+                port_str, e
+            ))
+        })?;
+    }
+
+    // CORS origins override (comma-separated)
+    if let Ok(origins_str) = std::env::var("KREUZBERG_CORS_ORIGINS") {
+        *cors_origins = origins_str
+            .split(',')
+            .map(|s| s.trim().to_string())
+            .filter(|s| !s.is_empty())
+            .collect();
+    }
+
+    // Max request body bytes override
+    if let Ok(bytes_str) = std::env::var("KREUZBERG_MAX_REQUEST_BODY_BYTES") {
+        *max_request_body_bytes = bytes_str.parse::<usize>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_MAX_REQUEST_BODY_BYTES must be a valid usize, got '{}': {}",
+                bytes_str, e
+            ))
+        })?;
+    }
+
+    // Max multipart field bytes override
+    if let Ok(bytes_str) = std::env::var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES") {
+        *max_multipart_field_bytes = bytes_str.parse::<usize>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_MAX_MULTIPART_FIELD_BYTES must be a valid usize, got '{}': {}",
+                bytes_str, e
+            ))
+        })?;
+    }
+
+    // Legacy max upload size override (in MB)
+    if let Ok(mb_str) = std::env::var("KREUZBERG_MAX_UPLOAD_SIZE_MB") {
+        let mb = mb_str.parse::<usize>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_MAX_UPLOAD_SIZE_MB must be a valid usize, got '{}': {}",
+                mb_str, e
+            ))
+        })?;
+        *max_upload_mb = Some(mb);
+    }
+
+    Ok(())
+}

data/vendor/kreuzberg/src/core/server_config/loader.rs

@@ -0,0 +1,202 @@
+//! File loading logic for server configuration.
+//!
+//! This module provides functionality to load server configuration from various
+//! file formats (TOML, YAML, JSON) with support for both flat and nested formats.
+
+use crate::{KreuzbergError, Result};
+use serde::Deserialize;
+use std::path::Path;
+
+use super::ServerConfig;
+
+/// Load server configuration from a file.
+///
+/// Automatically detects the file format based on extension:
+/// - `.toml` - TOML format
+/// - `.yaml` or `.yml` - YAML format
+/// - `.json` - JSON format
+///
+/// This function handles two config file formats:
+/// 1. Flat format: Server config at root level
+/// 2. Nested format: Server config under `[server]` section (combined with ExtractionConfig)
+///
+/// # Arguments
+///
+/// * `path` - Path to the configuration file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if:
+/// - File doesn't exist or cannot be read
+/// - File extension is not recognized
+/// - File content is invalid for the detected format
+pub fn from_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
+        KreuzbergError::validation(format!(
+            "Cannot determine file format: no extension found in {}",
+            path.display()
+        ))
+    })?;
+
+    let mut config = match extension.to_lowercase().as_str() {
+        "toml" => from_toml_str(&content, path)?,
+        "yaml" | "yml" => from_yaml_str(&content, path)?,
+        "json" => from_json_str(&content, path)?,
+        _ => {
+            return Err(KreuzbergError::validation(format!(
+                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .yml, .json",
+                extension
+            )));
+        }
+    };
+
+    // Normalize legacy fields
+    config.normalize_legacy_fields();
+
+    Ok(config)
+}
+
+/// Load server configuration from a TOML file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the TOML file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid TOML.
+pub fn from_toml_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let mut config: ServerConfig = toml::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))?;
+
+    config.normalize_legacy_fields();
+
+    Ok(config)
+}
+
+/// Load server configuration from a YAML file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the YAML file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid YAML.
+pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let mut config: ServerConfig = serde_yaml_ng::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))?;
+
+    config.normalize_legacy_fields();
+
+    Ok(config)
+}
+
+/// Load server configuration from a JSON file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the JSON file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid JSON.
+pub fn from_json_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let mut config: ServerConfig = serde_json::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))?;
+
+    config.normalize_legacy_fields();
+
+    Ok(config)
+}
+
+// Helper functions for parsing different formats
+
+fn from_toml_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with [server] section)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = toml::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No [server] section, try flat format
+            return toml::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    toml::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))
+}
+
+fn from_yaml_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with server: section)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = serde_yaml_ng::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No server section, try flat format
+            return serde_yaml_ng::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    serde_yaml_ng::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))
+}
+
+fn from_json_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with "server" key)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = serde_json::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No server key, try flat format
+            return serde_json::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    serde_json::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))
+}

data/vendor/kreuzberg/src/core/server_config/mod.rs

@@ -0,0 +1,380 @@
+//! Server configuration for the Kreuzberg API.
+//!
+//! This module provides the `ServerConfig` struct for managing API server settings
+//! including host, port, CORS, and upload size limits. Configuration can be loaded
+//! from TOML, YAML, or JSON files and can be overridden by environment variables.
+//!
+//! # Features
+//!
+//! - **Multi-format support**: Load configuration from TOML, YAML, or JSON files
+//! - **Environment overrides**: All settings can be overridden via environment variables
+//! - **Backward compatibility**: Supports legacy `max_upload_mb` field for smooth migrations
+//! - **Sensible defaults**: All fields have reasonable defaults matching current behavior
+//! - **Flexible CORS**: Support for all origins (default) or specific origin lists
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::core::ServerConfig;
+//!
+//! # fn example() -> kreuzberg::Result<()> {
+//! // Create with defaults
+//! let mut config = ServerConfig::default();
+//!
+//! // Or load from file
+//! let mut config = ServerConfig::from_file("kreuzberg.toml")?;
+//!
+//! // Apply environment variable overrides
+//! config.apply_env_overrides()?;
+//!
+//! # Ok(())
+//! # }
+//! ```
+
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+
+mod env;
+mod loader;
+mod validation;
+
+#[cfg(test)]
+mod tests;
+
+/// Default host address for API server
+const DEFAULT_HOST: &str = "127.0.0.1";
+
+/// Default port for API server
+const DEFAULT_PORT: u16 = 8000;
+
+/// Default maximum request body size: 100 MB
+const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 104_857_600;
+
+/// Default maximum multipart field size: 100 MB
+const DEFAULT_MAX_MULTIPART_FIELD_BYTES: usize = 104_857_600;
+
+/// API server configuration.
+///
+/// This struct holds all configuration options for the Kreuzberg API server,
+/// including host/port settings, CORS configuration, and upload limits.
+///
+/// # Defaults
+///
+/// - `host`: "127.0.0.1" (localhost only)
+/// - `port`: 8000
+/// - `cors_origins`: empty vector (allows all origins)
+/// - `max_request_body_bytes`: 104_857_600 (100 MB)
+/// - `max_multipart_field_bytes`: 104_857_600 (100 MB)
+/// - `max_upload_mb`: None (legacy field, not used if other fields set)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ServerConfig {
+    /// Server host address (e.g., "127.0.0.1", "0.0.0.0")
+    #[serde(default = "default_host")]
+    pub host: String,
+
+    /// Server port number
+    #[serde(default = "default_port")]
+    pub port: u16,
+
+    /// CORS allowed origins. Empty vector means allow all origins.
+    ///
+    /// If this is an empty vector, the server will accept requests from any origin.
+    /// If populated with specific origins (e.g., ["https://example.com"]), only
+    /// those origins will be allowed.
+    #[serde(default)]
+    pub cors_origins: Vec<String>,
+
+    /// Maximum size of request body in bytes (default: 100 MB)
+    #[serde(default = "default_max_request_body_bytes")]
+    pub max_request_body_bytes: usize,
+
+    /// Maximum size of multipart fields in bytes (default: 100 MB)
+    #[serde(default = "default_max_multipart_field_bytes")]
+    pub max_multipart_field_bytes: usize,
+
+    /// Legacy upload size limit in MB (for backward compatibility).
+    ///
+    /// This field is deprecated and only used for backward compatibility.
+    /// If set, it will override `max_multipart_field_bytes` during normalization.
+    /// New configurations should use `max_multipart_field_bytes` directly.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_upload_mb: Option<usize>,
+}
+
+impl Default for ServerConfig {
+    fn default() -> Self {
+        Self {
+            host: default_host(),
+            port: default_port(),
+            cors_origins: Vec::new(),
+            max_request_body_bytes: default_max_request_body_bytes(),
+            max_multipart_field_bytes: default_max_multipart_field_bytes(),
+            max_upload_mb: None,
+        }
+    }
+}
+
+// Default value functions for serde
+fn default_host() -> String {
+    DEFAULT_HOST.to_string()
+}
+
+fn default_port() -> u16 {
+    DEFAULT_PORT
+}
+
+fn default_max_request_body_bytes() -> usize {
+    DEFAULT_MAX_REQUEST_BODY_BYTES
+}
+
+fn default_max_multipart_field_bytes() -> usize {
+    DEFAULT_MAX_MULTIPART_FIELD_BYTES
+}
+
+impl ServerConfig {
+    /// Create a new `ServerConfig` with default values.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Get the server listen address (host:port).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let config = ServerConfig::default();
+    /// assert_eq!(config.listen_addr(), "127.0.0.1:8000");
+    /// ```
+    pub fn listen_addr(&self) -> String {
+        format!("{}:{}", self.host, self.port)
+    }
+
+    /// Check if CORS allows all origins.
+    ///
+    /// Returns `true` if the `cors_origins` vector is empty, meaning all origins
+    /// are allowed. Returns `false` if specific origins are configured.
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert!(config.cors_allows_all());
+    ///
+    /// config.cors_origins.push("https://example.com".to_string());
+    /// assert!(!config.cors_allows_all());
+    /// ```
+    pub fn cors_allows_all(&self) -> bool {
+        self.cors_origins.is_empty()
+    }
+
+    /// Check if a given origin is allowed by CORS configuration.
+    ///
+    /// Returns `true` if:
+    /// - CORS allows all origins (empty origins list), or
+    /// - The given origin is in the allowed origins list
+    ///
+    /// # Arguments
+    ///
+    /// * `origin` - The origin to check (e.g., "https://example.com")
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert!(config.is_origin_allowed("https://example.com"));
+    ///
+    /// config.cors_origins.push("https://allowed.com".to_string());
+    /// assert!(config.is_origin_allowed("https://allowed.com"));
+    /// assert!(!config.is_origin_allowed("https://denied.com"));
+    /// ```
+    pub fn is_origin_allowed(&self, origin: &str) -> bool {
+        self.cors_origins.is_empty() || self.cors_origins.contains(&origin.to_string())
+    }
+
+    /// Get maximum request body size in megabytes (rounded up).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert_eq!(config.max_request_body_mb(), 100);
+    /// ```
+    pub fn max_request_body_mb(&self) -> usize {
+        self.max_request_body_bytes.div_ceil(1_048_576)
+    }
+
+    /// Get maximum multipart field size in megabytes (rounded up).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert_eq!(config.max_multipart_field_mb(), 100);
+    /// ```
+    pub fn max_multipart_field_mb(&self) -> usize {
+        self.max_multipart_field_bytes.div_ceil(1_048_576)
+    }
+
+    /// Normalize legacy field values for backward compatibility.
+    ///
+    /// If `max_upload_mb` is set, it will be converted to bytes and used to
+    /// override `max_multipart_field_bytes`. This allows old configurations
+    /// using the legacy field to continue working.
+    ///
+    /// This method is automatically called by `apply_env_overrides()`.
+    pub fn normalize_legacy_fields(&mut self) {
+        validation::normalize_legacy_fields(self.max_upload_mb, &mut self.max_multipart_field_bytes);
+    }
+
+    /// Apply environment variable overrides to the configuration.
+    ///
+    /// Reads the following environment variables and overrides config values if set:
+    ///
+    /// - `KREUZBERG_HOST` - Server host address
+    /// - `KREUZBERG_PORT` - Server port number (parsed as u16)
+    /// - `KREUZBERG_CORS_ORIGINS` - Comma-separated list of allowed origins
+    /// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` - Max request body size in bytes
+    /// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` - Max multipart field size in bytes
+    /// - `KREUZBERG_MAX_UPLOAD_SIZE_MB` - Max upload size in MB (legacy)
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - `KREUZBERG_PORT` cannot be parsed as u16
+    /// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` cannot be parsed as usize
+    /// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` cannot be parsed as usize
+    /// - `KREUZBERG_MAX_UPLOAD_SIZE_MB` cannot be parsed as usize
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// unsafe {
+    ///     std::env::set_var("KREUZBERG_HOST", "0.0.0.0");
+    ///     std::env::set_var("KREUZBERG_PORT", "3000");
+    /// }
+    ///
+    /// let mut config = ServerConfig::default();
+    /// config.apply_env_overrides()?;
+    ///
+    /// assert_eq!(config.host, "0.0.0.0");
+    /// assert_eq!(config.port, 3000);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn apply_env_overrides(&mut self) -> Result<()> {
+        env::apply_env_overrides(
+            &mut self.host,
+            &mut self.port,
+            &mut self.cors_origins,
+            &mut self.max_request_body_bytes,
+            &mut self.max_multipart_field_bytes,
+            &mut self.max_upload_mb,
+        )?;
+
+        // Apply legacy field normalization
+        self.normalize_legacy_fields();
+
+        Ok(())
+    }
+
+    /// Load server configuration from a file.
+    ///
+    /// Automatically detects the file format based on extension:
+    /// - `.toml` - TOML format
+    /// - `.yaml` or `.yml` - YAML format
+    /// - `.json` - JSON format
+    ///
+    /// This function handles two config file formats:
+    /// 1. Flat format: Server config at root level
+    /// 2. Nested format: Server config under `[server]` section (combined with ExtractionConfig)
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the configuration file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - File doesn't exist or cannot be read
+    /// - File extension is not recognized
+    /// - File content is invalid for the detected format
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// let config = ServerConfig::from_file("kreuzberg.toml")?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_file(path)
+    }
+
+    /// Load server configuration from a TOML file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the TOML file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid TOML.
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// let config = ServerConfig::from_toml_file("kreuzberg.toml")?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_toml_file(path)
+    }
+
+    /// Load server configuration from a YAML file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the YAML file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid YAML.
+    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_yaml_file(path)
+    }
+
+    /// Load server configuration from a JSON file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the JSON file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid JSON.
+    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_json_file(path)
+    }
+}