kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/src/api/handlers.rs

@@ -0,0 +1,199 @@
+//! API request handlers.
+
+use axum::{
+    Json,
+    extract::{Multipart, State},
+};
+
+use crate::{batch_extract_bytes, cache, extract_bytes};
+
+use super::{
+    error::ApiError,
+    types::{ApiState, CacheClearResponse, CacheStatsResponse, ExtractResponse, HealthResponse, InfoResponse},
+};
+
+/// Extract endpoint handler.
+///
+/// POST /extract
+///
+/// Accepts multipart form data with:
+/// - `files`: One or more files to extract
+/// - `config` (optional): JSON extraction configuration (overrides server defaults)
+///
+/// Returns a list of extraction results, one per file.
+///
+/// # Size Limits
+///
+/// Request body size limits are enforced at the router layer via `RequestBodyLimitLayer`.
+/// Default limits:
+/// - Total request body: 100 MB (all files + form data combined)
+/// - Individual multipart fields: Controlled by Axum's default multipart limits
+///
+/// If a request exceeds the size limit, it will be rejected with HTTP 413 (Payload Too Large).
+///
+/// The server's default config (loaded from kreuzberg.toml/yaml/json via discovery)
+/// is used as the base, and any per-request config overrides those defaults.
+pub async fn extract_handler(
+    State(state): State<ApiState>,
+    mut multipart: Multipart,
+) -> Result<Json<ExtractResponse>, ApiError> {
+    let mut files = Vec::new();
+    let mut config = (*state.default_config).clone();
+
+    while let Some(field) = multipart
+        .next_field()
+        .await
+        .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?
+    {
+        let field_name = field.name().unwrap_or("").to_string();
+
+        match field_name.as_str() {
+            "files" => {
+                let file_name = field.file_name().map(|s| s.to_string());
+                let content_type = field.content_type().map(|s| s.to_string());
+                let data = field
+                    .bytes()
+                    .await
+                    .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?;
+
+                let mime_type = content_type.unwrap_or_else(|| "application/octet-stream".to_string());
+
+                files.push((data.to_vec(), mime_type, file_name));
+            }
+            "config" => {
+                let config_str = field
+                    .text()
+                    .await
+                    .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?;
+
+                config = serde_json::from_str(&config_str).map_err(|e| {
+                    ApiError::validation(crate::error::KreuzbergError::validation(format!(
+                        "Invalid extraction configuration: {}",
+                        e
+                    )))
+                })?;
+            }
+            _ => {}
+        }
+    }
+
+    if files.is_empty() {
+        return Err(ApiError::validation(crate::error::KreuzbergError::validation(
+            "No files provided for extraction",
+        )));
+    }
+
+    if files.len() == 1 {
+        let (data, mime_type, _file_name) = files
+            .into_iter()
+            .next()
+            .expect("files.len() == 1 guarantees one element exists");
+        let result = extract_bytes(&data, mime_type.as_str(), &config).await?;
+        return Ok(Json(vec![result]));
+    }
+
+    let files_data: Vec<(Vec<u8>, String)> = files.into_iter().map(|(data, mime, _name)| (data, mime)).collect();
+
+    let file_refs: Vec<(&[u8], &str)> = files_data
+        .iter()
+        .map(|(data, mime)| (data.as_slice(), mime.as_str()))
+        .collect();
+
+    let results = batch_extract_bytes(file_refs, &config).await?;
+    Ok(Json(results))
+}
+
+/// Health check endpoint handler.
+///
+/// GET /health
+pub async fn health_handler() -> Json<HealthResponse> {
+    Json(HealthResponse {
+        status: "healthy".to_string(),
+        version: env!("CARGO_PKG_VERSION").to_string(),
+    })
+}
+
+/// Server info endpoint handler.
+///
+/// GET /info
+pub async fn info_handler() -> Json<InfoResponse> {
+    Json(InfoResponse {
+        version: env!("CARGO_PKG_VERSION").to_string(),
+        rust_backend: true,
+    })
+}
+
+/// Cache stats endpoint handler.
+///
+/// GET /cache/stats
+///
+/// # Errors
+///
+/// Returns `ApiError::Internal` if:
+/// - Current directory cannot be determined
+/// - Cache directory path contains non-UTF8 characters
+/// - Cache metadata retrieval fails
+pub async fn cache_stats_handler() -> Result<Json<CacheStatsResponse>, ApiError> {
+    let cache_dir = std::env::current_dir()
+        .map_err(|e| {
+            ApiError::internal(crate::error::KreuzbergError::Other(format!(
+                "Failed to get current directory: {}",
+                e
+            )))
+        })?
+        .join(".kreuzberg");
+
+    let cache_dir_str = cache_dir.to_str().ok_or_else(|| {
+        ApiError::internal(crate::error::KreuzbergError::Other(format!(
+            "Cache directory path contains non-UTF8 characters: {}",
+            cache_dir.display()
+        )))
+    })?;
+
+    let stats = cache::get_cache_metadata(cache_dir_str).map_err(ApiError::internal)?;
+
+    Ok(Json(CacheStatsResponse {
+        directory: cache_dir.to_string_lossy().to_string(),
+        total_files: stats.total_files,
+        total_size_mb: stats.total_size_mb,
+        available_space_mb: stats.available_space_mb,
+        oldest_file_age_days: stats.oldest_file_age_days,
+        newest_file_age_days: stats.newest_file_age_days,
+    }))
+}
+
+/// Cache clear endpoint handler.
+///
+/// DELETE /cache/clear
+///
+/// # Errors
+///
+/// Returns `ApiError::Internal` if:
+/// - Current directory cannot be determined
+/// - Cache directory path contains non-UTF8 characters
+/// - Cache clearing operation fails
+pub async fn cache_clear_handler() -> Result<Json<CacheClearResponse>, ApiError> {
+    let cache_dir = std::env::current_dir()
+        .map_err(|e| {
+            ApiError::internal(crate::error::KreuzbergError::Other(format!(
+                "Failed to get current directory: {}",
+                e
+            )))
+        })?
+        .join(".kreuzberg");
+
+    let cache_dir_str = cache_dir.to_str().ok_or_else(|| {
+        ApiError::internal(crate::error::KreuzbergError::Other(format!(
+            "Cache directory path contains non-UTF8 characters: {}",
+            cache_dir.display()
+        )))
+    })?;
+
+    let (removed_files, freed_mb) = cache::clear_cache_directory(cache_dir_str).map_err(ApiError::internal)?;
+
+    Ok(Json(CacheClearResponse {
+        directory: cache_dir.to_string_lossy().to_string(),
+        removed_files,
+        freed_mb,
+    }))
+}

data/vendor/kreuzberg/src/api/mod.rs

@@ -0,0 +1,79 @@
+//! REST API server for Kreuzberg document extraction.
+//!
+//! This module provides an Axum-based HTTP server for document extraction
+//! with endpoints for single and batch extraction operations.
+//!
+//! # Endpoints
+//!
+//! - `POST /extract` - Extract text from uploaded files (multipart form data)
+//! - `GET /health` - Health check endpoint
+//! - `GET /info` - Server information
+//!
+//! # Examples
+//!
+//! ## Starting the server
+//!
+//! ```no_run
+//! use kreuzberg::api::serve;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Local development
+//!     serve("127.0.0.1", 8000).await?;
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Embedding the router in your app
+//!
+//! ```no_run
+//! use kreuzberg::{ExtractionConfig, api::create_router};
+//! use axum::Router;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Load config (from file or use default)
+//!     let config = ExtractionConfig::default();
+//!     let kreuzberg_router = create_router(config);
+//!
+//!     // Nest under /api prefix
+//!     let app = Router::new().nest("/api", kreuzberg_router);
+//!
+//!     // Add your own routes
+//!     // ...
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # cURL Examples
+//!
+//! ```bash
+//! # Single file extraction
+//! curl -F "files=@document.pdf" http://localhost:8000/extract
+//!
+//! # Multiple files with OCR config
+//! curl -F "files=@doc1.pdf" -F "files=@doc2.jpg" \
+//!      -F 'config={"ocr":{"language":"eng"}}' \
+//!      http://localhost:8000/extract
+//!
+//! # Health check
+//! curl http://localhost:8000/health
+//!
+//! # Server info
+//! curl http://localhost:8000/info
+//! ```
+
+mod error;
+mod handlers;
+mod server;
+mod types;
+
+pub use error::ApiError;
+pub use server::{
+    create_router, create_router_with_limits, serve, serve_default, serve_with_config, serve_with_config_and_limits,
+};
+pub use types::{
+    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, ErrorResponse, ExtractResponse, HealthResponse,
+    InfoResponse,
+};

data/vendor/kreuzberg/src/api/server.rs

@@ -0,0 +1,353 @@
+//! API server setup and configuration.
+
+use std::{
+    net::{IpAddr, SocketAddr},
+    sync::Arc,
+};
+
+use axum::{
+    Router,
+    routing::{delete, get, post},
+};
+use tower_http::{
+    cors::{AllowOrigin, Any, CorsLayer},
+    limit::RequestBodyLimitLayer,
+    trace::TraceLayer,
+};
+
+use crate::{ExtractionConfig, Result};
+
+use super::{
+    handlers::{cache_clear_handler, cache_stats_handler, extract_handler, health_handler, info_handler},
+    types::{ApiSizeLimits, ApiState},
+};
+
+/// Parse size limits from environment variables.
+///
+/// Reads `KREUZBERG_MAX_UPLOAD_SIZE_MB` to configure upload size limits.
+/// Falls back to default (100 MB) if not set or invalid.
+fn parse_size_limits_from_env() -> ApiSizeLimits {
+    match std::env::var("KREUZBERG_MAX_UPLOAD_SIZE_MB") {
+        Ok(value) => match value.parse::<usize>() {
+            Ok(mb) if mb > 0 => {
+                tracing::info!(
+                    "Upload size limit configured from environment: {} MB ({} bytes)",
+                    mb,
+                    mb * 1024 * 1024
+                );
+                ApiSizeLimits::from_mb(mb, mb)
+            }
+            Ok(_) => {
+                tracing::warn!("Invalid KREUZBERG_MAX_UPLOAD_SIZE_MB value (must be > 0), using default 100 MB");
+                let limits = ApiSizeLimits::default();
+                tracing::info!(
+                    "Upload size limit: 100 MB (default, {} bytes)",
+                    limits.max_request_body_bytes
+                );
+                limits
+            }
+            Err(e) => {
+                tracing::warn!(
+                    "Failed to parse KREUZBERG_MAX_UPLOAD_SIZE_MB='{}': {}, using default 100 MB",
+                    value,
+                    e
+                );
+                let limits = ApiSizeLimits::default();
+                tracing::info!(
+                    "Upload size limit: 100 MB (default, {} bytes)",
+                    limits.max_request_body_bytes
+                );
+                limits
+            }
+        },
+        Err(_) => {
+            let limits = ApiSizeLimits::default();
+            tracing::info!(
+                "Upload size limit: 100 MB (default, {} bytes)",
+                limits.max_request_body_bytes
+            );
+            limits
+        }
+    }
+}
+
+/// Create the API router with all routes configured.
+///
+/// This is public to allow users to embed the router in their own applications.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::create_router};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with default config and size limits
+/// let config = ExtractionConfig::default();
+/// let router = create_router(config);
+/// # }
+/// ```
+pub fn create_router(config: ExtractionConfig) -> Router {
+    create_router_with_limits(config, ApiSizeLimits::default())
+}
+
+/// Create the API router with custom size limits.
+///
+/// This allows fine-grained control over request body and multipart field size limits.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+/// * `limits` - Size limits for request bodies and multipart uploads.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with 50 MB limits
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(50, 50);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+/// use tower_http::limit::RequestBodyLimitLayer;
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Custom limits for very large documents (500 MB)
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(500, 500);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+pub fn create_router_with_limits(config: ExtractionConfig, limits: ApiSizeLimits) -> Router {
+    let state = ApiState {
+        default_config: Arc::new(config),
+    };
+
+    // SECURITY WARNING: The default allows all origins for development convenience,
+    let cors_layer = if let Ok(origins_str) = std::env::var("KREUZBERG_CORS_ORIGINS") {
+        let origins: Vec<_> = origins_str
+            .split(',')
+            .filter(|s| !s.trim().is_empty())
+            .filter_map(|s| s.trim().parse::<axum::http::HeaderValue>().ok())
+            .collect();
+
+        if !origins.is_empty() {
+            tracing::info!("CORS configured with {} explicit allowed origin(s)", origins.len());
+            CorsLayer::new()
+                .allow_origin(AllowOrigin::list(origins))
+                .allow_methods(Any)
+                .allow_headers(Any)
+        } else {
+            tracing::warn!(
+                "KREUZBERG_CORS_ORIGINS set but empty/invalid - falling back to permissive CORS. \
+                 This allows CSRF attacks. Set explicit origins for production."
+            );
+            CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+        }
+    } else {
+        tracing::warn!(
+            "CORS configured to allow all origins (default). This permits CSRF attacks. \
+             For production, set KREUZBERG_CORS_ORIGINS environment variable to comma-separated \
+             list of allowed origins (e.g., 'https://app.example.com,https://api.example.com')"
+        );
+        CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+    };
+
+    Router::new()
+        .route("/extract", post(extract_handler))
+        .route("/health", get(health_handler))
+        .route("/info", get(info_handler))
+        .route("/cache/stats", get(cache_stats_handler))
+        .route("/cache/clear", delete(cache_clear_handler))
+        .layer(RequestBodyLimitLayer::new(limits.max_request_body_bytes))
+        .layer(cors_layer)
+        .layer(TraceLayer::new_for_http())
+        .with_state(state)
+}
+
+/// Start the API server with config file discovery.
+///
+/// Searches for kreuzberg.toml/yaml/json in current and parent directories.
+/// If no config file is found, uses default configuration.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::api::serve;
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     // Local development
+///     serve("127.0.0.1", 8000).await?;
+///     Ok(())
+/// }
+/// ```
+///
+/// ```no_run
+/// use kreuzberg::api::serve;
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     // Docker/production (listen on all interfaces)
+///     serve("0.0.0.0", 8000).await?;
+///     Ok(())
+/// }
+/// ```
+///
+/// # Environment Variables
+///
+/// ```bash
+/// # Python/Docker usage
+/// export KREUZBERG_HOST=0.0.0.0
+/// export KREUZBERG_PORT=8000
+///
+/// # CORS configuration (IMPORTANT for production security)
+/// # Default: allows all origins (permits CSRF attacks)
+/// # Production: set to comma-separated list of allowed origins
+/// export KREUZBERG_CORS_ORIGINS="https://app.example.com,https://api.example.com"
+///
+/// # Upload size limit (default: 100 MB)
+/// export KREUZBERG_MAX_UPLOAD_SIZE_MB=200
+///
+/// python -m kreuzberg.api
+/// ```
+pub async fn serve(host: impl AsRef<str>, port: u16) -> Result<()> {
+    let config = match ExtractionConfig::discover()? {
+        Some(config) => {
+            tracing::info!("Loaded extraction config from discovered file");
+            config
+        }
+        None => {
+            tracing::info!("No config file found, using default configuration");
+            ExtractionConfig::default()
+        }
+    };
+
+    let limits = parse_size_limits_from_env();
+
+    serve_with_config_and_limits(host, port, config, limits).await
+}
+
+/// Start the API server with explicit config.
+///
+/// Uses default size limits (100 MB). For custom limits, use `serve_with_config_and_limits`.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+/// * `config` - Default extraction configuration for all requests
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::serve_with_config};
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     let config = ExtractionConfig::from_toml_file("config/kreuzberg.toml")?;
+///     serve_with_config("127.0.0.1", 8000, config).await?;
+///     Ok(())
+/// }
+/// ```
+pub async fn serve_with_config(host: impl AsRef<str>, port: u16, config: ExtractionConfig) -> Result<()> {
+    let limits = ApiSizeLimits::default();
+    tracing::info!(
+        "Upload size limit: 100 MB (default, {} bytes)",
+        limits.max_request_body_bytes
+    );
+    serve_with_config_and_limits(host, port, config, limits).await
+}
+
+/// Start the API server with explicit config and size limits.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+/// * `config` - Default extraction configuration for all requests
+/// * `limits` - Size limits for request bodies and multipart uploads
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::{serve_with_config_and_limits, ApiSizeLimits}};
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     let config = ExtractionConfig::from_toml_file("config/kreuzberg.toml")?;
+///     let limits = ApiSizeLimits::from_mb(200, 200);
+///     serve_with_config_and_limits("127.0.0.1", 8000, config, limits).await?;
+///     Ok(())
+/// }
+/// ```
+pub async fn serve_with_config_and_limits(
+    host: impl AsRef<str>,
+    port: u16,
+    config: ExtractionConfig,
+    limits: ApiSizeLimits,
+) -> Result<()> {
+    let ip: IpAddr = host
+        .as_ref()
+        .parse()
+        .map_err(|e| crate::error::KreuzbergError::validation(format!("Invalid host address: {}", e)))?;
+
+    let addr = SocketAddr::new(ip, port);
+    let app = create_router_with_limits(config, limits);
+
+    tracing::info!("Starting Kreuzberg API server on http://{}:{}", ip, port);
+
+    let listener = tokio::net::TcpListener::bind(addr)
+        .await
+        .map_err(crate::error::KreuzbergError::Io)?;
+
+    axum::serve(listener, app)
+        .await
+        .map_err(|e| crate::error::KreuzbergError::Other(e.to_string()))?;
+
+    Ok(())
+}
+
+/// Start the API server with default host and port.
+///
+/// Defaults: host = "127.0.0.1", port = 8000
+///
+/// Uses config file discovery (searches current/parent directories for kreuzberg.toml/yaml/json).
+pub async fn serve_default() -> Result<()> {
+    serve("127.0.0.1", 8000).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_router() {
+        let config = ExtractionConfig::default();
+        let _router = create_router(config);
+    }
+
+    #[test]
+    fn test_router_has_routes() {
+        let config = ExtractionConfig::default();
+        let router = create_router(config);
+        assert!(size_of_val(&router) > 0);
+    }
+}