kreuzberg 4.0.8 → 4.1.0

data/sig/kreuzberg.rbs

@@ -250,7 +250,56 @@ module Kreuzberg
     tables: Array[table_hash]?,
     detected_languages: Array[String]?,
     chunks: Array[chunk_hash]?,
-    images: Array[image_hash]?
+    images: Array[image_hash]?,
+    elements: Array[element_hash]?,
+    djot_content: djot_content_hash?
+  }
+
+  type djot_content_hash = {
+    plain_text: String,
+    blocks: Array[formatted_block_hash],
+    metadata_json: String,
+    tables: Array[table_hash],
+    images: Array[djot_image_hash],
+    links: Array[djot_link_hash],
+    footnotes: Array[footnote_hash],
+    attributes: Hash[String, attributes_hash]?
+  }
+
+  type formatted_block_hash = {
+    block_type: String,
+    level: Integer?,
+    content: String?,
+    children: Array[formatted_block_hash]?,
+    attributes: attributes_hash?
+  }
+
+  type djot_image_hash = {
+    url: String,
+    alt: String?,
+    title: String?,
+    attributes: attributes_hash?
+  }
+
+  type djot_link_hash = {
+    url: String,
+    text: String,
+    title: String?,
+    link_type: String?
+  }
+
+  type footnote_hash = {
+    label: String,
+    content: String
+  }
+
+  type attributes_hash = Hash[String, String | Integer | bool | Array[String] | nil]
+
+  type element_hash = {
+    element_id: String,
+    element_type: String,
+    text: String,
+    metadata: Hash[String, untyped]?
   }
 
   type table_hash = {
@@ -359,6 +408,60 @@ module Kreuzberg
       def to_h: () -> image_hash
     end
 
+    # Structured Djot document representation
+    class DjotContent
+      attr_reader plain_text: String
+      attr_reader blocks: Array[DjotContent::FormattedBlock]
+      attr_reader metadata: Hash[untyped, untyped]
+      attr_reader tables: Array[Table]
+      attr_reader images: Array[DjotContent::DjotImage]
+      attr_reader links: Array[DjotContent::DjotLink]
+      attr_reader footnotes: Array[DjotContent::Footnote]
+      attr_reader attributes: Hash[String, untyped]?
+
+      def initialize: (djot_content_hash hash) -> void
+      def to_h: () -> djot_content_hash
+
+      class FormattedBlock
+        attr_reader block_type: String
+        attr_reader level: Integer?
+        attr_reader content: String?
+        attr_reader children: Array[FormattedBlock]?
+        attr_reader attributes: Hash[String, untyped]?
+
+        def initialize: (formatted_block_hash hash) -> void
+        def to_h: () -> formatted_block_hash
+      end
+
+      class DjotImage
+        attr_reader url: String
+        attr_reader alt: String?
+        attr_reader title: String?
+        attr_reader attributes: Hash[String, untyped]?
+
+        def initialize: (djot_image_hash hash) -> void
+        def to_h: () -> djot_image_hash
+      end
+
+      class DjotLink
+        attr_reader url: String
+        attr_reader text: String
+        attr_reader title: String?
+        attr_reader link_type: String?
+
+        def initialize: (djot_link_hash hash) -> void
+        def to_h: () -> djot_link_hash
+      end
+
+      class Footnote
+        attr_reader label: String
+        attr_reader content: String
+
+        def initialize: (label: String, content: String) -> void
+        def to_h: () -> footnote_hash
+      end
+    end
+
     attr_reader content: String
     attr_reader mime_type: String
     attr_reader metadata: Hash[untyped, untyped]
@@ -367,6 +470,7 @@ module Kreuzberg
     attr_reader detected_languages: Array[String]?
     attr_reader chunks: Array[Chunk]?
     attr_reader images: Array[Image]?
+    attr_reader djot_content: DjotContent?
 
     def initialize: (extraction_result_hash hash) -> void
     def to_h: () -> Hash[Symbol, untyped]

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
 resolver = "2"
 
 [workspace.package]
-version = "4.0.8"
+version = "4.1.0"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -29,7 +29,7 @@ serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.149"
 
 # Error handling
-thiserror = "2.0.17"
+thiserror = "2.0.18"
 anyhow = "1.0"
 
 # Async utilities
@@ -47,7 +47,7 @@ hex = "0.4.3"
 toml = "0.9.11"
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
-html-to-markdown-rs = { version = "2.22.5", default-features = false }
+html-to-markdown-rs = { version = "2.23.4", default-features = false }
 reqwest = { version = "0.13.1", default-features = false, features = ["json", "rustls"] }
 image = { version = "0.25.9", default-features = false }
 lzma-rust2 = { version = "0.15.7" }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.0.8"
+version = "4.1.0"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -136,6 +136,7 @@ regex = "1.12.2"
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml_ng = "0.10.0"
+jotdown = "0.9"
 toml = { workspace = true }
 mime_guess = "2.0"
 rmp-serde = "1.3"
@@ -152,7 +153,7 @@ lopdf = { version = "0.39.0", optional = true }
 calamine = { version = "0.32.0", features = ["dates"], optional = true }
 polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
 roxmltree = { version = "0.21.1", optional = true }
-zip = { version = "7.1.0", optional = true }
+zip = { version = "7.2.0", optional = true }
 mail-parser = { version = "0.11.1", optional = true }
 msg_parser = { version = "0.1.1", optional = true }
 html-to-markdown-rs = { workspace = true, features = [
@@ -215,7 +216,7 @@ smartcore = { version = "0.4", default-features = false, features = ["serde"] }
 tempfile = { workspace = true }
 filetime = "0.2"
 tar = "0.4.44"
-zip = "7.1.0"
+zip = "7.2.0"
 serial_test = "3.3.1"
 anyhow = { workspace = true }
 tokio-test = "0.4"

data/vendor/kreuzberg/README.md

@@ -17,7 +17,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.0.0 Release Candidate**
+> **🚀 Version 4.1.0 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/api/config.rs

@@ -0,0 +1,69 @@
+//! API server configuration loading.
+
+use crate::{Result, core::ServerConfig};
+
+/// Load ServerConfig with proper precedence order.
+///
+/// This function implements the configuration hierarchy:
+/// 1. File (if provided)
+/// 2. Environment variables (via apply_env_overrides)
+/// 3. Defaults
+///
+/// The config file can be in flat format (server settings at root) or nested format
+/// (server settings under [server] section alongside other configs like [ocr]).
+///
+/// # Arguments
+///
+/// * `config_path` - Optional path to a ServerConfig file (TOML, YAML, or JSON)
+///
+/// # Returns
+///
+/// A configured ServerConfig with proper precedence applied.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The config file path is provided but cannot be read
+/// - The config file contains invalid server configuration
+/// - Environment variable overrides contain invalid values
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::api::load_server_config;
+/// use std::path::Path;
+///
+/// # fn example() -> kreuzberg::Result<()> {
+/// // Load from file with env overrides
+/// let config = load_server_config(Some(Path::new("server.toml")))?;
+///
+/// // Or use defaults with env overrides
+/// let config = load_server_config(None)?;
+/// # Ok(())
+/// # }
+/// ```
+pub fn load_server_config(config_path: Option<&std::path::Path>) -> Result<ServerConfig> {
+    let mut config = if let Some(path) = config_path {
+        ServerConfig::from_file(path)?
+    } else {
+        ServerConfig::default()
+    };
+
+    // Apply environment variable overrides with proper logging
+    config.apply_env_overrides()?;
+
+    tracing::info!(
+        "Server configuration loaded: host={}, port={}, request_body_limit={} MB, multipart_field_limit={} MB, CORS={}",
+        config.host,
+        config.port,
+        config.max_request_body_mb(),
+        config.max_multipart_field_mb(),
+        if config.cors_allows_all() {
+            "allow all origins".to_string()
+        } else {
+            format!("{} specific origins", config.cors_origins.len())
+        }
+    );
+
+    Ok(config)
+}

data/vendor/kreuzberg/src/api/handlers.rs

@@ -10,8 +10,8 @@ use crate::{batch_extract_bytes, cache, extract_bytes};
 use super::{
     error::ApiError,
     types::{
-        ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ExtractResponse, HealthResponse,
-        InfoResponse,
+        ApiState, CacheClearResponse, CacheStatsResponse, ChunkRequest, ChunkResponse, EmbedRequest, EmbedResponse,
+        ExtractResponse, HealthResponse, InfoResponse,
     },
 };
 
@@ -85,6 +85,25 @@ pub async fn extract_handler(
                     )))
                 })?;
             }
+            "output_format" => {
+                let format_str = field
+                    .text()
+                    .await
+                    .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?;
+
+                config.output_format = match format_str.to_lowercase().as_str() {
+                    "plain" => crate::core::config::OutputFormat::Plain,
+                    "markdown" => crate::core::config::OutputFormat::Markdown,
+                    "djot" => crate::core::config::OutputFormat::Djot,
+                    "html" => crate::core::config::OutputFormat::Html,
+                    _ => {
+                        return Err(ApiError::validation(crate::error::KreuzbergError::validation(format!(
+                            "Invalid output_format: '{}'. Valid values: 'plain', 'markdown', 'djot', 'html'",
+                            format_str
+                        ))));
+                    }
+                };
+            }
             _ => {}
         }
     }
@@ -318,3 +337,81 @@ pub async fn embed_handler(Json(_request): Json<EmbedRequest>) -> Result<Json<Em
         "Embeddings feature is not enabled. Rebuild with --features embeddings".to_string(),
     )))
 }
+
+/// Chunk text endpoint handler.
+///
+/// POST /chunk
+///
+/// Accepts JSON body with text and optional configuration.
+/// Returns chunks with metadata.
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(
+        name = "api.chunk",
+        skip(request),
+        fields(text_length = request.text.len(), chunker_type = request.chunker_type.as_str())
+    )
+)]
+pub async fn chunk_handler(Json(request): Json<ChunkRequest>) -> Result<Json<ChunkResponse>, ApiError> {
+    use super::types::{ChunkItem, ChunkingConfigResponse};
+    use crate::chunking::{ChunkerType, ChunkingConfig, chunk_text};
+
+    // Validate input
+    if request.text.is_empty() {
+        return Err(ApiError::validation(crate::error::KreuzbergError::validation(
+            "Text cannot be empty",
+        )));
+    }
+
+    // Parse chunker_type
+    let chunker_type = match request.chunker_type.to_lowercase().as_str() {
+        "text" | "" => ChunkerType::Text,
+        "markdown" => ChunkerType::Markdown,
+        other => {
+            return Err(ApiError::validation(crate::error::KreuzbergError::validation(format!(
+                "Invalid chunker_type: '{}'. Valid values: 'text', 'markdown'",
+                other
+            ))));
+        }
+    };
+
+    // Build config with defaults
+    let cfg = request.config.unwrap_or_default();
+    let config = ChunkingConfig {
+        max_characters: cfg.max_characters.unwrap_or(2000),
+        overlap: cfg.overlap.unwrap_or(100),
+        trim: cfg.trim.unwrap_or(true),
+        chunker_type,
+    };
+
+    // Perform chunking
+    let result = chunk_text(&request.text, &config, None).map_err(ApiError::internal)?;
+
+    // Transform to response
+    let chunks = result
+        .chunks
+        .into_iter()
+        .map(|chunk| ChunkItem {
+            content: chunk.content,
+            byte_start: chunk.metadata.byte_start,
+            byte_end: chunk.metadata.byte_end,
+            chunk_index: chunk.metadata.chunk_index,
+            total_chunks: chunk.metadata.total_chunks,
+            first_page: chunk.metadata.first_page,
+            last_page: chunk.metadata.last_page,
+        })
+        .collect();
+
+    Ok(Json(ChunkResponse {
+        chunks,
+        chunk_count: result.chunk_count,
+        config: ChunkingConfigResponse {
+            max_characters: config.max_characters,
+            overlap: config.overlap,
+            trim: config.trim,
+            chunker_type: format!("{:?}", config.chunker_type).to_lowercase(),
+        },
+        input_size_bytes: request.text.len(),
+        chunker_type: request.chunker_type.to_lowercase(),
+    }))
+}

data/vendor/kreuzberg/src/api/mod.rs

@@ -7,6 +7,7 @@
 //!
 //! - `POST /extract` - Extract text from uploaded files (multipart form data)
 //! - `POST /embed` - Generate embeddings for text (JSON body with texts array)
+//! - `POST /chunk` - Chunk text into smaller pieces (JSON body with text and config)
 //! - `GET /health` - Health check endpoint
 //! - `GET /info` - Server information
 //! - `GET /cache/stats` - Get cache statistics
@@ -76,19 +77,25 @@
 //! curl -X POST http://localhost:8000/embed \
 //!      -H "Content-Type: application/json" \
 //!      -d '{"texts":["Hello world","Second text"]}'
+//!
+//! # Chunk text
+//! curl -X POST http://localhost:8000/chunk \
+//!      -H "Content-Type: application/json" \
+//!      -d '{"text":"Long text to chunk...","chunker_type":"text"}'
 //! ```
 
+mod config;
 mod error;
 mod handlers;
-mod server;
+mod router;
+mod startup;
 mod types;
 
+pub use config::load_server_config;
 pub use error::ApiError;
-pub use server::{
-    create_router, create_router_with_limits, create_router_with_limits_and_server_config, load_server_config, serve,
-    serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config,
-};
+pub use router::{create_router, create_router_with_limits, create_router_with_limits_and_server_config};
+pub use startup::{serve, serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config};
 pub use types::{
-    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ErrorResponse,
-    ExtractResponse, HealthResponse, InfoResponse,
+    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, ChunkRequest, ChunkResponse, EmbedRequest,
+    EmbedResponse, ErrorResponse, ExtractResponse, HealthResponse, InfoResponse,
 };

data/vendor/kreuzberg/src/api/router.rs

@@ -0,0 +1,214 @@
+//! API router setup and configuration.
+
+use std::sync::Arc;
+
+use axum::{
+    Router,
+    extract::DefaultBodyLimit,
+    routing::{delete, get, post},
+};
+use tower_http::{
+    cors::{AllowOrigin, Any, CorsLayer},
+    limit::RequestBodyLimitLayer,
+    trace::TraceLayer,
+};
+
+use crate::{ExtractionConfig, core::ServerConfig};
+
+use super::{
+    handlers::{
+        cache_clear_handler, cache_stats_handler, chunk_handler, embed_handler, extract_handler, health_handler,
+        info_handler,
+    },
+    types::{ApiSizeLimits, ApiState},
+};
+
+/// Create the API router with all routes configured.
+///
+/// This is public to allow users to embed the router in their own applications.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::create_router};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with default config and size limits
+/// let config = ExtractionConfig::default();
+/// let router = create_router(config);
+/// # }
+/// ```
+pub fn create_router(config: ExtractionConfig) -> Router {
+    create_router_with_limits(config, ApiSizeLimits::default())
+}
+
+/// Create the API router with custom size limits.
+///
+/// This allows fine-grained control over request body and multipart field size limits.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+/// * `limits` - Size limits for request bodies and multipart uploads.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with 50 MB limits
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(50, 50);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+/// use tower_http::limit::RequestBodyLimitLayer;
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Custom limits for very large documents (500 MB)
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(500, 500);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+pub fn create_router_with_limits(config: ExtractionConfig, limits: ApiSizeLimits) -> Router {
+    create_router_with_limits_and_server_config(config, limits, ServerConfig::default())
+}
+
+/// Create the API router with custom size limits and server configuration.
+///
+/// This function provides full control over request limits, CORS, and server settings via ServerConfig.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+/// * `limits` - Size limits for request bodies and multipart uploads.
+/// * `server_config` - Server configuration including host, port, and CORS settings.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::create_router_with_limits, core::ServerConfig};
+///
+/// # #[tokio::main]
+/// # async fn main() -> kreuzberg::Result<()> {
+/// let extraction_config = ExtractionConfig::default();
+/// let mut server_config = ServerConfig::default();
+/// server_config.cors_origins = vec!["https://example.com".to_string()];
+/// let router = create_router_with_limits_and_server_config(
+///     extraction_config,
+///     Default::default(),
+///     server_config
+/// );
+/// # Ok(())
+/// # }
+/// ```
+pub fn create_router_with_limits_and_server_config(
+    config: ExtractionConfig,
+    limits: ApiSizeLimits,
+    server_config: ServerConfig,
+) -> Router {
+    let state = ApiState {
+        default_config: Arc::new(config),
+    };
+
+    // CORS configuration based on ServerConfig
+    let cors_layer = if server_config.cors_allows_all() {
+        tracing::warn!(
+            "CORS configured to allow all origins (default). This permits CSRF attacks. \
+             For production, set KREUZBERG_CORS_ORIGINS environment variable to comma-separated \
+             list of allowed origins (e.g., 'https://app.example.com,https://api.example.com')"
+        );
+        CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+    } else {
+        let origins: Vec<_> = server_config
+            .cors_origins
+            .iter()
+            .filter_map(|s| s.trim().parse::<axum::http::HeaderValue>().ok())
+            .collect();
+
+        if !origins.is_empty() {
+            tracing::info!("CORS configured with {} explicit allowed origin(s)", origins.len());
+            CorsLayer::new()
+                .allow_origin(AllowOrigin::list(origins))
+                .allow_methods(Any)
+                .allow_headers(Any)
+        } else {
+            tracing::warn!(
+                "CORS origins configured but empty/invalid - falling back to permissive CORS. \
+                 This allows CSRF attacks. Set explicit origins for production."
+            );
+            CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+        }
+    };
+
+    Router::new()
+        .route("/extract", post(extract_handler))
+        .route("/embed", post(embed_handler))
+        .route("/chunk", post(chunk_handler))
+        .route("/health", get(health_handler))
+        .route("/info", get(info_handler))
+        .route("/cache/stats", get(cache_stats_handler))
+        .route("/cache/clear", delete(cache_clear_handler))
+        .layer(DefaultBodyLimit::max(limits.max_request_body_bytes))
+        .layer(RequestBodyLimitLayer::new(limits.max_request_body_bytes))
+        .layer(cors_layer)
+        .layer(TraceLayer::new_for_http())
+        .with_state(state)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_router() {
+        let config = ExtractionConfig::default();
+        let _router = create_router(config);
+    }
+
+    #[test]
+    fn test_router_has_routes() {
+        use std::mem::size_of_val;
+        let config = ExtractionConfig::default();
+        let router = create_router(config);
+        assert!(size_of_val(&router) > 0);
+    }
+
+    #[test]
+    fn test_create_router_with_limits() {
+        let config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::from_mb(50, 50);
+        let _router = create_router_with_limits(config, limits);
+    }
+
+    #[test]
+    fn test_create_router_with_server_config() {
+        let extraction_config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::from_mb(100, 100);
+        let server_config = ServerConfig::default();
+        let _router = create_router_with_limits_and_server_config(extraction_config, limits, server_config);
+    }
+
+    #[test]
+    fn test_server_config_cors_handling() {
+        let extraction_config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::default();
+        let server_config = ServerConfig {
+            cors_origins: vec!["https://example.com".to_string()],
+            ..Default::default()
+        };
+        let _router = create_router_with_limits_and_server_config(extraction_config, limits, server_config);
+    }
+}