kreuzberg 4.0.8 → 4.1.1

data/lib/kreuzberg/types.rb

@@ -3,6 +3,110 @@
 require 'sorbet-runtime'
 
 module Kreuzberg
+  # Semantic element type classification.
+  #
+  # Categorizes text content into semantic units for downstream processing.
+  # Supports the element types commonly found in Unstructured documents.
+  #
+  # @example
+  #   type = Kreuzberg::ElementType::TITLE
+  #
+  ElementType = T.type_alias do
+    T.any(
+      'title',
+      'narrative_text',
+      'heading',
+      'list_item',
+      'table',
+      'image',
+      'page_break',
+      'code_block',
+      'block_quote',
+      'footer',
+      'header'
+    )
+  end
+
+  # Bounding box coordinates for element positioning.
+  #
+  # Represents rectangular coordinates for an element within a page.
+  #
+  # @example
+  #   bbox = Kreuzberg::BoundingBox.new(
+  #     x0: 10.0,
+  #     y0: 20.0,
+  #     x1: 100.0,
+  #     y1: 50.0
+  #   )
+  #   puts "Width: #{bbox.x1 - bbox.x0}"
+  #
+  class BoundingBox < T::Struct
+    extend T::Sig
+
+    const :x0, Float
+
+    const :y0, Float
+
+    const :x1, Float
+
+    const :y1, Float
+  end
+
+  # Metadata for a semantic element.
+  #
+  # Provides contextual information about an extracted element including
+  # its position within the document and custom metadata fields.
+  #
+  # @example
+  #   metadata = Kreuzberg::ElementMetadata.new(
+  #     page_number: 1,
+  #     filename: "document.pdf",
+  #     coordinates: bbox,
+  #     element_index: 5,
+  #     additional: { "style" => "bold" }
+  #   )
+  #
+  class ElementMetadata < T::Struct
+    extend T::Sig
+
+    const :page_number, T.nilable(Integer)
+
+    const :filename, T.nilable(String)
+
+    const :coordinates, T.nilable(BoundingBox)
+
+    const :element_index, T.nilable(Integer)
+
+    const :additional, T::Hash[String, String]
+  end
+
+  # Semantic element extracted from document.
+  #
+  # Represents a logical unit of content with semantic classification,
+  # unique identifier, and metadata for tracking origin and position.
+  # Compatible with Unstructured.io element format when output_format='element_based'.
+  #
+  # @example
+  #   element = Kreuzberg::Element.new(
+  #     element_id: "elem-abc123",
+  #     element_type: "narrative_text",
+  #     text: "This is the main content.",
+  #     metadata: metadata
+  #   )
+  #   puts "#{element.element_type}: #{element.text}"
+  #
+  class Element < T::Struct
+    extend T::Sig
+
+    const :element_id, String
+
+    const :element_type, String
+
+    const :text, String
+
+    const :metadata, ElementMetadata
+  end
+
   # Header/Heading metadata
   #
   # Represents a heading element found in the HTML document

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.8'
+  VERSION = '4.1.1'
 end

data/lib/kreuzberg.rb

@@ -83,10 +83,6 @@ module Kreuzberg
   module_function :validate_mime_type
 
   module_function :get_extensions_for_mime
-
-  module_function :list_embedding_presets
-
-  module_function :get_embedding_preset
 end
 
 require_relative 'kreuzberg/cache_api'

data/sig/kreuzberg.rbs

@@ -250,7 +250,56 @@ module Kreuzberg
     tables: Array[table_hash]?,
     detected_languages: Array[String]?,
     chunks: Array[chunk_hash]?,
-    images: Array[image_hash]?
+    images: Array[image_hash]?,
+    elements: Array[element_hash]?,
+    djot_content: djot_content_hash?
+  }
+
+  type djot_content_hash = {
+    plain_text: String,
+    blocks: Array[formatted_block_hash],
+    metadata_json: String,
+    tables: Array[table_hash],
+    images: Array[djot_image_hash],
+    links: Array[djot_link_hash],
+    footnotes: Array[footnote_hash],
+    attributes: Hash[String, attributes_hash]?
+  }
+
+  type formatted_block_hash = {
+    block_type: String,
+    level: Integer?,
+    content: String?,
+    children: Array[formatted_block_hash]?,
+    attributes: attributes_hash?
+  }
+
+  type djot_image_hash = {
+    url: String,
+    alt: String?,
+    title: String?,
+    attributes: attributes_hash?
+  }
+
+  type djot_link_hash = {
+    url: String,
+    text: String,
+    title: String?,
+    link_type: String?
+  }
+
+  type footnote_hash = {
+    label: String,
+    content: String
+  }
+
+  type attributes_hash = Hash[String, String | Integer | bool | Array[String] | nil]
+
+  type element_hash = {
+    element_id: String,
+    element_type: String,
+    text: String,
+    metadata: Hash[String, untyped]?
   }
 
   type table_hash = {
@@ -359,6 +408,60 @@ module Kreuzberg
       def to_h: () -> image_hash
     end
 
+    # Structured Djot document representation
+    class DjotContent
+      attr_reader plain_text: String
+      attr_reader blocks: Array[DjotContent::FormattedBlock]
+      attr_reader metadata: Hash[untyped, untyped]
+      attr_reader tables: Array[Table]
+      attr_reader images: Array[DjotContent::DjotImage]
+      attr_reader links: Array[DjotContent::DjotLink]
+      attr_reader footnotes: Array[DjotContent::Footnote]
+      attr_reader attributes: Hash[String, untyped]?
+
+      def initialize: (djot_content_hash hash) -> void
+      def to_h: () -> djot_content_hash
+
+      class FormattedBlock
+        attr_reader block_type: String
+        attr_reader level: Integer?
+        attr_reader content: String?
+        attr_reader children: Array[FormattedBlock]?
+        attr_reader attributes: Hash[String, untyped]?
+
+        def initialize: (formatted_block_hash hash) -> void
+        def to_h: () -> formatted_block_hash
+      end
+
+      class DjotImage
+        attr_reader url: String
+        attr_reader alt: String?
+        attr_reader title: String?
+        attr_reader attributes: Hash[String, untyped]?
+
+        def initialize: (djot_image_hash hash) -> void
+        def to_h: () -> djot_image_hash
+      end
+
+      class DjotLink
+        attr_reader url: String
+        attr_reader text: String
+        attr_reader title: String?
+        attr_reader link_type: String?
+
+        def initialize: (djot_link_hash hash) -> void
+        def to_h: () -> djot_link_hash
+      end
+
+      class Footnote
+        attr_reader label: String
+        attr_reader content: String
+
+        def initialize: (label: String, content: String) -> void
+        def to_h: () -> footnote_hash
+      end
+    end
+
     attr_reader content: String
     attr_reader mime_type: String
     attr_reader metadata: Hash[untyped, untyped]
@@ -367,6 +470,7 @@ module Kreuzberg
     attr_reader detected_languages: Array[String]?
     attr_reader chunks: Array[Chunk]?
     attr_reader images: Array[Image]?
+    attr_reader djot_content: DjotContent?
 
     def initialize: (extraction_result_hash hash) -> void
     def to_h: () -> Hash[Symbol, untyped]

data/spec/fixtures/config.toml

@@ -21,7 +21,7 @@ extract_images = true
 passwords = ["secret", "backup"]
 extract_metadata = true
 
-[images]
+[image_extraction]
 extract_images = true
 target_dpi = 600
 max_image_dimension = 2000

data/spec/fixtures/config.yaml

@@ -23,7 +23,7 @@ pdf_options:
     - password2
   extract_metadata: true
 
-images:
+image_extraction:
   extract_images: true
   target_dpi: 300
   max_image_dimension: 4096

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
 resolver = "2"
 
 [workspace.package]
-version = "4.0.8"
+version = "4.1.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -29,7 +29,7 @@ serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.149"
 
 # Error handling
-thiserror = "2.0.17"
+thiserror = "2.0.18"
 anyhow = "1.0"
 
 # Async utilities
@@ -47,7 +47,7 @@ hex = "0.4.3"
 toml = "0.9.11"
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
-html-to-markdown-rs = { version = "2.22.5", default-features = false }
+html-to-markdown-rs = { version = "2.23.4", default-features = false }
 reqwest = { version = "0.13.1", default-features = false, features = ["json", "rustls"] }
 image = { version = "0.25.9", default-features = false }
 lzma-rust2 = { version = "0.15.7" }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.0.8"
+version = "4.1.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -136,6 +136,7 @@ regex = "1.12.2"
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml_ng = "0.10.0"
+jotdown = "0.9"
 toml = { workspace = true }
 mime_guess = "2.0"
 rmp-serde = "1.3"
@@ -152,7 +153,7 @@ lopdf = { version = "0.39.0", optional = true }
 calamine = { version = "0.32.0", features = ["dates"], optional = true }
 polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
 roxmltree = { version = "0.21.1", optional = true }
-zip = { version = "7.1.0", optional = true }
+zip = { version = "7.2.0", optional = true }
 mail-parser = { version = "0.11.1", optional = true }
 msg_parser = { version = "0.1.1", optional = true }
 html-to-markdown-rs = { workspace = true, features = [
@@ -173,7 +174,7 @@ rst_parser = { version = "0.4", optional = true }
 fb2 = { version = "0.4", optional = true }
 typst-syntax = { version = "0.14", optional = true }
 
-kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
+kreuzberg-tesseract = { path = "../kreuzberg-tesseract", version = "4.1", optional = true }
 image = { workspace = true, default-features = false, features = [
     "png",
     "jpeg",
@@ -215,7 +216,7 @@ smartcore = { version = "0.4", default-features = false, features = ["serde"] }
 tempfile = { workspace = true }
 filetime = "0.2"
 tar = "0.4.44"
-zip = "7.1.0"
+zip = "7.2.0"
 serial_test = "3.3.1"
 anyhow = { workspace = true }
 tokio-test = "0.4"

data/vendor/kreuzberg/README.md

@@ -17,7 +17,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.0.0 Release Candidate**
+> **🚀 Version 4.1.1 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/api/config.rs

@@ -0,0 +1,69 @@
+//! API server configuration loading.
+
+use crate::{Result, core::ServerConfig};
+
+/// Load ServerConfig with proper precedence order.
+///
+/// This function implements the configuration hierarchy:
+/// 1. File (if provided)
+/// 2. Environment variables (via apply_env_overrides)
+/// 3. Defaults
+///
+/// The config file can be in flat format (server settings at root) or nested format
+/// (server settings under [server] section alongside other configs like [ocr]).
+///
+/// # Arguments
+///
+/// * `config_path` - Optional path to a ServerConfig file (TOML, YAML, or JSON)
+///
+/// # Returns
+///
+/// A configured ServerConfig with proper precedence applied.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The config file path is provided but cannot be read
+/// - The config file contains invalid server configuration
+/// - Environment variable overrides contain invalid values
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::api::load_server_config;
+/// use std::path::Path;
+///
+/// # fn example() -> kreuzberg::Result<()> {
+/// // Load from file with env overrides
+/// let config = load_server_config(Some(Path::new("server.toml")))?;
+///
+/// // Or use defaults with env overrides
+/// let config = load_server_config(None)?;
+/// # Ok(())
+/// # }
+/// ```
+pub fn load_server_config(config_path: Option<&std::path::Path>) -> Result<ServerConfig> {
+    let mut config = if let Some(path) = config_path {
+        ServerConfig::from_file(path)?
+    } else {
+        ServerConfig::default()
+    };
+
+    // Apply environment variable overrides with proper logging
+    config.apply_env_overrides()?;
+
+    tracing::info!(
+        "Server configuration loaded: host={}, port={}, request_body_limit={} MB, multipart_field_limit={} MB, CORS={}",
+        config.host,
+        config.port,
+        config.max_request_body_mb(),
+        config.max_multipart_field_mb(),
+        if config.cors_allows_all() {
+            "allow all origins".to_string()
+        } else {
+            format!("{} specific origins", config.cors_origins.len())
+        }
+    );
+
+    Ok(config)
+}

data/vendor/kreuzberg/src/api/handlers.rs

@@ -10,8 +10,8 @@ use crate::{batch_extract_bytes, cache, extract_bytes};
 use super::{
     error::ApiError,
     types::{
-        ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ExtractResponse, HealthResponse,
-        InfoResponse,
+        ApiState, CacheClearResponse, CacheStatsResponse, ChunkRequest, ChunkResponse, EmbedRequest, EmbedResponse,
+        ExtractResponse, HealthResponse, InfoResponse,
     },
 };
 
@@ -85,6 +85,25 @@ pub async fn extract_handler(
                     )))
                 })?;
             }
+            "output_format" => {
+                let format_str = field
+                    .text()
+                    .await
+                    .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?;
+
+                config.output_format = match format_str.to_lowercase().as_str() {
+                    "plain" => crate::core::config::OutputFormat::Plain,
+                    "markdown" => crate::core::config::OutputFormat::Markdown,
+                    "djot" => crate::core::config::OutputFormat::Djot,
+                    "html" => crate::core::config::OutputFormat::Html,
+                    _ => {
+                        return Err(ApiError::validation(crate::error::KreuzbergError::validation(format!(
+                            "Invalid output_format: '{}'. Valid values: 'plain', 'markdown', 'djot', 'html'",
+                            format_str
+                        ))));
+                    }
+                };
+            }
             _ => {}
         }
     }
@@ -318,3 +337,81 @@ pub async fn embed_handler(Json(_request): Json<EmbedRequest>) -> Result<Json<Em
         "Embeddings feature is not enabled. Rebuild with --features embeddings".to_string(),
     )))
 }
+
+/// Chunk text endpoint handler.
+///
+/// POST /chunk
+///
+/// Accepts JSON body with text and optional configuration.
+/// Returns chunks with metadata.
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(
+        name = "api.chunk",
+        skip(request),
+        fields(text_length = request.text.len(), chunker_type = request.chunker_type.as_str())
+    )
+)]
+pub async fn chunk_handler(Json(request): Json<ChunkRequest>) -> Result<Json<ChunkResponse>, ApiError> {
+    use super::types::{ChunkItem, ChunkingConfigResponse};
+    use crate::chunking::{ChunkerType, ChunkingConfig, chunk_text};
+
+    // Validate input
+    if request.text.is_empty() {
+        return Err(ApiError::validation(crate::error::KreuzbergError::validation(
+            "Text cannot be empty",
+        )));
+    }
+
+    // Parse chunker_type
+    let chunker_type = match request.chunker_type.to_lowercase().as_str() {
+        "text" | "" => ChunkerType::Text,
+        "markdown" => ChunkerType::Markdown,
+        other => {
+            return Err(ApiError::validation(crate::error::KreuzbergError::validation(format!(
+                "Invalid chunker_type: '{}'. Valid values: 'text', 'markdown'",
+                other
+            ))));
+        }
+    };
+
+    // Build config with defaults
+    let cfg = request.config.unwrap_or_default();
+    let config = ChunkingConfig {
+        max_characters: cfg.max_characters.unwrap_or(2000),
+        overlap: cfg.overlap.unwrap_or(100),
+        trim: cfg.trim.unwrap_or(true),
+        chunker_type,
+    };
+
+    // Perform chunking
+    let result = chunk_text(&request.text, &config, None).map_err(ApiError::internal)?;
+
+    // Transform to response
+    let chunks = result
+        .chunks
+        .into_iter()
+        .map(|chunk| ChunkItem {
+            content: chunk.content,
+            byte_start: chunk.metadata.byte_start,
+            byte_end: chunk.metadata.byte_end,
+            chunk_index: chunk.metadata.chunk_index,
+            total_chunks: chunk.metadata.total_chunks,
+            first_page: chunk.metadata.first_page,
+            last_page: chunk.metadata.last_page,
+        })
+        .collect();
+
+    Ok(Json(ChunkResponse {
+        chunks,
+        chunk_count: result.chunk_count,
+        config: ChunkingConfigResponse {
+            max_characters: config.max_characters,
+            overlap: config.overlap,
+            trim: config.trim,
+            chunker_type: format!("{:?}", config.chunker_type).to_lowercase(),
+        },
+        input_size_bytes: request.text.len(),
+        chunker_type: request.chunker_type.to_lowercase(),
+    }))
+}

data/vendor/kreuzberg/src/api/mod.rs

@@ -7,6 +7,7 @@
 //!
 //! - `POST /extract` - Extract text from uploaded files (multipart form data)
 //! - `POST /embed` - Generate embeddings for text (JSON body with texts array)
+//! - `POST /chunk` - Chunk text into smaller pieces (JSON body with text and config)
 //! - `GET /health` - Health check endpoint
 //! - `GET /info` - Server information
 //! - `GET /cache/stats` - Get cache statistics
@@ -76,19 +77,25 @@
 //! curl -X POST http://localhost:8000/embed \
 //!      -H "Content-Type: application/json" \
 //!      -d '{"texts":["Hello world","Second text"]}'
+//!
+//! # Chunk text
+//! curl -X POST http://localhost:8000/chunk \
+//!      -H "Content-Type: application/json" \
+//!      -d '{"text":"Long text to chunk...","chunker_type":"text"}'
 //! ```
 
+mod config;
 mod error;
 mod handlers;
-mod server;
+mod router;
+mod startup;
 mod types;
 
+pub use config::load_server_config;
 pub use error::ApiError;
-pub use server::{
-    create_router, create_router_with_limits, create_router_with_limits_and_server_config, load_server_config, serve,
-    serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config,
-};
+pub use router::{create_router, create_router_with_limits, create_router_with_limits_and_server_config};
+pub use startup::{serve, serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config};
 pub use types::{
-    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ErrorResponse,
-    ExtractResponse, HealthResponse, InfoResponse,
+    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, ChunkRequest, ChunkResponse, EmbedRequest,
+    EmbedResponse, ErrorResponse, ExtractResponse, HealthResponse, InfoResponse,
 };