kreuzberg 4.0.0.pre.rc.27 → 4.0.0.pre.rc.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db85e33f6712e0a72c327af0e5c91e24f5c937fd97cb198ed566fa3e8779b00f
-  data.tar.gz: 85f37cdec92c6f410a39d4d05f7537d013ff14a5d5d692911fd9e7d051e82139
+  metadata.gz: 3516f4c7752e59f020d36750e06b493b7bf1670d1e5ca56423324cf90c48e5f8
+  data.tar.gz: 6fe9f382e6741eaa69c28ffc70647f6b0cccebaa7072b23f8811d0072843fc09
 SHA512:
-  metadata.gz: 4840194f9a38ac7df13ae3f847daea0d2b521ee5fc2f43ae7405ebdeeb972d8e0fd112b49ebedc7d0188b9aaf5ad63b9a00a16fd38a5ca5865d550a2ee1fd60f
-  data.tar.gz: b63c74e8955c6d70cff05eb4f466b878eb64681cd59ce7077dc834f5e07fa0098dca64d9f51851d7fc7bb00c64de6cf748f897d577a3a3772fbc338803d6606e
+  metadata.gz: 53b28f45c8831830c269580c73f0480afddf3550ff0fa248a6155b0478f022696e7025f20699fc36be358bd04e3a0e5a279259c3960ee6ed9d0218464897d928
+  data.tar.gz: 32b50779cf5b0ed5aaab7804f522297f9b12fe3b9374dc8efadeec2ec78aada25d1372dc50ffe81baccddfcca271b1345d09cb41e239498c7d061d0a31982979

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.0.0.pre.rc.27)
+    kreuzberg (4.0.0.pre.rc.28)
 
 GEM
   remote: https://rubygems.org/
@@ -76,7 +76,7 @@ GEM
       ffi (~> 1.0)
     rb_sys (0.9.119)
       rake-compiler-dock (= 1.10.0)
-    rbs (3.10.0)
+    rbs (3.10.1)
       logger
     regexp_parser (2.11.3)
     rspec (3.13.2)
@@ -132,7 +132,7 @@ GEM
       strscan (>= 1.0.0)
       terminal-table (>= 2, < 5)
       uri (>= 0.12.0)
-    strscan (3.1.6)
+    strscan (3.1.7)
     terminal-table (4.0.0)
       unicode-display_width (>= 1.1.1, < 4)
     tzinfo (2.0.6)
@@ -198,7 +198,7 @@ CHECKSUMS
   fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  kreuzberg (4.0.0.pre.rc.27)
+  kreuzberg (4.0.0.pre.rc.28)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.9.0) sha256=db9e4424e0e5834480385197c139cb6b0ae0ef28cc13310cfd1ca78377d59c67
@@ -219,7 +219,7 @@ CHECKSUMS
   rb-fsevent (0.11.2) sha256=43900b972e7301d6570f64b850a5aa67833ee7d87b458ee92805d56b7318aefe
   rb-inotify (0.11.1) sha256=a0a700441239b0ff18eb65e3866236cd78613d6b9f78fea1f9ac47a85e47be6e
   rb_sys (0.9.119) sha256=64393fa148e402e1b79b64496d2aabfc7df79da6b822b8bb48dc1141eaf40b4b
-  rbs (3.10.0) sha256=e75b5f1313c71c9ee0fcea68bf97d3e5fe8ec7a641d4b5cd18bbc28c94ddf298
+  rbs (3.10.1) sha256=4e0a9e460dd2b0b763be24734b113da32fc621d383c1119005fe7fb18c73d0c9
   regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4
   rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
   rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
@@ -233,7 +233,7 @@ CHECKSUMS
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
   steep (1.10.0) sha256=1b295b55f9aaff1b8d3ee42453ee55bc2a1078fda0268f288edb2dc014f4d7d1
-  strscan (3.1.6) sha256=ebd56df0b0468b00a1f2004b4078c34df58c3506b2bba939e7531892aece81f3
+  strscan (3.1.7) sha256=5f76462b94a3ea50b44973225b7d75b2cb96d4e1bee9ef1319b99ca117b72c8c
   terminal-table (4.0.0) sha256=f504793203f8251b2ea7c7068333053f0beeea26093ec9962e62ea79f94301d2
   tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -2578,7 +2578,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.26"
+version = "4.0.0-rc.27"
 dependencies = [
  "async-trait",
  "html-to-markdown-rs",

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,8 +1,11 @@
 [workspace]
 
+[workspace.lints.clippy]
+collapsible_if = "allow"
+
 [package]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.27"
+version = "4.0.0-rc.28"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -15,6 +18,9 @@ description = "Ruby bindings (Magnus) for Kreuzberg - high-performance document
 keywords = ["ruby", "magnus", "document", "extraction", "bindings"]
 categories = ["api-bindings", "text-processing"]
 
+[lints]
+workspace = true
+
 [lib]
 name = "kreuzberg_rb"
 crate-type = ["cdylib", "rlib"]

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.0-rc.27'
+  VERSION = '4.0.0-rc.28'
 end

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
 resolver = "2"
 
 [workspace.package]
-version = "4.0.0-rc.27"
+version = "4.0.0-rc.28"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -26,7 +26,7 @@ tokio = { version = "1.49.0", features = [
 
 # Serialization
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.148"
+serde_json = "1.0.149"
 
 # Error handling
 thiserror = "2.0.17"
@@ -47,10 +47,10 @@ hex = "0.4.3"
 toml = "0.9.10"
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
-html-to-markdown-rs = { version = "2.19.5", default-features = false }
-reqwest = { version = "0.12.28", default-features = false, features = ["json", "rustls-tls"] }
+html-to-markdown-rs = { version = "2.20.0", default-features = false }
+reqwest = { version = "0.13.1", default-features = false, features = ["json", "rustls"] }
 image = { version = "0.25.9", default-features = false }
-lzma-rust2 = { version = "0.15.4" }
+lzma-rust2 = { version = "0.15.6" }
 
 # Testing (dev)
 tempfile = "3.24.0"

data/vendor/kreuzberg/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "kreuzberg"
-version = "4.0.0-rc.27"
+version = "4.0.0-rc.28"
 edition = "2024"
-lints.workspace = true
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
@@ -145,10 +144,6 @@ tokio = { workspace = true, optional = true }
 uuid = { version = "1.19.0", features = ["v4", "js"] }
 indexmap = "2.12.1"
 tracing = { workspace = true }
-reqwest = { workspace = true, default-features = false, features = [
-    "json",
-    "rustls-tls",
-], optional = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", version = "0.9.0", features = [
     "thread_safe",
     "image_latest",
@@ -188,16 +183,12 @@ image = { workspace = true, default-features = false, features = [
     "gif",
     "rayon",
 ], optional = true }
-tiff = { version = "0.10", optional = true }
+tiff = { version = "0.11", optional = true }
 fast_image_resize = { version = "5.5.0", optional = true }
 ndarray = { version = "0.17.1", optional = true }
 kamadak-exif = { version = "0.6.1", optional = true }
 whatlang = { version = "0.18.0", optional = true }
 text-splitter = { version = "0.29.3", features = ["markdown"], optional = true }
-fastembed = { version = "5.6", default-features = false, features = [
-    "hf-hub-rustls-tls",
-    "ort-load-dynamic",
-], optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 chardetng = { version = "0.1.17", optional = true }
 encoding_rs = { version = "0.8.35", optional = true }
@@ -225,15 +216,44 @@ tempfile = { workspace = true }
 filetime = "0.2"
 tar = "0.4.44"
 zip = "7.0.0"
-serial_test = "3.2.0"
+serial_test = "3.3.1"
 anyhow = { workspace = true }
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 criterion = { workspace = true }
 image = { workspace = true, default-features = false, features = ["png"] }
 
-[target.'cfg(not(target_os = "windows"))'.dependencies]
+[target.'cfg(all(not(target_os = "windows"), not(target_arch = "wasm32")))'.dependencies]
 pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
+# Use rustls on non-Windows platforms (Linux, macOS)
+reqwest = { workspace = true, default-features = false, features = [
+    "json",
+    "rustls",
+], optional = true }
+# Use rustls-tls for fastembed on non-Windows platforms
+fastembed = { version = "5.6", default-features = false, features = [
+    "hf-hub-rustls-tls",
+    "ort-load-dynamic",
+], optional = true }
+# Force ureq (transitive dep via hf-hub) to use rustls on non-Windows
+ureq = { version = "2.12", default-features = false, features = ["tls", "json"] }
+
+# Use native-tls on Windows to avoid aws-lc-sys CMake build issues with MinGW
+[target.'cfg(all(target_os = "windows", not(target_arch = "wasm32")))'.dependencies]
+reqwest = { workspace = true, default-features = false, features = [
+    "json",
+    "native-tls",
+], optional = true }
+# Use native-tls for fastembed on Windows
+fastembed = { version = "5.6", default-features = false, features = [
+    "hf-hub-native-tls",
+    "ort-load-dynamic",
+], optional = true }
+# Force ureq (transitive dep via hf-hub) to use native-tls on Windows
+ureq = { version = "2.12", default-features = false, features = ["native-tls", "json"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen-rayon = { version = "1.3", optional = true }
+# Override getrandom to enable js feature for WASM targets
+# This is needed because ring/rustls (via ureq) depend on getrandom without js feature
+getrandom = { workspace = true }

data/vendor/kreuzberg/src/api/handlers.rs

@@ -9,7 +9,10 @@ use crate::{batch_extract_bytes, cache, extract_bytes};
 
 use super::{
     error::ApiError,
-    types::{ApiState, CacheClearResponse, CacheStatsResponse, ExtractResponse, HealthResponse, InfoResponse},
+    types::{
+        ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ExtractResponse, HealthResponse,
+        InfoResponse,
+    },
 };
 
 /// Extract endpoint handler.
@@ -34,6 +37,14 @@ use super::{
 ///
 /// The server's default config (loaded from kreuzberg.toml/yaml/json via discovery)
 /// is used as the base, and any per-request config overrides those defaults.
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(
+        name = "api.extract",
+        skip(state, multipart),
+        fields(files_count = tracing::field::Empty)
+    )
+)]
 pub async fn extract_handler(
     State(state): State<ApiState>,
     mut multipart: Multipart,
@@ -84,6 +95,9 @@ pub async fn extract_handler(
         )));
     }
 
+    #[cfg(feature = "otel")]
+    tracing::Span::current().record("files_count", files.len());
+
     if files.len() == 1 {
         let (data, mime_type, _file_name) = files
             .into_iter()
@@ -102,6 +116,7 @@ pub async fn extract_handler(
 /// Health check endpoint handler.
 ///
 /// GET /health
+#[cfg_attr(feature = "otel", tracing::instrument(name = "api.health"))]
 pub async fn health_handler() -> Json<HealthResponse> {
     Json(HealthResponse {
         status: "healthy".to_string(),
@@ -112,6 +127,7 @@ pub async fn health_handler() -> Json<HealthResponse> {
 /// Server info endpoint handler.
 ///
 /// GET /info
+#[cfg_attr(feature = "otel", tracing::instrument(name = "api.info"))]
 pub async fn info_handler() -> Json<InfoResponse> {
     Json(InfoResponse {
         version: env!("CARGO_PKG_VERSION").to_string(),
@@ -129,6 +145,7 @@ pub async fn info_handler() -> Json<InfoResponse> {
 /// - Current directory cannot be determined
 /// - Cache directory path contains non-UTF8 characters
 /// - Cache metadata retrieval fails
+#[cfg_attr(feature = "otel", tracing::instrument(name = "api.cache_stats"))]
 pub async fn cache_stats_handler() -> Result<Json<CacheStatsResponse>, ApiError> {
     let cache_dir = std::env::current_dir()
         .map_err(|e| {
@@ -168,6 +185,7 @@ pub async fn cache_stats_handler() -> Result<Json<CacheStatsResponse>, ApiError>
 /// - Current directory cannot be determined
 /// - Cache directory path contains non-UTF8 characters
 /// - Cache clearing operation fails
+#[cfg_attr(feature = "otel", tracing::instrument(name = "api.cache_clear"))]
 pub async fn cache_clear_handler() -> Result<Json<CacheClearResponse>, ApiError> {
     let cache_dir = std::env::current_dir()
         .map_err(|e| {
@@ -193,3 +211,110 @@ pub async fn cache_clear_handler() -> Result<Json<CacheClearResponse>, ApiError>
         freed_mb,
     }))
 }
+
+/// Embedding endpoint handler.
+///
+/// POST /embed
+///
+/// Accepts JSON body with:
+/// - `texts`: Array of strings to generate embeddings for
+/// - `config` (optional): Embedding configuration (model, batch size, cache_dir)
+///
+/// Returns embeddings for each input text.
+///
+/// # Errors
+///
+/// Returns `ApiError::Internal` if:
+/// - Embeddings feature is not enabled
+/// - ONNX Runtime is not available
+/// - Model initialization fails
+/// - Embedding generation fails
+#[cfg(feature = "embeddings")]
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(
+        name = "api.embed",
+        skip(request),
+        fields(
+            texts_count = request.texts.len(),
+            model = tracing::field::Empty
+        )
+    )
+)]
+pub async fn embed_handler(Json(request): Json<EmbedRequest>) -> Result<Json<EmbedResponse>, ApiError> {
+    use crate::types::{Chunk, ChunkMetadata};
+
+    if request.texts.is_empty() {
+        return Err(ApiError::validation(crate::error::KreuzbergError::validation(
+            "No texts provided for embedding generation",
+        )));
+    }
+
+    // Use default config if none provided
+    let config = request.config.unwrap_or_default();
+
+    // Create chunks from input texts
+    let mut chunks: Vec<Chunk> = request
+        .texts
+        .iter()
+        .enumerate()
+        .map(|(idx, text)| Chunk {
+            content: text.clone(),
+            embedding: None,
+            metadata: ChunkMetadata {
+                byte_start: 0,
+                byte_end: text.len(),
+                token_count: None,
+                chunk_index: idx,
+                total_chunks: request.texts.len(),
+                first_page: None,
+                last_page: None,
+            },
+        })
+        .collect();
+
+    // Generate embeddings
+    crate::embeddings::generate_embeddings_for_chunks(&mut chunks, &config).map_err(ApiError::internal)?;
+
+    // Extract embeddings from chunks
+    let embeddings: Vec<Vec<f32>> = chunks
+        .into_iter()
+        .map(|chunk| {
+            chunk.embedding.ok_or_else(|| {
+                ApiError::internal(crate::error::KreuzbergError::Other(
+                    "Failed to generate embedding for text".to_string(),
+                ))
+            })
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let dimensions = embeddings.first().map(|e| e.len()).unwrap_or(0);
+
+    // Get model name from config
+    let model_name = match &config.model {
+        crate::core::config::EmbeddingModelType::Preset { name } => name.clone(),
+        #[cfg(feature = "embeddings")]
+        crate::core::config::EmbeddingModelType::FastEmbed { model, .. } => model.clone(),
+        crate::core::config::EmbeddingModelType::Custom { .. } => "custom".to_string(),
+    };
+
+    #[cfg(feature = "otel")]
+    tracing::Span::current().record("model", &model_name);
+
+    Ok(Json(EmbedResponse {
+        embeddings,
+        model: model_name,
+        dimensions,
+        count: request.texts.len(),
+    }))
+}
+
+/// Embedding endpoint handler (when embeddings feature is disabled).
+///
+/// Returns an error indicating embeddings feature is not enabled.
+#[cfg(not(feature = "embeddings"))]
+pub async fn embed_handler(Json(_request): Json<EmbedRequest>) -> Result<Json<EmbedResponse>, ApiError> {
+    Err(ApiError::internal(crate::error::KreuzbergError::MissingDependency(
+        "Embeddings feature is not enabled. Rebuild with --features embeddings".to_string(),
+    )))
+}

data/vendor/kreuzberg/src/api/mod.rs

@@ -6,6 +6,7 @@
 //! # Endpoints
 //!
 //! - `POST /extract` - Extract text from uploaded files (multipart form data)
+//! - `POST /embed` - Generate embeddings for text (JSON body with texts array)
 //! - `GET /health` - Health check endpoint
 //! - `GET /info` - Server information
 //! - `GET /cache/stats` - Get cache statistics
@@ -70,6 +71,11 @@
 //!
 //! # Clear cache
 //! curl -X DELETE http://localhost:8000/cache/clear
+//!
+//! # Generate embeddings
+//! curl -X POST http://localhost:8000/embed \
+//!      -H "Content-Type: application/json" \
+//!      -d '{"texts":["Hello world","Second text"]}'
 //! ```
 
 mod error;
@@ -79,9 +85,10 @@ mod types;
 
 pub use error::ApiError;
 pub use server::{
-    create_router, create_router_with_limits, serve, serve_default, serve_with_config, serve_with_config_and_limits,
+    create_router, create_router_with_limits, create_router_with_limits_and_server_config, load_server_config, serve,
+    serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config,
 };
 pub use types::{
-    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, ErrorResponse, ExtractResponse, HealthResponse,
-    InfoResponse,
+    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, EmbedRequest, EmbedResponse, ErrorResponse,
+    ExtractResponse, HealthResponse, InfoResponse,
 };