kreuzberg 4.0.0.pre.rc.7 → 4.0.0.pre.rc.11

data/sig/kreuzberg.rbs

@@ -226,6 +226,15 @@ module Kreuzberg
   # Alias for Config::Extraction (for API consistency with other language bindings)
   ExtractionConfig: singleton(Config::Extraction)
 
+  # Alias for Config::PageConfig (for API consistency with other language bindings)
+  PageConfig: singleton(Config::PageConfig)
+
+  # Keyword algorithm constants
+  module KeywordAlgorithm
+    YAKE: Symbol
+    RAKE: Symbol
+  end
+
   # Extraction result type
   type extraction_result_hash = {
     content: String,

data/vendor/Cargo.toml

@@ -0,0 +1,44 @@
+[workspace]
+members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract"]
+
+[workspace.package]
+version = "4.0.0-rc.11"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+license = "MIT"
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+
+[workspace.dependencies]
+# Core async runtime
+tokio = { version = "1.48.0", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
+
+# Serialization
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = { version = "1.0.145" }
+
+# Error handling
+thiserror = "2.0.17"
+anyhow = "1.0"
+
+# Async utilities
+async-trait = "0.1.89"
+libc = "0.2.178"
+
+# Tracing/observability
+tracing = "0.1"
+
+# Utilities
+ahash = "0.8.12"
+base64 = "0.22.1"
+hex = "0.4.3"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+html-to-markdown-rs = { version = "2.14.11", default-features = false }
+reqwest = { version = "0.12.25", default-features = false }
+image = { version = "0.25.9", default-features = false }
+
+# Testing (dev)
+tempfile = "3.23.0"
+criterion = { version = "0.8", features = ["html_reports"] }

data/vendor/kreuzberg/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "kreuzberg"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-authors.workspace = true
+version = "4.0.0-rc.11"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
-license.workspace = true
+license = "MIT"
 repository = "https://github.com/kreuzberg-dev/kreuzberg"
 homepage = "https://kreuzberg.dev"
 documentation = "https://docs.rs/kreuzberg"
@@ -26,6 +26,13 @@ tokio-runtime = ["dep:tokio"]
 
 # Format extractors
 pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
+static-pdfium = ["pdf"]                               # Static link (no runtime dep) - requires PDFIUM_STATIC_LIB_PATH env var
+bundled-pdfium = ["pdf"]                              # Embed library in binary (self-contained, dynamic link)
+system-pdfium = ["pdf"]                               # Use system-installed pdfium via pkg-config
+# Legacy names for backward compatibility
+pdf-static = ["static-pdfium"]
+pdf-bundled = ["bundled-pdfium"]
+pdf-system = ["system-pdfium"]
 excel = ["dep:calamine", "dep:polars", "tokio-runtime"]
 office = [
     "dep:roxmltree",
@@ -59,7 +66,12 @@ ocr = [
 ]
 language-detection = ["dep:whatlang"]
 chunking = ["dep:text-splitter"]
-embeddings = ["dep:fastembed", "dep:reqwest", "chunking", "tokio-runtime"]
+embeddings = [
+    "dep:fastembed",
+    "dep:reqwest",
+    "chunking",
+    "tokio-runtime",
+] # Requires system ONNX Runtime
 stopwords = [] # Stopwords for keyword extraction and token reduction
 quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
 
@@ -71,14 +83,16 @@ keywords = ["keywords-yake", "keywords-rake"]
 # Server features
 api = ["dep:axum", "dep:tower", "dep:tower-http", "tokio-runtime"]
 mcp = ["dep:rmcp", "tokio-runtime"]
+mcp-http = ["mcp", "api"]                                          # NEW - enables HTTP transport
 
 # Observability features
 otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
 
 # WASM-compatible feature bundle
-wasm-target = ["html", "xml", "email", "language-detection", "chunking", "quality"]
+wasm-target = ["pdf", "html", "xml", "email", "language-detection", "chunking", "quality"]
+wasm-threads = ["dep:wasm-bindgen-rayon"]
 
-# Convenience bundles
+# Convenience bundles (all use static PDF linking except wasm)
 full = [
     "pdf",
     "excel",
@@ -90,42 +104,56 @@ full = [
     "ocr",
     "language-detection",
     "chunking",
+    "embeddings",
     "quality",
     "keywords",
+    "api",
+    "mcp",
+    "otel",
+]
+server = ["static-pdfium", "excel", "html", "ocr", "api", "mcp"]
+cli = [
+    "static-pdfium",
+    "excel",
+    "office",
+    "html",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
 ]
-server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
-cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
 
 [build-dependencies]
-tracing = { workspace = true }
+tracing = "0.1"
+pkg-config = "0.3"             # For system pdfium detection
 
 [dependencies]
 # Core dependencies (always included)
-ahash = { workspace = true }
-async-trait = { workspace = true }
-base64 = { workspace = true }
+ahash = "0.8.12"
+async-trait = "0.1.89"
+base64 = "0.22.1"
 base64-simd = "0.8"
-hex = { workspace = true }
+hex = "0.4.3"
 lazy_static = "1.5.0"
-libc = { workspace = true }
+libc = "0.2.178"
 memchr = "2.7.6"
-num_cpus = { workspace = true }
+num_cpus = "1.17.0"
 once_cell = "1.21.3"
-paste = "1.0"
+pastey = "0.2"
 rayon = "1.11.0"
 regex = "1.12.2"
-serde = { workspace = true }
-serde_json = { workspace = true }
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.145"
 serde_yaml_ng = "0.10.0"
-toml = "0.9.8"
+toml = { workspace = true }
 mime_guess = "2.0"
 rmp-serde = "1.3"
-thiserror = { workspace = true }
+thiserror = "2.0.17"
 tokio = { workspace = true, optional = true }
 uuid = { version = "1.19.0", features = ["v4", "js"] }
 indexmap = "2.12.1"
-tracing = { workspace = true }
-reqwest = { workspace = true, default-features = false, features = [
+tracing = "0.1"
+reqwest = { version = "0.12.25", default-features = false, features = [
     "json",
     "rustls-tls",
 ], optional = true }
@@ -138,7 +166,7 @@ roxmltree = { version = "0.21.1", optional = true }
 zip = { version = "6.0.0", optional = true }
 mail-parser = { version = "0.11.1", optional = true }
 msg_parser = { version = "0.1.1", optional = true }
-html-to-markdown-rs = { version = "2.14.1", features = ["inline-images"], optional = true }
+html-to-markdown-rs = { version = "2.14.11", default-features = false, features = ["inline-images"], optional = true }
 quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
 tar = { version = "0.4.44", optional = true }
 sevenz-rust = { version = "0.6.1", optional = true }
@@ -153,8 +181,8 @@ fb2 = { version = "0.4", optional = true }
 typst-syntax = { version = "0.14", optional = true }
 
 # Processing features (optional)
-kreuzberg-tesseract = { version = "4.0.0-rc.7", optional = true }
-image = { workspace = true, default-features = false, features = [
+kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
+image = { version = "0.25.9", default-features = false, features = [
     "png",
     "jpeg",
     "webp",
@@ -169,9 +197,9 @@ ndarray = { version = "0.17.1", optional = true }
 kamadak-exif = { version = "0.6.1", optional = true }
 whatlang = { version = "0.18.0", optional = true }
 text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
-fastembed = { version = "5.4", default-features = false, features = [
+fastembed = { version = "5.5", default-features = false, features = [
     "hf-hub-rustls-tls",
-    "ort-download-binaries",
+    "ort-load-dynamic",  # Runtime detection, requires system ONNX Runtime
 ], optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 chardetng = { version = "0.1.17", optional = true }
@@ -182,11 +210,13 @@ rake = { version = "0.3.6", optional = true }
 axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
 tower = { version = "0.5", optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.11.0", features = [
+rmcp = { version = "0.12.0", features = [
     "server",
     "macros",
     "base64",
     "transport-io",
+    "transport-streamable-http-server",
+    "axum",
 ], optional = true }
 # Observability features (optional)
 opentelemetry = { version = "0.31", features = ["trace"], optional = true }
@@ -195,16 +225,16 @@ tracing-opentelemetry = { version = "0.32", optional = true }
 infer = "0.19.0"
 
 [dev-dependencies]
-tempfile = { workspace = true }
+tempfile = "3.23.0"
 filetime = "0.2"
 tar = "0.4.44"
 zip = "6.0.0"
 serial_test = "3.2.0"
-anyhow = { workspace = true }
+anyhow = "1.0"
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-criterion = { workspace = true }
-image = { workspace = true, default-features = false, features = ["png"] }
+criterion = { version = "0.8", features = ["html_reports"] }
+image = { version = "0.25.9", default-features = false, features = ["png"] }
 
 [[bench]]
 name = "otel_overhead"
@@ -215,4 +245,4 @@ harness = false
 pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-wasm-bindgen-rayon = "1.2"
+wasm-bindgen-rayon = { version = "1.3", optional = true }

data/vendor/kreuzberg/README.md

@@ -30,6 +30,26 @@ kreuzberg = "4.0"
 tokio = { version = "1", features = ["rt", "macros"] }
 ```
 
+## System Requirements
+
+### ONNX Runtime (for embeddings)
+
+If using embeddings functionality, ONNX Runtime must be installed:
+
+```bash
+# macOS
+brew install onnxruntime
+
+# Ubuntu/Debian
+sudo apt install libonnxruntime libonnxruntime-dev
+
+# Windows (MSVC)
+scoop install onnxruntime
+# OR download from https://github.com/microsoft/onnxruntime/releases
+```
+
+Without ONNX Runtime, embeddings will raise `MissingDependencyError` with installation instructions.
+
 ## Quick Start
 
 ```rust
@@ -169,6 +189,36 @@ kreuzberg = { version = "4.0", features = ["server"] }
 kreuzberg = { version = "4.0", features = ["cli"] }
 ```
 
+## PDF Support and Linking Options
+
+Kreuzberg supports three PDFium linking strategies. **Default is `bundled-pdfium`** (best developer experience).
+
+| Strategy | Feature | Use Case | Binary Size | Runtime Deps |
+|----------|---------|----------|-------------|--------------|
+| **Bundled (default)** | `bundled-pdfium` | Development, production | +8-15MB | None |
+| **Static** | `static-pdfium` | Docker, musl, standalone binaries | +200MB | None |
+| **System** | `system-pdfium` | Package managers, distros | +2MB | libpdfium.so |
+
+### Quick Start
+
+```toml
+# Default - bundled PDFium (recommended)
+[dependencies]
+kreuzberg = "4.0"
+
+# Static linking (Docker, musl)
+[dependencies]
+kreuzberg = { version = "4.0", features = ["static-pdfium"] }
+
+# System PDFium (package managers)
+[dependencies]
+kreuzberg = { version = "4.0", features = ["system-pdfium"] }
+```
+
+For detailed information, see the [PDFium Linking Guide](../../docs/guides/pdfium-linking.md).
+
+**Note:** Language bindings (Python, TypeScript, Ruby, Java, Go) automatically bundle PDFium. No configuration needed.
+
 ## Documentation
 
 **[API Documentation](https://docs.rs/kreuzberg)** – Complete API reference with examples