kreuzberg 4.0.0.pre.rc.8 → 4.0.0.pre.rc.11

data/sig/kreuzberg.rbs

@@ -226,6 +226,15 @@ module Kreuzberg
   # Alias for Config::Extraction (for API consistency with other language bindings)
   ExtractionConfig: singleton(Config::Extraction)
 
+  # Alias for Config::PageConfig (for API consistency with other language bindings)
+  PageConfig: singleton(Config::PageConfig)
+
+  # Keyword algorithm constants
+  module KeywordAlgorithm
+    YAKE: Symbol
+    RAKE: Symbol
+  end
+
   # Extraction result type
   type extraction_result_hash = {
     content: String,

data/vendor/Cargo.toml

@@ -0,0 +1,44 @@
+[workspace]
+members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract"]
+
+[workspace.package]
+version = "4.0.0-rc.11"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+license = "MIT"
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+
+[workspace.dependencies]
+# Core async runtime
+tokio = { version = "1.48.0", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
+
+# Serialization
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = { version = "1.0.145" }
+
+# Error handling
+thiserror = "2.0.17"
+anyhow = "1.0"
+
+# Async utilities
+async-trait = "0.1.89"
+libc = "0.2.178"
+
+# Tracing/observability
+tracing = "0.1"
+
+# Utilities
+ahash = "0.8.12"
+base64 = "0.22.1"
+hex = "0.4.3"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+html-to-markdown-rs = { version = "2.14.11", default-features = false }
+reqwest = { version = "0.12.25", default-features = false }
+image = { version = "0.25.9", default-features = false }
+
+# Testing (dev)
+tempfile = "3.23.0"
+criterion = { version = "0.8", features = ["html_reports"] }

data/vendor/kreuzberg/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "kreuzberg"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-authors.workspace = true
+version = "4.0.0-rc.11"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
-license.workspace = true
+license = "MIT"
 repository = "https://github.com/kreuzberg-dev/kreuzberg"
 homepage = "https://kreuzberg.dev"
 documentation = "https://docs.rs/kreuzberg"
@@ -26,9 +26,13 @@ tokio-runtime = ["dep:tokio"]
 
 # Format extractors
 pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
-pdf-static = ["pdf"] # Download + static link (no runtime dependency)
-pdf-bundled = ["pdf"] # Embed library in binary (self-contained)
-pdf-system = ["pdf"] # Use system-installed pdfium via pkg-config
+static-pdfium = ["pdf"]                               # Static link (no runtime dep) - requires PDFIUM_STATIC_LIB_PATH env var
+bundled-pdfium = ["pdf"]                              # Embed library in binary (self-contained, dynamic link)
+system-pdfium = ["pdf"]                               # Use system-installed pdfium via pkg-config
+# Legacy names for backward compatibility
+pdf-static = ["static-pdfium"]
+pdf-bundled = ["bundled-pdfium"]
+pdf-system = ["system-pdfium"]
 excel = ["dep:calamine", "dep:polars", "tokio-runtime"]
 office = [
     "dep:roxmltree",
@@ -62,7 +66,12 @@ ocr = [
 ]
 language-detection = ["dep:whatlang"]
 chunking = ["dep:text-splitter"]
-embeddings = ["dep:fastembed", "dep:reqwest", "chunking", "tokio-runtime"]
+embeddings = [
+    "dep:fastembed",
+    "dep:reqwest",
+    "chunking",
+    "tokio-runtime",
+] # Requires system ONNX Runtime
 stopwords = [] # Stopwords for keyword extraction and token reduction
 quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
 
@@ -80,9 +89,10 @@ mcp-http = ["mcp", "api"]                                          # NEW - enabl
 otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
 
 # WASM-compatible feature bundle
-wasm-target = ["html", "xml", "email", "language-detection", "chunking", "quality"]
+wasm-target = ["pdf", "html", "xml", "email", "language-detection", "chunking", "quality"]
+wasm-threads = ["dep:wasm-bindgen-rayon"]
 
-# Convenience bundles
+# Convenience bundles (all use static PDF linking except wasm)
 full = [
     "pdf",
     "excel",
@@ -94,43 +104,56 @@ full = [
     "ocr",
     "language-detection",
     "chunking",
+    "embeddings",
     "quality",
     "keywords",
+    "api",
+    "mcp",
+    "otel",
+]
+server = ["static-pdfium", "excel", "html", "ocr", "api", "mcp"]
+cli = [
+    "static-pdfium",
+    "excel",
+    "office",
+    "html",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
 ]
-server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
-cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
 
 [build-dependencies]
-tracing = { workspace = true }
+tracing = "0.1"
 pkg-config = "0.3"             # For system pdfium detection
 
 [dependencies]
 # Core dependencies (always included)
-ahash = { workspace = true }
-async-trait = { workspace = true }
-base64 = { workspace = true }
+ahash = "0.8.12"
+async-trait = "0.1.89"
+base64 = "0.22.1"
 base64-simd = "0.8"
-hex = { workspace = true }
+hex = "0.4.3"
 lazy_static = "1.5.0"
-libc = { workspace = true }
+libc = "0.2.178"
 memchr = "2.7.6"
-num_cpus = { workspace = true }
+num_cpus = "1.17.0"
 once_cell = "1.21.3"
-paste = "1.0"
+pastey = "0.2"
 rayon = "1.11.0"
 regex = "1.12.2"
-serde = { workspace = true }
-serde_json = { workspace = true }
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.145"
 serde_yaml_ng = "0.10.0"
-toml = "0.9.8"
+toml = { workspace = true }
 mime_guess = "2.0"
 rmp-serde = "1.3"
-thiserror = { workspace = true }
+thiserror = "2.0.17"
 tokio = { workspace = true, optional = true }
 uuid = { version = "1.19.0", features = ["v4", "js"] }
 indexmap = "2.12.1"
-tracing = { workspace = true }
-reqwest = { workspace = true, default-features = false, features = [
+tracing = "0.1"
+reqwest = { version = "0.12.25", default-features = false, features = [
     "json",
     "rustls-tls",
 ], optional = true }
@@ -143,7 +166,7 @@ roxmltree = { version = "0.21.1", optional = true }
 zip = { version = "6.0.0", optional = true }
 mail-parser = { version = "0.11.1", optional = true }
 msg_parser = { version = "0.1.1", optional = true }
-html-to-markdown-rs = { version = "2.14.2", features = ["inline-images"], optional = true }
+html-to-markdown-rs = { version = "2.14.11", default-features = false, features = ["inline-images"], optional = true }
 quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
 tar = { version = "0.4.44", optional = true }
 sevenz-rust = { version = "0.6.1", optional = true }
@@ -158,8 +181,8 @@ fb2 = { version = "0.4", optional = true }
 typst-syntax = { version = "0.14", optional = true }
 
 # Processing features (optional)
-kreuzberg-tesseract = { version = "4.0.0-rc.7", optional = true }
-image = { workspace = true, default-features = false, features = [
+kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
+image = { version = "0.25.9", default-features = false, features = [
     "png",
     "jpeg",
     "webp",
@@ -174,9 +197,9 @@ ndarray = { version = "0.17.1", optional = true }
 kamadak-exif = { version = "0.6.1", optional = true }
 whatlang = { version = "0.18.0", optional = true }
 text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
-fastembed = { version = "5.4", default-features = false, features = [
+fastembed = { version = "5.5", default-features = false, features = [
     "hf-hub-rustls-tls",
-    "ort-download-binaries",
+    "ort-load-dynamic",  # Runtime detection, requires system ONNX Runtime
 ], optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 chardetng = { version = "0.1.17", optional = true }
@@ -187,7 +210,7 @@ rake = { version = "0.3.6", optional = true }
 axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
 tower = { version = "0.5", optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.11.0", features = [
+rmcp = { version = "0.12.0", features = [
     "server",
     "macros",
     "base64",
@@ -202,16 +225,16 @@ tracing-opentelemetry = { version = "0.32", optional = true }
 infer = "0.19.0"
 
 [dev-dependencies]
-tempfile = { workspace = true }
+tempfile = "3.23.0"
 filetime = "0.2"
 tar = "0.4.44"
 zip = "6.0.0"
 serial_test = "3.2.0"
-anyhow = { workspace = true }
+anyhow = "1.0"
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-criterion = { workspace = true }
-image = { workspace = true, default-features = false, features = ["png"] }
+criterion = { version = "0.8", features = ["html_reports"] }
+image = { version = "0.25.9", default-features = false, features = ["png"] }
 
 [[bench]]
 name = "otel_overhead"
@@ -222,4 +245,4 @@ harness = false
 pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-wasm-bindgen-rayon = "1.3"
+wasm-bindgen-rayon = { version = "1.3", optional = true }

data/vendor/kreuzberg/README.md

@@ -30,6 +30,26 @@ kreuzberg = "4.0"
 tokio = { version = "1", features = ["rt", "macros"] }
 ```
 
+## System Requirements
+
+### ONNX Runtime (for embeddings)
+
+If using embeddings functionality, ONNX Runtime must be installed:
+
+```bash
+# macOS
+brew install onnxruntime
+
+# Ubuntu/Debian
+sudo apt install libonnxruntime libonnxruntime-dev
+
+# Windows (MSVC)
+scoop install onnxruntime
+# OR download from https://github.com/microsoft/onnxruntime/releases
+```
+
+Without ONNX Runtime, embeddings will raise `MissingDependencyError` with installation instructions.
+
 ## Quick Start
 
 ```rust
@@ -169,46 +189,35 @@ kreuzberg = { version = "4.0", features = ["server"] }
 kreuzberg = { version = "4.0", features = ["cli"] }
 ```
 
-## PDFium Linking Options
+## PDF Support and Linking Options
 
-When using the `pdf` feature, you can choose how PDFium is linked to your binary. Four strategies are supported:
+Kreuzberg supports three PDFium linking strategies. **Default is `bundled-pdfium`** (best developer experience).
 
-| Strategy | Feature | Use Case |
-|----------|---------|----------|
-| **Dynamic (default)** | `pdf` | Fast builds, runtime library dependency |
-| **Static** | `pdf`, `pdf-static` | Embed PDFium in binary, larger binary size |
-| **Bundled** | `pdf`, `pdf-bundled` | Self-contained per-binary copies |
-| **System** | `pdf`, `pdf-system` | Use system-installed PDFium |
+| Strategy | Feature | Use Case | Binary Size | Runtime Deps |
+|----------|---------|----------|-------------|--------------|
+| **Bundled (default)** | `bundled-pdfium` | Development, production | +8-15MB | None |
+| **Static** | `static-pdfium` | Docker, musl, standalone binaries | +200MB | None |
+| **System** | `system-pdfium` | Package managers, distros | +2MB | libpdfium.so |
 
-### Examples
+### Quick Start
 
-**Default (dynamic linking)** - Fastest compilation, requires libpdfium at runtime:
 ```toml
+# Default - bundled PDFium (recommended)
 [dependencies]
-kreuzberg = { version = "4.0", features = ["pdf"] }
-```
-
-**Static linking** - Larger binary, no runtime dependency:
-```toml
-[dependencies]
-kreuzberg = { version = "4.0", features = ["pdf", "pdf-static"] }
-```
+kreuzberg = "4.0"
 
-**Bundled** - Each binary extracts its own copy:
-```toml
+# Static linking (Docker, musl)
 [dependencies]
-kreuzberg = { version = "4.0", features = ["pdf", "pdf-bundled"] }
-```
+kreuzberg = { version = "4.0", features = ["static-pdfium"] }
 
-**System-installed** - Use pkg-config or manual paths:
-```toml
+# System PDFium (package managers)
 [dependencies]
-kreuzberg = { version = "4.0", features = ["pdf", "pdf-system"] }
+kreuzberg = { version = "4.0", features = ["system-pdfium"] }
 ```
 
-For comprehensive guidance on linking strategies, environment variables, and troubleshooting, see the [PDFium Linking Guide](../../docs/guides/pdfium-linking.md).
+For detailed information, see the [PDFium Linking Guide](../../docs/guides/pdfium-linking.md).
 
-**Note:** Language bindings (Python, TypeScript, Ruby, Java, Go) bundle PDFium automatically and do not expose linking options.
+**Note:** Language bindings (Python, TypeScript, Ruby, Java, Go) automatically bundle PDFium. No configuration needed.
 
 ## Documentation