kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/Cargo.toml

@@ -1,259 +1,134 @@
 [package]
 name = "kreuzberg"
-version = "4.0.0-rc.29"
-edition = "2024"
-rust-version = "1.91"
+version = "4.0.0-rc.1"
+edition.workspace = true
+rust-version.workspace = true
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
 license = "MIT"
-repository = "https://github.com/kreuzberg-dev/kreuzberg"
+repository = "https://github.com/Goldziher/kreuzberg"
 homepage = "https://kreuzberg.dev"
 documentation = "https://docs.rs/kreuzberg"
-keywords = ["document", "extraction", "pdf", "ocr", "parser"]
-categories = ["parser-implementations", "text-processing"]
+keywords = ["document", "extraction", "pdf", "ocr", "parsing"]
+categories = ["parsing", "text-processing", "asynchronous", "data-structures"]
 readme = "README.md"
 
 [lib]
 crate-type = ["rlib"]
 
 [features]
-default = ["tokio-runtime", "simd-utf8"]
+default = []
 
 profiling = ["dep:pprof"]
 
-pool-metrics = []
-
-simd-utf8 = ["dep:simdutf8"]
-
-tokio-runtime = ["dep:tokio"]
-
+# Format extractors
 pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
-static-pdfium = ["pdf"]
-bundled-pdfium = ["pdf"]
-system-pdfium = ["pdf"]
-excel = ["dep:calamine", "dep:polars", "tokio-runtime"]
-office = [
-    "dep:roxmltree",
-    "dep:zip",
-    "dep:docx-lite",
-    "dep:quick-xml",
-    "dep:pulldown-cmark",
-    "dep:biblatex",
-    "dep:org",
-    "dep:rtf-parser",
-    "dep:rst_parser",
-    "dep:fb2",
-    "dep:typst-syntax",
-    "html",
-    "tokio-runtime",
-]
+excel = ["dep:calamine", "dep:polars"]
+office = ["dep:roxmltree", "dep:zip", "dep:docx-lite"]
 email = ["dep:mail-parser", "dep:msg_parser"]
-html = ["dep:html-to-markdown-rs"]
+html = ["dep:html-to-markdown-rs", "dep:html-escape", "dep:scraper"]
 xml = ["dep:quick-xml", "dep:roxmltree"]
-archives = ["dep:zip", "dep:tar", "dep:sevenz-rust2", "dep:lzma-rust2"]
+archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
 
-ocr = [
-    "dep:kreuzberg-tesseract",
-    "dep:image",
-    "dep:tiff",
-    "dep:fast_image_resize",
-    "dep:ndarray",
-    "dep:kamadak-exif",
-    "html",
-]
+# Processing features
+ocr = ["dep:kreuzberg-tesseract", "dep:image", "dep:fast_image_resize", "dep:ndarray", "dep:kamadak-exif", "html"]
 language-detection = ["dep:whatlang"]
 chunking = ["dep:text-splitter"]
-embeddings = ["dep:fastembed", "dep:reqwest", "chunking", "tokio-runtime"]
-stopwords = []
+embeddings = ["dep:fastembed", "chunking"]
+stopwords = []  # Stopwords for keyword extraction and token reduction
 quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
 
+# Keyword extraction (requires stopwords)
 keywords-yake = ["dep:yake-rust", "stopwords"]
 keywords-rake = ["dep:rake", "stopwords"]
 keywords = ["keywords-yake", "keywords-rake"]
 
-api = ["dep:axum", "dep:tower", "dep:tower-http", "tokio-runtime"]
-mcp = ["dep:rmcp", "tokio-runtime"]
-mcp-http = ["mcp", "api"]
-
-otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
-
-wasm-target = ["pdf", "html", "xml", "email", "language-detection", "chunking", "quality"]
-wasm-threads = ["dep:wasm-bindgen-rayon"]
+# Server features
+api = ["dep:axum", "dep:tower", "dep:tower-http"]
+mcp = ["dep:rmcp"]
 
+# Convenience bundles
 full = [
-    "pdf",
-    "excel",
-    "office",
-    "email",
-    "html",
-    "xml",
-    "archives",
-    "ocr",
-    "language-detection",
-    "chunking",
-    "embeddings",
-    "quality",
-    "keywords",
-    "api",
-    "mcp",
-    "otel",
-]
-server = ["static-pdfium", "excel", "html", "ocr", "api", "mcp"]
-cli = [
-    "static-pdfium",
-    "excel",
-    "office",
-    "html",
-    "ocr",
-    "language-detection",
-    "chunking",
-    "quality",
+    "pdf", "excel", "office", "email", "html", "xml", "archives",
+    "ocr", "language-detection", "chunking", "quality", "keywords"
 ]
+server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
+cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
 
 [build-dependencies]
-tracing = { workspace = true }
-pkg-config = "0.3"
+tracing = "0.1"
 
 [dependencies]
-ahash = { workspace = true }
-async-trait = { workspace = true }
-base64 = { workspace = true }
-base64-simd = "0.8"
-bitvec = "1.0"
-dashmap = "6.1"
-dirs = "6.0"
-simdutf8 = { version = "0.1", optional = true }
-hex = { workspace = true }
+# Core dependencies (always included)
+ahash = "0.8.12"
+async-trait = "0.1.89"
+base64 = "0.22.1"
+hex = "0.4.3"
 lazy_static = "1.5.0"
-libc = { workspace = true }
+libc = "0.2"
 memchr = "2.7.6"
-num_cpus = { workspace = true }
-once_cell = { workspace = true }
-parking_lot = { workspace = true }
-pastey = "0.2"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+paste = "1.0"
 rayon = "1.11.0"
 regex = "1.12.2"
-serde = { workspace = true }
-serde_json = { workspace = true }
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.145"
 serde_yaml_ng = "0.10.0"
-toml = { workspace = true }
+toml = "0.9.8"
 mime_guess = "2.0"
 rmp-serde = "1.3"
-thiserror = { workspace = true }
-tokio = { workspace = true, optional = true }
-uuid = { version = "1.19.0", features = ["v4", "js"] }
-indexmap = "2.13.0"
-tracing = { workspace = true }
-pdfium-render = { package = "kreuzberg-pdfium-render", version = "0.9.0", features = [
-    "thread_safe",
-    "image_latest",
-], optional = true }
+thiserror = "2.0.17"
+tokio = { version = "1.48.0", features = ["process", "fs", "rt", "rt-multi-thread", "macros", "time", "sync", "io-util"] }
+uuid = { version = "1.18.1", features = ["v4"] }
+indexmap = "2.12.1"
+tracing = "0.1"
+pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+base64-simd = "0.8"
+
+# Format extractors (optional)
+pdfium-render = { version = "0.8.36", features = ["thread_safe", "image"], optional = true }
 lopdf = { version = "0.38.0", optional = true }
 calamine = { version = "0.32.0", features = ["dates"], optional = true }
 polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
 roxmltree = { version = "0.21.1", optional = true }
-zip = { version = "7.0.0", optional = true }
+zip = { version = "6.0.0", optional = true }
 mail-parser = { version = "0.11.1", optional = true }
 msg_parser = { version = "0.1.1", optional = true }
-html-to-markdown-rs = { workspace = true, features = [
-    "inline-images",
-    "metadata",
-], optional = true }
-quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
+html-to-markdown-rs = { version = "2.9.1", features = ["inline-images"], optional = true }
+html-escape = { version = "0.2.13", optional = true }
+scraper = { version = "0.24.0", optional = true }
+quick-xml = { version = "0.38.4", optional = true }
 tar = { version = "0.4.44", optional = true }
-sevenz-rust2 = { version = "0.20.1", optional = true }
-lzma-rust2 = { workspace = true, optional = true }
+sevenz-rust = { version = "0.6.1", optional = true }
 docx-lite = { version = "0.2.0", optional = true }
 
-pulldown-cmark = { version = "0.13", optional = true }
-biblatex = { version = "0.11", optional = true }
-org = { version = "0.3", optional = true }
-rtf-parser = { version = "0.4", optional = true }
-rst_parser = { version = "0.4", optional = true }
-fb2 = { version = "0.4", optional = true }
-typst-syntax = { version = "0.14", optional = true }
-
-kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
-image = { workspace = true, default-features = false, features = [
-    "png",
-    "jpeg",
-    "webp",
-    "bmp",
-    "tiff",
-    "gif",
-    "rayon",
-], optional = true }
-tiff = { version = "0.11", optional = true }
-fast_image_resize = { version = "5.6.0", optional = true }
+# Processing features (optional)
+kreuzberg-tesseract = { version = "1.0.0-rc.1", optional = true }
+image = { version = "0.25.9", default-features = false, features = ["png", "jpeg", "webp", "bmp", "tiff", "gif", "rayon"], optional = true }
+fast_image_resize = { version = "5.1.4", optional = true }
 ndarray = { version = "0.17.1", optional = true }
 kamadak-exif = { version = "0.6.1", optional = true }
 whatlang = { version = "0.18.0", optional = true }
-text-splitter = { version = "0.29.3", features = ["markdown"], optional = true }
+text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
+fastembed = { version = "*", git = "https://github.com/kreuzberg-dev/fastembed-rs", default-features = false, features = ["hf-hub", "hf-hub-rustls-tls", "ort-download-binaries"], optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 chardetng = { version = "0.1.17", optional = true }
 encoding_rs = { version = "0.8.35", optional = true }
 yake-rust = { version = "1.0.3", optional = true }
 rake = { version = "0.3.6", optional = true }
+
+# Server features (optional)
 axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
 tower = { version = "0.5", optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.12.0", features = [
-    "server",
-    "macros",
-    "base64",
-    "transport-io",
-    "transport-streamable-http-server",
-    "axum",
-], optional = true }
-opentelemetry = { version = "0.31", features = ["trace"], optional = true }
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
-tracing-opentelemetry = { version = "0.32", optional = true }
+rmcp = { version = "0.9.0", features = ["server", "macros", "base64", "transport-io"], optional = true }
 infer = "0.19.0"
-smartcore = { version = "0.4", default-features = false, features = ["serde"] }
 
 [dev-dependencies]
-tempfile = { workspace = true }
+tempfile = "3.23.0"
 filetime = "0.2"
 tar = "0.4.44"
-zip = "7.0.0"
-serial_test = "3.3.1"
-anyhow = { workspace = true }
-tokio-test = "0.4"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-criterion = { workspace = true }
-image = { workspace = true, default-features = false, features = ["png"] }
-
-[target.'cfg(all(not(target_os = "windows"), not(target_arch = "wasm32")))'.dependencies]
-pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
-# Use rustls on non-Windows platforms (Linux, macOS)
-reqwest = { workspace = true, default-features = false, features = [
-    "json",
-    "rustls",
-], optional = true }
-# Use rustls-tls for fastembed on non-Windows platforms
-fastembed = { version = "5.7", default-features = false, features = [
-    "hf-hub-rustls-tls",
-    "ort-load-dynamic",
-], optional = true }
-# Force ureq (transitive dep via hf-hub) to use rustls on non-Windows
-ureq = { version = "3.1", default-features = false, features = ["rustls", "json"] }
-
-# Use native-tls on Windows to avoid aws-lc-sys CMake build issues with MinGW
-[target.'cfg(all(target_os = "windows", not(target_arch = "wasm32")))'.dependencies]
-reqwest = { workspace = true, default-features = false, features = [
-    "json",
-    "native-tls",
-], optional = true }
-# Use native-tls for fastembed on Windows
-fastembed = { version = "5.7", default-features = false, features = [
-    "hf-hub-native-tls",
-    "ort-load-dynamic",
-], optional = true }
-# Force ureq (transitive dep via hf-hub) to use native-tls on Windows
-ureq = { version = "3.1", default-features = false, features = ["native-tls", "json"] }
-
-[target.'cfg(target_arch = "wasm32")'.dependencies]
-wasm-bindgen-rayon = { version = "1.3", optional = true }
-# Override getrandom to enable js feature for WASM targets
-# This is needed because ring/rustls (via ureq) depend on getrandom without js feature
-getrandom = { workspace = true }
+zip = "6.0.0"
+serial_test = "3.2.0"

data/vendor/kreuzberg/README.md

@@ -1,24 +1,19 @@
-# Kreuzberg
-
-[![Rust](https://img.shields.io/crates/v/kreuzberg?label=Rust)](https://crates.io/crates/kreuzberg)
-[![Python](https://img.shields.io/pypi/v/kreuzberg?label=Python)](https://pypi.org/project/kreuzberg/)
-[![TypeScript](https://img.shields.io/npm/v/@kreuzberg/node?label=TypeScript)](https://www.npmjs.com/package/@kreuzberg/node)
-[![WASM](https://img.shields.io/npm/v/@kreuzberg/wasm?label=WASM)](https://www.npmjs.com/package/@kreuzberg/wasm)
-[![Ruby](https://img.shields.io/gem/v/kreuzberg?label=Ruby)](https://rubygems.org/gems/kreuzberg)
-[![Java](https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java)](https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg)
-[![Go](https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go)](https://pkg.go.dev/github.com/kreuzberg-dev/kreuzberg)
-[![C#](https://img.shields.io/nuget/v/Goldziher.Kreuzberg?label=C%23)](https://www.nuget.org/packages/Goldziher.Kreuzberg/)
+# Kreuzberg (Rust Core)
 
+[![Crates.io](https://img.shields.io/crates/v/kreuzberg)](https://crates.io/crates/kreuzberg)
+[![PyPI](https://img.shields.io/pypi/v/kreuzberg)](https://pypi.org/project/kreuzberg/)
+[![npm](https://img.shields.io/npm/v/@goldziher/kreuzberg)](https://www.npmjs.com/package/@goldziher/kreuzberg)
+[![RubyGems](https://img.shields.io/gem/v/kreuzberg)](https://rubygems.org/gems/kreuzberg)
+[![docs.rs](https://docs.rs/kreuzberg/badge.svg)](https://docs.rs/kreuzberg)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev/)
-[![Discord](https://img.shields.io/badge/Discord-Join%20our%20community-7289da)](https://discord.gg/pXxagNK2zN)
+[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev)
 
-High-performance document intelligence library for Rust. Extract text, metadata, and structured information from PDFs, Office documents, images, and 56 formats.
+High-performance document intelligence library for Rust. Extract text, metadata, and structured information from PDFs, Office documents, images, and 50+ formats.
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
 > **🚀 Version 4.0.0 Release Candidate**
-> This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
+> This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/Goldziher/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.
 
@@ -30,59 +25,6 @@ kreuzberg = "4.0"
 tokio = { version = "1", features = ["rt", "macros"] }
 ```
 
-## PDFium Linking Options
-
-Kreuzberg offers flexible PDFium linking strategies for different deployment scenarios. **Note:** Language bindings (Python, TypeScript, Ruby, Java, Go, C#, PHP, Elixir) automatically bundle PDFium—no configuration needed. This section applies only to the Rust crate.
-
-| Strategy | Feature Flag | Description | Use Case |
-|----------|-------------|-------------|----------|
-| **Default (Dynamic)** | None | Links to system PDFium at runtime | Development, system package users |
-| **Static** | `pdf-static` | Statically links PDFium into binary | Single binary distribution, no runtime dependencies |
-| **Bundled** | `pdf-bundled` | Downloads and embeds PDFium in binary | CI/CD, hermetic builds, largest binary size |
-| **System** | `pdf-system` | Uses system PDFium via pkg-config | Linux distributions with PDFium package |
-
-**Example Cargo.toml configurations:**
-
-```toml
-# Default (dynamic linking)
-[dependencies]
-kreuzberg = "4.0"
-
-# Static linking
-[dependencies]
-kreuzberg = { version = "4.0", features = ["pdf-static"] }
-
-# Bundled in binary
-[dependencies]
-kreuzberg = { version = "4.0", features = ["pdf-bundled"] }
-
-# System library (requires PDFium installed)
-[dependencies]
-kreuzberg = { version = "4.0", features = ["pdf-system"] }
-```
-
-For more details on feature flags and configuration options, see the [Features documentation](https://docs.kreuzberg.dev/rust/features/).
-
-## System Requirements
-
-### ONNX Runtime (for embeddings)
-
-If using embeddings functionality, ONNX Runtime must be installed:
-
-```bash
-# macOS
-brew install onnxruntime
-
-# Ubuntu/Debian
-sudo apt install libonnxruntime libonnxruntime-dev
-
-# Windows (MSVC)
-scoop install onnxruntime
-# OR download from https://github.com/microsoft/onnxruntime/releases
-```
-
-Without ONNX Runtime, embeddings will raise `MissingDependencyError` with installation instructions.
-
 ## Quick Start
 
 ```rust
@@ -222,36 +164,6 @@ kreuzberg = { version = "4.0", features = ["server"] }
 kreuzberg = { version = "4.0", features = ["cli"] }
 ```
 
-## PDF Support and Linking Options
-
-Kreuzberg supports three PDFium linking strategies. **Default is `bundled-pdfium`** (best developer experience).
-
-| Strategy | Feature | Use Case | Binary Size | Runtime Deps |
-|----------|---------|----------|-------------|--------------|
-| **Bundled (default)** | `bundled-pdfium` | Development, production | +8-15MB | None |
-| **Static** | `static-pdfium` | Docker, musl, standalone binaries | +200MB | None |
-| **System** | `system-pdfium` | Package managers, distros | +2MB | libpdfium.so |
-
-### Quick Start
-
-```toml
-# Default - bundled PDFium (recommended)
-[dependencies]
-kreuzberg = "4.0"
-
-# Static linking (Docker, musl)
-[dependencies]
-kreuzberg = { version = "4.0", features = ["static-pdfium"] }
-
-# System PDFium (package managers)
-[dependencies]
-kreuzberg = { version = "4.0", features = ["system-pdfium"] }
-```
-
-For detailed information, see the [PDFium Linking Guide](../../docs/guides/pdfium-linking.md).
-
-**Note:** Language bindings (Python, TypeScript, Ruby, Java, Go) automatically bundle PDFium. No configuration needed.
-
 ## Documentation
 
 **[API Documentation](https://docs.rs/kreuzberg)** – Complete API reference with examples