kreuzberg 4.0.0.rc2 → 4.0.0

data/vendor/Cargo.toml

@@ -0,0 +1,61 @@
+[workspace]
+members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
+resolver = "2"
+
+[workspace.package]
+version = "4.0.0"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+license = "MIT"
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+
+[workspace.dependencies]
+# Core async runtime
+tokio = { version = "1.49.0", features = [
+    "rt",
+    "rt-multi-thread",
+    "macros",
+    "sync",
+    "process",
+    "fs",
+    "time",
+    "io-util",
+] }
+
+# Serialization
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.149"
+
+# Error handling
+thiserror = "2.0.17"
+anyhow = "1.0"
+
+# Async utilities
+async-trait = "0.1.89"
+libc = "0.2.180"
+parking_lot = "0.12.5"
+
+# Tracing/observability
+tracing = "0.1"
+
+# Utilities
+ahash = "0.8.12"
+base64 = "0.22.1"
+hex = "0.4.3"
+toml = "0.9.11"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+html-to-markdown-rs = { version = "2.20.0", default-features = false }
+reqwest = { version = "0.13.1", default-features = false, features = ["json", "rustls"] }
+image = { version = "0.25.9", default-features = false }
+lzma-rust2 = { version = "0.15.6" }
+
+# Fix for WASM builds: ensure getrandom has wasm_js feature enabled
+# This is needed because ring/rustls depend on getrandom without the wasm_js feature
+getrandom = { version = "0.3.4", features = ["wasm_js"] }
+
+# Testing (dev)
+tempfile = "3.24.0"
+criterion = { version = "0.8", features = ["html_reports"] }

data/vendor/kreuzberg/Cargo.toml

@@ -1,204 +1,259 @@
-[package]
-name = "kreuzberg"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-authors.workspace = true
-description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
-license.workspace = true
-repository = "https://github.com/kreuzberg-dev/kreuzberg"
-homepage = "https://kreuzberg.dev"
-documentation = "https://docs.rs/kreuzberg"
-keywords = ["document", "extraction", "pdf", "ocr", "parser"]
-categories = ["parser-implementations", "text-processing"]
-readme = "README.md"
-
-[lib]
-crate-type = ["rlib"]
-
-[features]
-default = ["tokio-runtime"]
-
-tokio-runtime = []
-profiling = ["dep:pprof"]
-
-# Format extractors
-pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
-excel = ["dep:calamine", "dep:polars"]
-office = [
-    "dep:roxmltree",
-    "dep:zip",
-    "dep:docx-lite",
-    "dep:quick-xml",
-    "dep:pulldown-cmark",
-    "dep:biblatex",
-    "dep:org",
-    "dep:rtf-parser",
-    "dep:rst_parser",
-    "dep:fb2",
-    "dep:typst-syntax",
-    "html",  # EPUB needs HTML parsing (zip + roxmltree + html-to-markdown-rs)
-]
-email = ["dep:mail-parser", "dep:msg_parser"]
-html = ["dep:html-to-markdown-rs"]
-xml = ["dep:quick-xml", "dep:roxmltree"]
-archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
-
-# Processing features
-ocr = [
-    "dep:kreuzberg-tesseract",
-    "dep:image",
-    "dep:fast_image_resize",
-    "dep:ndarray",
-    "dep:kamadak-exif",
-    "html",
-]
-language-detection = ["dep:whatlang"]
-chunking = ["dep:text-splitter"]
-embeddings = ["dep:fastembed", "chunking"]
-stopwords = [] # Stopwords for keyword extraction and token reduction
-quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
-
-# Keyword extraction (requires stopwords)
-keywords-yake = ["dep:yake-rust", "stopwords"]
-keywords-rake = ["dep:rake", "stopwords"]
-keywords = ["keywords-yake", "keywords-rake"]
-
-# Server features
-api = ["dep:axum", "dep:tower", "dep:tower-http"]
-mcp = ["dep:rmcp"]
-
-# Observability features
-otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
-
-# Convenience bundles
-full = [
-    "pdf",
-    "excel",
-    "office",
-    "email",
-    "html",
-    "xml",
-    "archives",
-    "ocr",
-    "language-detection",
-    "chunking",
-    "quality",
-    "keywords",
-]
-server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
-cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
-
-[build-dependencies]
-tracing = { workspace = true }
-
-[dependencies]
-# Core dependencies (always included)
-ahash = { workspace = true }
-async-trait = { workspace = true }
-base64 = { workspace = true }
-base64-simd = "0.8"
-hex = { workspace = true }
-lazy_static = "1.5.0"
-libc = { workspace = true }
-memchr = "2.7.6"
-num_cpus = { workspace = true }
-once_cell = "1.21.3"
-paste = "1.0"
-rayon = "1.11.0"
-regex = "1.12.2"
-serde = { workspace = true }
-serde_json = { workspace = true }
-serde_yaml_ng = "0.10.0"
-toml = "0.9.8"
-mime_guess = "2.0"
-rmp-serde = "1.3"
-thiserror = { workspace = true }
-tokio = { workspace = true }
-uuid = { version = "1.19.0", features = ["v4"] }
-indexmap = "2.12.1"
-tracing = { workspace = true }
-reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] }
-# Format extractors (optional)
-pdfium-render = { version = "0.8.37", features = ["thread_safe", "image"], optional = true }
-lopdf = { version = "0.38.0", optional = true }
-calamine = { version = "0.32.0", features = ["dates"], optional = true }
-polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
-roxmltree = { version = "0.21.1", optional = true }
-zip = { version = "6.0.0", optional = true }
-mail-parser = { version = "0.11.1", optional = true }
-msg_parser = { version = "0.1.1", optional = true }
-html-to-markdown-rs = { version = "2.12.0", features = ["inline-images"], optional = true }
-quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
-tar = { version = "0.4.44", optional = true }
-sevenz-rust = { version = "0.6.1", optional = true }
-docx-lite = { version = "0.2.0", optional = true }
-
-pulldown-cmark = { version = "0.13", optional = true }
-biblatex = { version = "0.11", optional = true }
-org = { version = "0.3", optional = true }
-rtf-parser = { version = "0.4", optional = true }
-rst_parser = { version = "0.4", optional = true }
-fb2 = { version = "0.4", optional = true }
-typst-syntax = { version = "0.14", optional = true }
-
-# Processing features (optional)
-kreuzberg-tesseract = { version = "4.0.0-rc.6", optional = true }
-image = { workspace = true, default-features = false, features = [
-    "png",
-    "jpeg",
-    "webp",
-    "bmp",
-    "tiff",
-    "gif",
-    "rayon",
-], optional = true }
-fast_image_resize = { version = "5.4.0", optional = true }
-ndarray = { version = "0.17.1", optional = true }
-kamadak-exif = { version = "0.6.1", optional = true }
-whatlang = { version = "0.18.0", optional = true }
-text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
-fastembed = { version = "5.4", default-features = false, features = [
-    "hf-hub-rustls-tls",
-    "ort-download-binaries",
-], optional = true }
-unicode-normalization = { version = "0.1.25", optional = true }
-chardetng = { version = "0.1.17", optional = true }
-encoding_rs = { version = "0.8.35", optional = true }
-yake-rust = { version = "1.0.3", optional = true }
-rake = { version = "0.3.6", optional = true }
-# Server features (optional)
-axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
-tower = { version = "0.5", optional = true }
-tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.11.0", features = [
-    "server",
-    "macros",
-    "base64",
-    "transport-io",
-], optional = true }
-# Observability features (optional)
-opentelemetry = { version = "0.31", features = ["trace"], optional = true }
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
-tracing-opentelemetry = { version = "0.32", optional = true }
-infer = "0.19.0"
-
-[dev-dependencies]
-tempfile = { workspace = true }
-filetime = "0.2"
-tar = "0.4.44"
-zip = "6.0.0"
-serial_test = "3.2.0"
-anyhow = { workspace = true }
-tokio-test = "0.4"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-criterion = { workspace = true }
-image = { workspace = true, default-features = false, features = ["png"] }
-
-[[bench]]
-name = "otel_overhead"
-harness = false
-
-# Only build profiling tooling on non-Windows targets (pprof depends on Unix APIs)
-[target.'cfg(not(target_os = "windows"))'.dependencies]
-pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
+[package]
+name = "kreuzberg"
+version = "4.0.0"
+edition = "2024"
+rust-version = "1.91"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
+license = "MIT"
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg"
+keywords = ["document", "extraction", "pdf", "ocr", "parser"]
+categories = ["parser-implementations", "text-processing"]
+readme = "README.md"
+
+[lib]
+crate-type = ["rlib"]
+
+[features]
+default = ["tokio-runtime", "simd-utf8"]
+
+profiling = ["dep:pprof"]
+
+pool-metrics = []
+
+simd-utf8 = ["dep:simdutf8"]
+
+tokio-runtime = ["dep:tokio"]
+
+pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
+static-pdfium = ["pdf"]
+bundled-pdfium = ["pdf"]
+system-pdfium = ["pdf"]
+excel = ["dep:calamine", "dep:polars", "tokio-runtime"]
+office = [
+    "dep:roxmltree",
+    "dep:zip",
+    "dep:docx-lite",
+    "dep:quick-xml",
+    "dep:pulldown-cmark",
+    "dep:biblatex",
+    "dep:org",
+    "dep:rtf-parser",
+    "dep:rst_parser",
+    "dep:fb2",
+    "dep:typst-syntax",
+    "html",
+    "tokio-runtime",
+]
+email = ["dep:mail-parser", "dep:msg_parser"]
+html = ["dep:html-to-markdown-rs"]
+xml = ["dep:quick-xml", "dep:roxmltree"]
+archives = ["dep:zip", "dep:tar", "dep:sevenz-rust2", "dep:lzma-rust2"]
+
+ocr = [
+    "dep:kreuzberg-tesseract",
+    "dep:image",
+    "dep:tiff",
+    "dep:fast_image_resize",
+    "dep:ndarray",
+    "dep:kamadak-exif",
+    "html",
+]
+language-detection = ["dep:whatlang"]
+chunking = ["dep:text-splitter"]
+embeddings = ["dep:fastembed", "dep:reqwest", "chunking", "tokio-runtime"]
+stopwords = []
+quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
+
+keywords-yake = ["dep:yake-rust", "stopwords"]
+keywords-rake = ["dep:rake", "stopwords"]
+keywords = ["keywords-yake", "keywords-rake"]
+
+api = ["dep:axum", "dep:tower", "dep:tower-http", "tokio-runtime"]
+mcp = ["dep:rmcp", "tokio-runtime"]
+mcp-http = ["mcp", "api"]
+
+otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
+
+wasm-target = ["pdf", "html", "xml", "email", "language-detection", "chunking", "quality"]
+wasm-threads = ["dep:wasm-bindgen-rayon"]
+
+full = [
+    "pdf",
+    "excel",
+    "office",
+    "email",
+    "html",
+    "xml",
+    "archives",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "embeddings",
+    "quality",
+    "keywords",
+    "api",
+    "mcp",
+    "otel",
+]
+server = ["static-pdfium", "excel", "html", "ocr", "api", "mcp"]
+cli = [
+    "static-pdfium",
+    "excel",
+    "office",
+    "html",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
+]
+
+[build-dependencies]
+tracing = { workspace = true }
+pkg-config = "0.3"
+
+[dependencies]
+ahash = { workspace = true }
+async-trait = { workspace = true }
+base64 = { workspace = true }
+base64-simd = "0.8"
+bitvec = "1.0"
+dashmap = "6.1"
+dirs = "6.0"
+simdutf8 = { version = "0.1", optional = true }
+hex = { workspace = true }
+lazy_static = "1.5.0"
+libc = { workspace = true }
+memchr = "2.7.6"
+num_cpus = { workspace = true }
+once_cell = { workspace = true }
+parking_lot = { workspace = true }
+pastey = "0.2"
+rayon = "1.11.0"
+regex = "1.12.2"
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_yaml_ng = "0.10.0"
+toml = { workspace = true }
+mime_guess = "2.0"
+rmp-serde = "1.3"
+thiserror = { workspace = true }
+tokio = { workspace = true, optional = true }
+uuid = { version = "1.19.0", features = ["v4", "js"] }
+indexmap = "2.13.0"
+tracing = { workspace = true }
+pdfium-render = { package = "kreuzberg-pdfium-render", version = "0.9.0", features = [
+    "thread_safe",
+    "image_latest",
+], optional = true }
+lopdf = { version = "0.38.0", optional = true }
+calamine = { version = "0.32.0", features = ["dates"], optional = true }
+polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
+roxmltree = { version = "0.21.1", optional = true }
+zip = { version = "7.0.0", optional = true }
+mail-parser = { version = "0.11.1", optional = true }
+msg_parser = { version = "0.1.1", optional = true }
+html-to-markdown-rs = { workspace = true, features = [
+    "inline-images",
+    "metadata",
+], optional = true }
+quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
+tar = { version = "0.4.44", optional = true }
+sevenz-rust2 = { version = "0.20.1", optional = true }
+lzma-rust2 = { workspace = true, optional = true }
+docx-lite = { version = "0.2.0", optional = true }
+
+pulldown-cmark = { version = "0.13", optional = true }
+biblatex = { version = "0.11", optional = true }
+org = { version = "0.3", optional = true }
+rtf-parser = { version = "0.4", optional = true }
+rst_parser = { version = "0.4", optional = true }
+fb2 = { version = "0.4", optional = true }
+typst-syntax = { version = "0.14", optional = true }
+
+kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
+image = { workspace = true, default-features = false, features = [
+    "png",
+    "jpeg",
+    "webp",
+    "bmp",
+    "tiff",
+    "gif",
+    "rayon",
+], optional = true }
+tiff = { version = "0.11", optional = true }
+fast_image_resize = { version = "5.6.0", optional = true }
+ndarray = { version = "0.17.1", optional = true }
+kamadak-exif = { version = "0.6.1", optional = true }
+whatlang = { version = "0.18.0", optional = true }
+text-splitter = { version = "0.29.3", features = ["markdown"], optional = true }
+unicode-normalization = { version = "0.1.25", optional = true }
+chardetng = { version = "0.1.17", optional = true }
+encoding_rs = { version = "0.8.35", optional = true }
+yake-rust = { version = "1.0.3", optional = true }
+rake = { version = "0.3.6", optional = true }
+axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
+tower = { version = "0.5", optional = true }
+tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
+rmcp = { version = "0.12.0", features = [
+    "server",
+    "macros",
+    "base64",
+    "transport-io",
+    "transport-streamable-http-server",
+    "axum",
+], optional = true }
+opentelemetry = { version = "0.31", features = ["trace"], optional = true }
+opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
+tracing-opentelemetry = { version = "0.32", optional = true }
+infer = "0.19.0"
+smartcore = { version = "0.4", default-features = false, features = ["serde"] }
+
+[dev-dependencies]
+tempfile = { workspace = true }
+filetime = "0.2"
+tar = "0.4.44"
+zip = "7.0.0"
+serial_test = "3.3.1"
+anyhow = { workspace = true }
+tokio-test = "0.4"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+criterion = { workspace = true }
+image = { workspace = true, default-features = false, features = ["png"] }
+
+[target.'cfg(all(not(target_os = "windows"), not(target_arch = "wasm32")))'.dependencies]
+pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
+# Use rustls on non-Windows platforms (Linux, macOS)
+reqwest = { workspace = true, default-features = false, features = [
+    "json",
+    "rustls",
+], optional = true }
+# Use rustls-tls for fastembed on non-Windows platforms
+fastembed = { version = "5.7", default-features = false, features = [
+    "hf-hub-rustls-tls",
+    "ort-load-dynamic",
+], optional = true }
+# Force ureq (transitive dep via hf-hub) to use rustls on non-Windows
+ureq = { version = "3.1", default-features = false, features = ["rustls", "json"] }
+
+# Use native-tls on Windows to avoid aws-lc-sys CMake build issues with MinGW
+[target.'cfg(all(target_os = "windows", not(target_arch = "wasm32")))'.dependencies]
+reqwest = { workspace = true, default-features = false, features = [
+    "json",
+    "native-tls",
+], optional = true }
+# Use native-tls for fastembed on Windows
+fastembed = { version = "5.7", default-features = false, features = [
+    "hf-hub-native-tls",
+    "ort-load-dynamic",
+], optional = true }
+# Force ureq (transitive dep via hf-hub) to use native-tls on Windows
+ureq = { version = "3.1", default-features = false, features = ["native-tls", "json"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-bindgen-rayon = { version = "1.3", optional = true }
+# Override getrandom to enable js feature for WASM targets
+# This is needed because ring/rustls (via ureq) depend on getrandom without js feature
+getrandom = { workspace = true }