kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/Cargo.toml

@@ -1,134 +1,204 @@
-[package]
-name = "kreuzberg"
-version = "4.0.0-rc.1"
-edition.workspace = true
-rust-version.workspace = true
-authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
-description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
-license = "MIT"
-repository = "https://github.com/Goldziher/kreuzberg"
-homepage = "https://kreuzberg.dev"
-documentation = "https://docs.rs/kreuzberg"
-keywords = ["document", "extraction", "pdf", "ocr", "parsing"]
-categories = ["parsing", "text-processing", "asynchronous", "data-structures"]
-readme = "README.md"
-
-[lib]
-crate-type = ["rlib"]
-
-[features]
-default = []
-
-profiling = ["dep:pprof"]
-
-# Format extractors
-pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
-excel = ["dep:calamine", "dep:polars"]
-office = ["dep:roxmltree", "dep:zip", "dep:docx-lite"]
-email = ["dep:mail-parser", "dep:msg_parser"]
-html = ["dep:html-to-markdown-rs", "dep:html-escape", "dep:scraper"]
-xml = ["dep:quick-xml", "dep:roxmltree"]
-archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
-
-# Processing features
-ocr = ["dep:kreuzberg-tesseract", "dep:image", "dep:fast_image_resize", "dep:ndarray", "dep:kamadak-exif", "html"]
-language-detection = ["dep:whatlang"]
-chunking = ["dep:text-splitter"]
-embeddings = ["dep:fastembed", "chunking"]
-stopwords = []  # Stopwords for keyword extraction and token reduction
-quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
-
-# Keyword extraction (requires stopwords)
-keywords-yake = ["dep:yake-rust", "stopwords"]
-keywords-rake = ["dep:rake", "stopwords"]
-keywords = ["keywords-yake", "keywords-rake"]
-
-# Server features
-api = ["dep:axum", "dep:tower", "dep:tower-http"]
-mcp = ["dep:rmcp"]
-
-# Convenience bundles
-full = [
-    "pdf", "excel", "office", "email", "html", "xml", "archives",
-    "ocr", "language-detection", "chunking", "quality", "keywords"
-]
-server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
-cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
-
-[build-dependencies]
-tracing = "0.1"
-
-[dependencies]
-# Core dependencies (always included)
-ahash = "0.8.12"
-async-trait = "0.1.89"
-base64 = "0.22.1"
-hex = "0.4.3"
-lazy_static = "1.5.0"
-libc = "0.2"
-memchr = "2.7.6"
-num_cpus = "1.17.0"
-once_cell = "1.21.3"
-paste = "1.0"
-rayon = "1.11.0"
-regex = "1.12.2"
-serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.145"
-serde_yaml_ng = "0.10.0"
-toml = "0.9.8"
-mime_guess = "2.0"
-rmp-serde = "1.3"
-thiserror = "2.0.17"
-tokio = { version = "1.48.0", features = ["process", "fs", "rt", "rt-multi-thread", "macros", "time", "sync", "io-util"] }
-uuid = { version = "1.18.1", features = ["v4"] }
-indexmap = "2.12.1"
-tracing = "0.1"
-pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
-reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
-base64-simd = "0.8"
-
-# Format extractors (optional)
-pdfium-render = { version = "0.8.36", features = ["thread_safe", "image"], optional = true }
-lopdf = { version = "0.38.0", optional = true }
-calamine = { version = "0.32.0", features = ["dates"], optional = true }
-polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
-roxmltree = { version = "0.21.1", optional = true }
-zip = { version = "6.0.0", optional = true }
-mail-parser = { version = "0.11.1", optional = true }
-msg_parser = { version = "0.1.1", optional = true }
-html-to-markdown-rs = { version = "2.9.1", features = ["inline-images"], optional = true }
-html-escape = { version = "0.2.13", optional = true }
-scraper = { version = "0.24.0", optional = true }
-quick-xml = { version = "0.38.4", optional = true }
-tar = { version = "0.4.44", optional = true }
-sevenz-rust = { version = "0.6.1", optional = true }
-docx-lite = { version = "0.2.0", optional = true }
-
-# Processing features (optional)
-kreuzberg-tesseract = { version = "1.0.0-rc.1", optional = true }
-image = { version = "0.25.9", default-features = false, features = ["png", "jpeg", "webp", "bmp", "tiff", "gif", "rayon"], optional = true }
-fast_image_resize = { version = "5.1.4", optional = true }
-ndarray = { version = "0.17.1", optional = true }
-kamadak-exif = { version = "0.6.1", optional = true }
-whatlang = { version = "0.18.0", optional = true }
-text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
-fastembed = { version = "*", git = "https://github.com/kreuzberg-dev/fastembed-rs", default-features = false, features = ["hf-hub", "hf-hub-rustls-tls", "ort-download-binaries"], optional = true }
-unicode-normalization = { version = "0.1.25", optional = true }
-chardetng = { version = "0.1.17", optional = true }
-encoding_rs = { version = "0.8.35", optional = true }
-yake-rust = { version = "1.0.3", optional = true }
-rake = { version = "0.3.6", optional = true }
-
-# Server features (optional)
-axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
-tower = { version = "0.5", optional = true }
-tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.9.0", features = ["server", "macros", "base64", "transport-io"], optional = true }
-infer = "0.19.0"
-
-[dev-dependencies]
-tempfile = "3.23.0"
-filetime = "0.2"
-tar = "0.4.44"
-zip = "6.0.0"
-serial_test = "3.2.0"
+[package]
+name = "kreuzberg"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
+license.workspace = true
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg"
+keywords = ["document", "extraction", "pdf", "ocr", "parser"]
+categories = ["parser-implementations", "text-processing"]
+readme = "README.md"
+
+[lib]
+crate-type = ["rlib"]
+
+[features]
+default = ["tokio-runtime"]
+
+tokio-runtime = []
+profiling = ["dep:pprof"]
+
+# Format extractors
+pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
+excel = ["dep:calamine", "dep:polars"]
+office = [
+    "dep:roxmltree",
+    "dep:zip",
+    "dep:docx-lite",
+    "dep:quick-xml",
+    "dep:pulldown-cmark",
+    "dep:biblatex",
+    "dep:org",
+    "dep:rtf-parser",
+    "dep:rst_parser",
+    "dep:fb2",
+    "dep:typst-syntax",
+    "html",  # EPUB needs HTML parsing (zip + roxmltree + html-to-markdown-rs)
+]
+email = ["dep:mail-parser", "dep:msg_parser"]
+html = ["dep:html-to-markdown-rs"]
+xml = ["dep:quick-xml", "dep:roxmltree"]
+archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
+
+# Processing features
+ocr = [
+    "dep:kreuzberg-tesseract",
+    "dep:image",
+    "dep:fast_image_resize",
+    "dep:ndarray",
+    "dep:kamadak-exif",
+    "html",
+]
+language-detection = ["dep:whatlang"]
+chunking = ["dep:text-splitter"]
+embeddings = ["dep:fastembed", "chunking"]
+stopwords = [] # Stopwords for keyword extraction and token reduction
+quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
+
+# Keyword extraction (requires stopwords)
+keywords-yake = ["dep:yake-rust", "stopwords"]
+keywords-rake = ["dep:rake", "stopwords"]
+keywords = ["keywords-yake", "keywords-rake"]
+
+# Server features
+api = ["dep:axum", "dep:tower", "dep:tower-http"]
+mcp = ["dep:rmcp"]
+
+# Observability features
+otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
+
+# Convenience bundles
+full = [
+    "pdf",
+    "excel",
+    "office",
+    "email",
+    "html",
+    "xml",
+    "archives",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
+    "keywords",
+]
+server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
+cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
+
+[build-dependencies]
+tracing = { workspace = true }
+
+[dependencies]
+# Core dependencies (always included)
+ahash = { workspace = true }
+async-trait = { workspace = true }
+base64 = { workspace = true }
+base64-simd = "0.8"
+hex = { workspace = true }
+lazy_static = "1.5.0"
+libc = { workspace = true }
+memchr = "2.7.6"
+num_cpus = { workspace = true }
+once_cell = "1.21.3"
+paste = "1.0"
+rayon = "1.11.0"
+regex = "1.12.2"
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_yaml_ng = "0.10.0"
+toml = "0.9.8"
+mime_guess = "2.0"
+rmp-serde = "1.3"
+thiserror = { workspace = true }
+tokio = { workspace = true }
+uuid = { version = "1.19.0", features = ["v4"] }
+indexmap = "2.12.1"
+tracing = { workspace = true }
+reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] }
+# Format extractors (optional)
+pdfium-render = { version = "0.8.37", features = ["thread_safe", "image"], optional = true }
+lopdf = { version = "0.38.0", optional = true }
+calamine = { version = "0.32.0", features = ["dates"], optional = true }
+polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
+roxmltree = { version = "0.21.1", optional = true }
+zip = { version = "6.0.0", optional = true }
+mail-parser = { version = "0.11.1", optional = true }
+msg_parser = { version = "0.1.1", optional = true }
+html-to-markdown-rs = { version = "2.12.0", features = ["inline-images"], optional = true }
+quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
+tar = { version = "0.4.44", optional = true }
+sevenz-rust = { version = "0.6.1", optional = true }
+docx-lite = { version = "0.2.0", optional = true }
+
+pulldown-cmark = { version = "0.13", optional = true }
+biblatex = { version = "0.11", optional = true }
+org = { version = "0.3", optional = true }
+rtf-parser = { version = "0.4", optional = true }
+rst_parser = { version = "0.4", optional = true }
+fb2 = { version = "0.4", optional = true }
+typst-syntax = { version = "0.14", optional = true }
+
+# Processing features (optional)
+kreuzberg-tesseract = { version = "4.0.0-rc.6", optional = true }
+image = { workspace = true, default-features = false, features = [
+    "png",
+    "jpeg",
+    "webp",
+    "bmp",
+    "tiff",
+    "gif",
+    "rayon",
+], optional = true }
+fast_image_resize = { version = "5.4.0", optional = true }
+ndarray = { version = "0.17.1", optional = true }
+kamadak-exif = { version = "0.6.1", optional = true }
+whatlang = { version = "0.18.0", optional = true }
+text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
+fastembed = { version = "5.4", default-features = false, features = [
+    "hf-hub-rustls-tls",
+    "ort-download-binaries",
+], optional = true }
+unicode-normalization = { version = "0.1.25", optional = true }
+chardetng = { version = "0.1.17", optional = true }
+encoding_rs = { version = "0.8.35", optional = true }
+yake-rust = { version = "1.0.3", optional = true }
+rake = { version = "0.3.6", optional = true }
+# Server features (optional)
+axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
+tower = { version = "0.5", optional = true }
+tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
+rmcp = { version = "0.11.0", features = [
+    "server",
+    "macros",
+    "base64",
+    "transport-io",
+], optional = true }
+# Observability features (optional)
+opentelemetry = { version = "0.31", features = ["trace"], optional = true }
+opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
+tracing-opentelemetry = { version = "0.32", optional = true }
+infer = "0.19.0"
+
+[dev-dependencies]
+tempfile = { workspace = true }
+filetime = "0.2"
+tar = "0.4.44"
+zip = "6.0.0"
+serial_test = "3.2.0"
+anyhow = { workspace = true }
+tokio-test = "0.4"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+criterion = { workspace = true }
+image = { workspace = true, default-features = false, features = ["png"] }
+
+[[bench]]
+name = "otel_overhead"
+harness = false
+
+# Only build profiling tooling on non-Windows targets (pprof depends on Unix APIs)
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }