kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/Cargo.toml

@@ -0,0 +1,204 @@
+[package]
+name = "kreuzberg"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
+license.workspace = true
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg"
+keywords = ["document", "extraction", "pdf", "ocr", "parser"]
+categories = ["parser-implementations", "text-processing"]
+readme = "README.md"
+
+[lib]
+crate-type = ["rlib"]
+
+[features]
+default = ["tokio-runtime"]
+
+tokio-runtime = []
+profiling = ["dep:pprof"]
+
+# Format extractors
+pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
+excel = ["dep:calamine", "dep:polars"]
+office = [
+    "dep:roxmltree",
+    "dep:zip",
+    "dep:docx-lite",
+    "dep:quick-xml",
+    "dep:pulldown-cmark",
+    "dep:biblatex",
+    "dep:org",
+    "dep:rtf-parser",
+    "dep:rst_parser",
+    "dep:fb2",
+    "dep:typst-syntax",
+    "html",  # EPUB needs HTML parsing (zip + roxmltree + html-to-markdown-rs)
+]
+email = ["dep:mail-parser", "dep:msg_parser"]
+html = ["dep:html-to-markdown-rs"]
+xml = ["dep:quick-xml", "dep:roxmltree"]
+archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
+
+# Processing features
+ocr = [
+    "dep:kreuzberg-tesseract",
+    "dep:image",
+    "dep:fast_image_resize",
+    "dep:ndarray",
+    "dep:kamadak-exif",
+    "html",
+]
+language-detection = ["dep:whatlang"]
+chunking = ["dep:text-splitter"]
+embeddings = ["dep:fastembed", "chunking"]
+stopwords = [] # Stopwords for keyword extraction and token reduction
+quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
+
+# Keyword extraction (requires stopwords)
+keywords-yake = ["dep:yake-rust", "stopwords"]
+keywords-rake = ["dep:rake", "stopwords"]
+keywords = ["keywords-yake", "keywords-rake"]
+
+# Server features
+api = ["dep:axum", "dep:tower", "dep:tower-http"]
+mcp = ["dep:rmcp"]
+
+# Observability features
+otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry"]
+
+# Convenience bundles
+full = [
+    "pdf",
+    "excel",
+    "office",
+    "email",
+    "html",
+    "xml",
+    "archives",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
+    "keywords",
+]
+server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
+cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
+
+[build-dependencies]
+tracing = { workspace = true }
+
+[dependencies]
+# Core dependencies (always included)
+ahash = { workspace = true }
+async-trait = { workspace = true }
+base64 = { workspace = true }
+base64-simd = "0.8"
+hex = { workspace = true }
+lazy_static = "1.5.0"
+libc = { workspace = true }
+memchr = "2.7.6"
+num_cpus = { workspace = true }
+once_cell = "1.21.3"
+paste = "1.0"
+rayon = "1.11.0"
+regex = "1.12.2"
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_yaml_ng = "0.10.0"
+toml = "0.9.8"
+mime_guess = "2.0"
+rmp-serde = "1.3"
+thiserror = { workspace = true }
+tokio = { workspace = true }
+uuid = { version = "1.19.0", features = ["v4"] }
+indexmap = "2.12.1"
+tracing = { workspace = true }
+reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] }
+# Format extractors (optional)
+pdfium-render = { version = "0.8.37", features = ["thread_safe", "image"], optional = true }
+lopdf = { version = "0.38.0", optional = true }
+calamine = { version = "0.32.0", features = ["dates"], optional = true }
+polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
+roxmltree = { version = "0.21.1", optional = true }
+zip = { version = "6.0.0", optional = true }
+mail-parser = { version = "0.11.1", optional = true }
+msg_parser = { version = "0.1.1", optional = true }
+html-to-markdown-rs = { version = "2.12.0", features = ["inline-images"], optional = true }
+quick-xml = { version = "0.38.4", features = ["serialize"], optional = true }
+tar = { version = "0.4.44", optional = true }
+sevenz-rust = { version = "0.6.1", optional = true }
+docx-lite = { version = "0.2.0", optional = true }
+
+pulldown-cmark = { version = "0.13", optional = true }
+biblatex = { version = "0.11", optional = true }
+org = { version = "0.3", optional = true }
+rtf-parser = { version = "0.4", optional = true }
+rst_parser = { version = "0.4", optional = true }
+fb2 = { version = "0.4", optional = true }
+typst-syntax = { version = "0.14", optional = true }
+
+# Processing features (optional)
+kreuzberg-tesseract = { version = "4.0.0-rc.6", optional = true }
+image = { workspace = true, default-features = false, features = [
+    "png",
+    "jpeg",
+    "webp",
+    "bmp",
+    "tiff",
+    "gif",
+    "rayon",
+], optional = true }
+fast_image_resize = { version = "5.4.0", optional = true }
+ndarray = { version = "0.17.1", optional = true }
+kamadak-exif = { version = "0.6.1", optional = true }
+whatlang = { version = "0.18.0", optional = true }
+text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
+fastembed = { version = "5.4", default-features = false, features = [
+    "hf-hub-rustls-tls",
+    "ort-download-binaries",
+], optional = true }
+unicode-normalization = { version = "0.1.25", optional = true }
+chardetng = { version = "0.1.17", optional = true }
+encoding_rs = { version = "0.8.35", optional = true }
+yake-rust = { version = "1.0.3", optional = true }
+rake = { version = "0.3.6", optional = true }
+# Server features (optional)
+axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
+tower = { version = "0.5", optional = true }
+tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
+rmcp = { version = "0.11.0", features = [
+    "server",
+    "macros",
+    "base64",
+    "transport-io",
+], optional = true }
+# Observability features (optional)
+opentelemetry = { version = "0.31", features = ["trace"], optional = true }
+opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
+tracing-opentelemetry = { version = "0.32", optional = true }
+infer = "0.19.0"
+
+[dev-dependencies]
+tempfile = { workspace = true }
+filetime = "0.2"
+tar = "0.4.44"
+zip = "6.0.0"
+serial_test = "3.2.0"
+anyhow = { workspace = true }
+tokio-test = "0.4"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+criterion = { workspace = true }
+image = { workspace = true, default-features = false, features = ["png"] }
+
+[[bench]]
+name = "otel_overhead"
+harness = false
+
+# Only build profiling tooling on non-Windows targets (pprof depends on Unix APIs)
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }

data/vendor/kreuzberg/README.md

@@ -0,0 +1,175 @@
+# Kreuzberg (Rust Core)
+
+[![Crates.io](https://img.shields.io/crates/v/kreuzberg)](https://crates.io/crates/kreuzberg)
+[![PyPI](https://img.shields.io/pypi/v/kreuzberg)](https://pypi.org/project/kreuzberg/)
+[![npm](https://img.shields.io/npm/v/kreuzberg)](https://www.npmjs.com/package/kreuzberg)
+[![RubyGems](https://img.shields.io/gem/v/kreuzberg)](https://rubygems.org/gems/kreuzberg)
+[![docs.rs](https://docs.rs/kreuzberg/badge.svg)](https://docs.rs/kreuzberg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev)
+
+High-performance document intelligence library for Rust. Extract text, metadata, and structured information from PDFs, Office documents, images, and 50+ formats.
+
+This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
+
+> **🚀 Version 4.0.0 Release Candidate**
+> This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
+>
+> **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.
+
+## Installation
+
+```toml
+[dependencies]
+kreuzberg = "4.0"
+tokio = { version = "1", features = ["rt", "macros"] }
+```
+
+## Quick Start
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let result = extract_file_sync("document.pdf", None, &config)?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+### Async Extraction
+
+```rust
+use kreuzberg::{extract_file, ExtractionConfig};
+
+#[tokio::main]
+async fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let result = extract_file("document.pdf", None, &config).await?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+### Batch Processing
+
+```rust
+use kreuzberg::{batch_extract_file, ExtractionConfig};
+
+#[tokio::main]
+async fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let files = vec!["doc1.pdf", "doc2.pdf", "doc3.pdf"];
+    let results = batch_extract_file(&files, None, &config).await?;
+
+    for result in results {
+        println!("{}", result.content);
+    }
+    Ok(())
+}
+```
+
+## OCR with Table Extraction
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig, OcrConfig, TesseractConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: Some(TesseractConfig {
+                enable_table_detection: true,
+                ..Default::default()
+            }),
+        }),
+        ..Default::default()
+    };
+
+    let result = extract_file_sync("invoice.pdf", None, &config)?;
+
+    for table in &result.tables {
+        println!("{}", table.markdown);
+    }
+    Ok(())
+}
+```
+
+## Password-Protected PDFs
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig, PdfConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig {
+        pdf_options: Some(PdfConfig {
+            passwords: Some(vec!["password1".to_string(), "password2".to_string()]),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let result = extract_file_sync("protected.pdf", None, &config)?;
+    Ok(())
+}
+```
+
+## Extract from Bytes
+
+```rust
+use kreuzberg::{extract_bytes_sync, ExtractionConfig};
+use std::fs;
+
+fn main() -> kreuzberg::Result<()> {
+    let data = fs::read("document.pdf")?;
+    let config = ExtractionConfig::default();
+    let result = extract_bytes_sync(&data, "application/pdf", &config)?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+## Features
+
+The crate uses feature flags for optional functionality:
+
+```toml
+[dependencies]
+kreuzberg = { version = "4.0", features = ["pdf", "excel", "ocr"] }
+```
+
+### Available Features
+
+| Feature | Description | Binary Size |
+|---------|-------------|-------------|
+| `pdf` | PDF extraction via pdfium | +25MB |
+| `excel` | Excel/spreadsheet parsing | +3MB |
+| `office` | DOCX, PPTX extraction | +1MB |
+| `email` | EML, MSG extraction | +500KB |
+| `html` | HTML to markdown | +1MB |
+| `xml` | XML streaming parser | +500KB |
+| `archives` | ZIP, TAR, 7Z extraction | +2MB |
+| `ocr` | OCR with Tesseract | +5MB |
+| `language-detection` | Language detection | +100KB |
+| `chunking` | Text chunking | +200KB |
+| `quality` | Text quality processing | +500KB |
+
+### Feature Bundles
+
+```toml
+kreuzberg = { version = "4.0", features = ["full"] }
+kreuzberg = { version = "4.0", features = ["server"] }
+kreuzberg = { version = "4.0", features = ["cli"] }
+```
+
+## Documentation
+
+**[API Documentation](https://docs.rs/kreuzberg)** – Complete API reference with examples
+
+**[https://kreuzberg.dev](https://kreuzberg.dev)** – User guide and tutorials
+
+## License
+
+MIT License - see [LICENSE](../../LICENSE) for details.

data/vendor/kreuzberg/benches/otel_overhead.rs

@@ -0,0 +1,48 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use std::hint::black_box;
+
+fn bench_text_extraction(c: &mut Criterion) {
+    let runtime = tokio::runtime::Runtime::new().unwrap();
+
+    c.bench_function("extract_text_no_otel", |b| {
+        b.iter(|| {
+            runtime.block_on(async {
+                use kreuzberg::core::config::ExtractionConfig;
+                use kreuzberg::core::extractor::extract_bytes;
+
+                let test_content = black_box(b"Hello, World! This is a test document.");
+                let config = ExtractionConfig::default();
+
+                extract_bytes(test_content, "text/plain", &config).await
+            })
+        });
+    });
+}
+
+fn bench_cache_operations(c: &mut Criterion) {
+    use kreuzberg::cache::GenericCache;
+    use tempfile::tempdir;
+
+    let temp_dir = tempdir().unwrap();
+    let cache = GenericCache::new(
+        "bench".to_string(),
+        Some(temp_dir.path().to_str().unwrap().to_string()),
+        30.0,
+        500.0,
+        1000.0,
+    )
+    .unwrap();
+
+    c.bench_function("cache_set_get", |b| {
+        b.iter(|| {
+            let key = black_box("bench_key");
+            let data = black_box(b"benchmark data".to_vec());
+
+            cache.set(key, data.clone(), None).unwrap();
+            cache.get(key, None).unwrap()
+        });
+    });
+}
+
+criterion_group!(benches, bench_text_extraction, bench_cache_operations);
+criterion_main!(benches);