kreuzberg 4.0.0.rc1

data/spec/spec_helper.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'kreuzberg'
+require 'tmpdir'
+require 'fileutils'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_run_when_matching :focus
+  config.example_status_persistence_file_path = 'spec/examples.txt'
+  config.disable_monkey_patching!
+  config.warnings = true
+  config.default_formatter = 'doc' if config.files_to_run.one?
+  config.order = :random
+  Kernel.srand config.seed
+
+  # Helpers
+  config.include(Module.new do
+    def fixture_path(filename)
+      File.join(__dir__, 'fixtures', filename)
+    end
+
+    def test_document_path(relative_path)
+      # Go up from packages/ruby/spec to project root, then into test_documents
+      File.join(__dir__, '..', '..', '..', 'test_documents', relative_path)
+    end
+
+    def create_test_file(content, filename: 'test.txt')
+      path = File.join(Dir.tmpdir, filename)
+      File.write(path, content)
+      path
+    end
+  end)
+end

data/vendor/kreuzberg/Cargo.toml

@@ -0,0 +1,134 @@
+[package]
+name = "kreuzberg"
+version = "4.0.0-rc.1"
+edition.workspace = true
+rust-version.workspace = true
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+description = "High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 50+ formats with async/sync APIs."
+license = "MIT"
+repository = "https://github.com/Goldziher/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg"
+keywords = ["document", "extraction", "pdf", "ocr", "parsing"]
+categories = ["parsing", "text-processing", "asynchronous", "data-structures"]
+readme = "README.md"
+
+[lib]
+crate-type = ["rlib"]
+
+[features]
+default = []
+
+profiling = ["dep:pprof"]
+
+# Format extractors
+pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image"]
+excel = ["dep:calamine", "dep:polars"]
+office = ["dep:roxmltree", "dep:zip", "dep:docx-lite"]
+email = ["dep:mail-parser", "dep:msg_parser"]
+html = ["dep:html-to-markdown-rs", "dep:html-escape", "dep:scraper"]
+xml = ["dep:quick-xml", "dep:roxmltree"]
+archives = ["dep:zip", "dep:tar", "dep:sevenz-rust"]
+
+# Processing features
+ocr = ["dep:kreuzberg-tesseract", "dep:image", "dep:fast_image_resize", "dep:ndarray", "dep:kamadak-exif", "html"]
+language-detection = ["dep:whatlang"]
+chunking = ["dep:text-splitter"]
+embeddings = ["dep:fastembed", "chunking"]
+stopwords = []  # Stopwords for keyword extraction and token reduction
+quality = ["dep:unicode-normalization", "dep:chardetng", "dep:encoding_rs", "stopwords"]
+
+# Keyword extraction (requires stopwords)
+keywords-yake = ["dep:yake-rust", "stopwords"]
+keywords-rake = ["dep:rake", "stopwords"]
+keywords = ["keywords-yake", "keywords-rake"]
+
+# Server features
+api = ["dep:axum", "dep:tower", "dep:tower-http"]
+mcp = ["dep:rmcp"]
+
+# Convenience bundles
+full = [
+    "pdf", "excel", "office", "email", "html", "xml", "archives",
+    "ocr", "language-detection", "chunking", "quality", "keywords"
+]
+server = ["pdf", "excel", "html", "ocr", "api", "mcp"]
+cli = ["pdf", "excel", "office", "html", "ocr", "language-detection", "chunking", "quality"]
+
+[build-dependencies]
+tracing = "0.1"
+
+[dependencies]
+# Core dependencies (always included)
+ahash = "0.8.12"
+async-trait = "0.1.89"
+base64 = "0.22.1"
+hex = "0.4.3"
+lazy_static = "1.5.0"
+libc = "0.2"
+memchr = "2.7.6"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+paste = "1.0"
+rayon = "1.11.0"
+regex = "1.12.2"
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.145"
+serde_yaml_ng = "0.10.0"
+toml = "0.9.8"
+mime_guess = "2.0"
+rmp-serde = "1.3"
+thiserror = "2.0.17"
+tokio = { version = "1.48.0", features = ["process", "fs", "rt", "rt-multi-thread", "macros", "time", "sync", "io-util"] }
+uuid = { version = "1.18.1", features = ["v4"] }
+indexmap = "2.12.1"
+tracing = "0.1"
+pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+base64-simd = "0.8"
+
+# Format extractors (optional)
+pdfium-render = { version = "0.8.36", features = ["thread_safe", "image"], optional = true }
+lopdf = { version = "0.38.0", optional = true }
+calamine = { version = "0.32.0", features = ["dates"], optional = true }
+polars = { version = "0.52.0", default-features = false, features = ["ipc"], optional = true }
+roxmltree = { version = "0.21.1", optional = true }
+zip = { version = "6.0.0", optional = true }
+mail-parser = { version = "0.11.1", optional = true }
+msg_parser = { version = "0.1.1", optional = true }
+html-to-markdown-rs = { version = "2.9.1", features = ["inline-images"], optional = true }
+html-escape = { version = "0.2.13", optional = true }
+scraper = { version = "0.24.0", optional = true }
+quick-xml = { version = "0.38.4", optional = true }
+tar = { version = "0.4.44", optional = true }
+sevenz-rust = { version = "0.6.1", optional = true }
+docx-lite = { version = "0.2.0", optional = true }
+
+# Processing features (optional)
+kreuzberg-tesseract = { version = "1.0.0-rc.1", optional = true }
+image = { version = "0.25.9", default-features = false, features = ["png", "jpeg", "webp", "bmp", "tiff", "gif", "rayon"], optional = true }
+fast_image_resize = { version = "5.1.4", optional = true }
+ndarray = { version = "0.17.1", optional = true }
+kamadak-exif = { version = "0.6.1", optional = true }
+whatlang = { version = "0.18.0", optional = true }
+text-splitter = { version = "0.28.0", features = ["markdown"], optional = true }
+fastembed = { version = "*", git = "https://github.com/kreuzberg-dev/fastembed-rs", default-features = false, features = ["hf-hub", "hf-hub-rustls-tls", "ort-download-binaries"], optional = true }
+unicode-normalization = { version = "0.1.25", optional = true }
+chardetng = { version = "0.1.17", optional = true }
+encoding_rs = { version = "0.8.35", optional = true }
+yake-rust = { version = "1.0.3", optional = true }
+rake = { version = "0.3.6", optional = true }
+
+# Server features (optional)
+axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
+tower = { version = "0.5", optional = true }
+tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
+rmcp = { version = "0.9.0", features = ["server", "macros", "base64", "transport-io"], optional = true }
+infer = "0.19.0"
+
+[dev-dependencies]
+tempfile = "3.23.0"
+filetime = "0.2"
+tar = "0.4.44"
+zip = "6.0.0"
+serial_test = "3.2.0"

data/vendor/kreuzberg/README.md

@@ -0,0 +1,175 @@
+# Kreuzberg (Rust Core)
+
+[![Crates.io](https://img.shields.io/crates/v/kreuzberg)](https://crates.io/crates/kreuzberg)
+[![PyPI](https://img.shields.io/pypi/v/kreuzberg)](https://pypi.org/project/kreuzberg/)
+[![npm](https://img.shields.io/npm/v/@goldziher/kreuzberg)](https://www.npmjs.com/package/@goldziher/kreuzberg)
+[![RubyGems](https://img.shields.io/gem/v/kreuzberg)](https://rubygems.org/gems/kreuzberg)
+[![docs.rs](https://docs.rs/kreuzberg/badge.svg)](https://docs.rs/kreuzberg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Documentation](https://img.shields.io/badge/docs-kreuzberg.dev-blue)](https://kreuzberg.dev)
+
+High-performance document intelligence library for Rust. Extract text, metadata, and structured information from PDFs, Office documents, images, and 50+ formats.
+
+This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
+
+> **🚀 Version 4.0.0 Release Candidate**
+> This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/Goldziher/kreuzberg/issues) you encounter.
+>
+> **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.
+
+## Installation
+
+```toml
+[dependencies]
+kreuzberg = "4.0"
+tokio = { version = "1", features = ["rt", "macros"] }
+```
+
+## Quick Start
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let result = extract_file_sync("document.pdf", None, &config)?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+### Async Extraction
+
+```rust
+use kreuzberg::{extract_file, ExtractionConfig};
+
+#[tokio::main]
+async fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let result = extract_file("document.pdf", None, &config).await?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+### Batch Processing
+
+```rust
+use kreuzberg::{batch_extract_file, ExtractionConfig};
+
+#[tokio::main]
+async fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig::default();
+    let files = vec!["doc1.pdf", "doc2.pdf", "doc3.pdf"];
+    let results = batch_extract_file(&files, None, &config).await?;
+
+    for result in results {
+        println!("{}", result.content);
+    }
+    Ok(())
+}
+```
+
+## OCR with Table Extraction
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig, OcrConfig, TesseractConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: Some(TesseractConfig {
+                enable_table_detection: true,
+                ..Default::default()
+            }),
+        }),
+        ..Default::default()
+    };
+
+    let result = extract_file_sync("invoice.pdf", None, &config)?;
+
+    for table in &result.tables {
+        println!("{}", table.markdown);
+    }
+    Ok(())
+}
+```
+
+## Password-Protected PDFs
+
+```rust
+use kreuzberg::{extract_file_sync, ExtractionConfig, PdfConfig};
+
+fn main() -> kreuzberg::Result<()> {
+    let config = ExtractionConfig {
+        pdf_options: Some(PdfConfig {
+            passwords: Some(vec!["password1".to_string(), "password2".to_string()]),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let result = extract_file_sync("protected.pdf", None, &config)?;
+    Ok(())
+}
+```
+
+## Extract from Bytes
+
+```rust
+use kreuzberg::{extract_bytes_sync, ExtractionConfig};
+use std::fs;
+
+fn main() -> kreuzberg::Result<()> {
+    let data = fs::read("document.pdf")?;
+    let config = ExtractionConfig::default();
+    let result = extract_bytes_sync(&data, "application/pdf", &config)?;
+    println!("{}", result.content);
+    Ok(())
+}
+```
+
+## Features
+
+The crate uses feature flags for optional functionality:
+
+```toml
+[dependencies]
+kreuzberg = { version = "4.0", features = ["pdf", "excel", "ocr"] }
+```
+
+### Available Features
+
+| Feature | Description | Binary Size |
+|---------|-------------|-------------|
+| `pdf` | PDF extraction via pdfium | +25MB |
+| `excel` | Excel/spreadsheet parsing | +3MB |
+| `office` | DOCX, PPTX extraction | +1MB |
+| `email` | EML, MSG extraction | +500KB |
+| `html` | HTML to markdown | +1MB |
+| `xml` | XML streaming parser | +500KB |
+| `archives` | ZIP, TAR, 7Z extraction | +2MB |
+| `ocr` | OCR with Tesseract | +5MB |
+| `language-detection` | Language detection | +100KB |
+| `chunking` | Text chunking | +200KB |
+| `quality` | Text quality processing | +500KB |
+
+### Feature Bundles
+
+```toml
+kreuzberg = { version = "4.0", features = ["full"] }
+kreuzberg = { version = "4.0", features = ["server"] }
+kreuzberg = { version = "4.0", features = ["cli"] }
+```
+
+## Documentation
+
+**[API Documentation](https://docs.rs/kreuzberg)** – Complete API reference with examples
+
+**[https://kreuzberg.dev](https://kreuzberg.dev)** – User guide and tutorials
+
+## License
+
+MIT License - see [LICENSE](../../LICENSE) for details.