kreuzberg 4.0.0.pre.rc.16 → 4.0.0.pre.rc.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe3add89c26722e26baf090f7b9a0c32671c449be6a34ea4285a5f6d15548b72
-  data.tar.gz: 49147ceab3fddc3161ff0df55f7c535134d63da7ce2577aad905c91179e875f3
+  metadata.gz: a1c1d7ddcc45217bf5d9ea47a4d9d3ef9f41ed5a4bd87f4ff1f2ada7cfe0bca6
+  data.tar.gz: 167fb6c623c9e4368bcd2388e0ef4631d170d0e167cc0555c2dc7cd814bff9eb
 SHA512:
-  metadata.gz: 5f2e0ab3d3dd4c975a99dcbf4a2e81347673eb74687034f8ef72cc3ece6561fbbed70811edc7363c911385c2d7c2eb0be2d8fa990872845458a3d3f5f019422c
-  data.tar.gz: 530bf825eb92e9a3df838ab14ec68277b17e575833ecdf0af11e32d8749e101e4fe68195841524c5e6b41c31a2330076e9da4507f5edf047d8670ad26c9dd928
+  metadata.gz: 37661576ba03012b1549c2388e0aee4c24cdb0e05cb34164fc85ca654c20236b28829678531c50fe46260433c43907c3f03cf516753f12c78ee2026a0b14a446
+  data.tar.gz: 96e1b3b10589fa7f47fc9609158da556e7d889a61963953c72e5f92337897828d68934d8dc0a66d7b2e28774204b6b1bf2c95226d5c26c84a98a01ce797e24df

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.0.0.pre.rc.16)
+    kreuzberg (4.0.0.pre.rc.18)
 
 GEM
   remote: https://rubygems.org/
@@ -33,7 +33,7 @@ GEM
     ffi (1.17.2-arm64-darwin)
     ffi (1.17.2-x86_64-linux-gnu)
     fileutils (1.8.0)
-    i18n (1.14.7)
+    i18n (1.14.8)
       concurrent-ruby (~> 1.0)
     json (2.18.0)
     language_server-protocol (3.17.0.5)

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -455,6 +455,18 @@ dependencies = [
  "core2",
 ]
 
+[[package]]
+name = "bitvec"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "blake3"
 version = "1.8.2"
@@ -987,6 +999,20 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "dashmap"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "debug_unsafe"
 version = "0.1.3"
@@ -1434,6 +1460,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "funty"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
+
 [[package]]
 name = "futf"
 version = "0.1.5"
@@ -1638,6 +1670,12 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
@@ -2280,7 +2318,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg"
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.17"
 dependencies = [
  "ahash",
  "async-trait",
@@ -2288,8 +2326,10 @@ dependencies = [
  "base64 0.22.1",
  "base64-simd",
  "biblatex",
+ "bitvec",
  "calamine",
  "chardetng",
+ "dashmap",
  "docx-lite",
  "encoding_rs",
  "fast_image_resize",
@@ -2334,6 +2374,7 @@ dependencies = [
  "serde_json",
  "serde_yaml_ng",
  "sevenz-rust2",
+ "simdutf8",
  "tar",
  "text-splitter",
  "thiserror 2.0.17",
@@ -2354,7 +2395,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.16"
+version = "4.0.0-rc.17"
 dependencies = [
  "async-trait",
  "html-to-markdown-rs",
@@ -2368,7 +2409,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-tesseract"
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.17"
 dependencies = [
  "cc",
  "cmake",
@@ -4124,6 +4165,12 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
+[[package]]
+name = "radium"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
+
 [[package]]
 name = "rake"
 version = "0.3.6"
@@ -5218,6 +5265,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "tar"
 version = "0.4.44"
@@ -6439,6 +6492,15 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
 
+[[package]]
+name = "wyz"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
+dependencies = [
+ "tap",
+]
+
 [[package]]
 name = "xattr"
 version = "1.6.1"

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -7,7 +7,7 @@ rb-sys = { path = "../../../vendor/rb-sys" }
 
 [package]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.16"
+version = "4.0.0-rc.18"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/ext/kreuzberg_rb/native/build.rs

@@ -1,52 +1,77 @@
-#[cfg(target_os = "macos")]
+use std::env;
+use std::path::PathBuf;
+
 fn main() {
-    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        let lib_path = std::path::Path::new(&cargo_manifest_dir)
-            .parent()
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .map(|p| p.join("target/release"))
-            .expect("Failed to construct lib path");
-        println!("cargo:rustc-link-search={}", lib_path.display());
+    let target = env::var("TARGET").unwrap();
+    let profile = env::var("PROFILE").unwrap_or_else(|_| "release".to_string());
+
+    // Try to locate kreuzberg-ffi library built alongside this crate
+    let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+    let manifest_path = PathBuf::from(&cargo_manifest_dir);
+
+    // Prefer host target layout, but include target-triple layout for cross builds.
+    // IMPORTANT: Only search lib directories, NOT deps directories.
+    // The deps/ directories may contain dylibs with hardcoded install_name paths,
+    // which causes load errors on macOS when users install the gem.
+    if let Some(packages_root) = manifest_path
+        .parent()
+        .and_then(|p| p.parent())
+        .and_then(|p| p.parent())
+        .and_then(|p| p.parent())
+        .and_then(|p| p.parent())
+    {
+        let host_lib_dir = packages_root.join("target").join(&profile);
+        let target_lib_dir = packages_root.join("target").join(&target).join(&profile);
+
+        // Try to find the static library and link it directly on Unix-like systems
+        // to avoid the linker preferring dylib over static lib.
+        if !target.contains("windows") {
+            let static_lib_name = if target.contains("windows") {
+                "kreuzberg_ffi.lib"
+            } else {
+                "libkreuzberg_ffi.a"
+            };
+
+            // Check both host and target lib directories for the static library
+            for lib_dir in [&host_lib_dir, &target_lib_dir] {
+                let static_lib = lib_dir.join(static_lib_name);
+                if static_lib.exists() {
+                    // Found static library, link it directly by passing the full path
+                    println!("cargo:rustc-link-arg={}", static_lib.display());
+                    // Don't add the library search path or -l flag
+                    // Jump to platform-specific configuration
+                    if target.contains("darwin") {
+                        println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
+                        println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
+                    } else if target.contains("linux") {
+                        println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
+                    }
+                    println!("cargo:rerun-if-changed=build.rs");
+                    return;
+                }
+            }
+        }
+
+        // Fallback: Add search paths and use standard linking
+        for dir in [host_lib_dir, target_lib_dir] {
+            println!("cargo:rustc-link-search=native={}", dir.display());
+        }
     }
-    println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
-    println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
-    println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path/.");
-}
 
-#[cfg(target_os = "linux")]
-fn main() {
-    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        let lib_path = std::path::Path::new(&cargo_manifest_dir)
-            .parent()
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .map(|p| p.join("target/release"))
-            .expect("Failed to construct lib path");
-        println!("cargo:rustc-link-search={}", lib_path.display());
+    // Link the kreuzberg-ffi library
+    // When kreuzberg-ffi is built, its symbols become available for linking
+    if target.contains("windows") {
+        println!("cargo:rustc-link-lib=dylib=kreuzberg_ffi");
+    } else {
+        println!("cargo:rustc-link-lib=static=kreuzberg_ffi");
     }
-    println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
-    println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/.");
-}
 
-#[cfg(target_os = "windows")]
-fn main() {
-    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        let lib_path = std::path::Path::new(&cargo_manifest_dir)
-            .parent()
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .and_then(|p| p.parent())
-            .map(|p| p.join("target/release"))
-            .expect("Failed to construct lib path");
-        println!("cargo:rustc-link-search={}", lib_path.display());
+    if target.contains("darwin") {
+        println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
+        println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
+    } else if target.contains("linux") {
+        println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
     }
-}
 
-#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
-fn main() {}
+    println!("cargo:rerun-if-changed=build.rs");
+}

data/ext/kreuzberg_rb/native/src/lib.rs

@@ -78,8 +78,7 @@ pub struct CMetadataField {
 }
 
 // These C ABI functions are provided by the kreuzberg-ffi crate
-// We declare them here to ensure proper linking on all platforms
-#[link(name = "kreuzberg_ffi", kind = "static")]
+// Linking is handled by build.rs to ensure static linking
 unsafe extern "C" {
     pub fn kreuzberg_last_error_code() -> i32;
     pub fn kreuzberg_last_panic_context() -> *mut c_char;
@@ -3071,7 +3070,7 @@ fn validate_chunking_params(max_chars: usize, max_overlap: usize) -> Result<i32,
 /// Gets valid binarization methods as a JSON string
 ///
 /// @return [String] JSON array of valid binarization methods
-fn get_valid_binarization_methods(ruby: &Ruby) -> Result<String, Error> {
+fn get_valid_binarization_methods(_ruby: &Ruby) -> Result<String, Error> {
     let ptr = unsafe { kreuzberg_get_valid_binarization_methods() };
     if ptr.is_null() {
         return Err(runtime_error("Failed to get valid binarization methods"));
@@ -3095,7 +3094,7 @@ fn get_valid_binarization_methods(ruby: &Ruby) -> Result<String, Error> {
 /// Gets valid language codes as a JSON string
 ///
 /// @return [String] JSON array of valid language codes
-fn get_valid_language_codes(ruby: &Ruby) -> Result<String, Error> {
+fn get_valid_language_codes(_ruby: &Ruby) -> Result<String, Error> {
     let ptr = unsafe { kreuzberg_get_valid_language_codes() };
     if ptr.is_null() {
         return Err(runtime_error("Failed to get valid language codes"));
@@ -3119,7 +3118,7 @@ fn get_valid_language_codes(ruby: &Ruby) -> Result<String, Error> {
 /// Gets valid OCR backends as a JSON string
 ///
 /// @return [String] JSON array of valid OCR backends
-fn get_valid_ocr_backends(ruby: &Ruby) -> Result<String, Error> {
+fn get_valid_ocr_backends(_ruby: &Ruby) -> Result<String, Error> {
     let ptr = unsafe { kreuzberg_get_valid_ocr_backends() };
     if ptr.is_null() {
         return Err(runtime_error("Failed to get valid OCR backends"));
@@ -3143,7 +3142,7 @@ fn get_valid_ocr_backends(ruby: &Ruby) -> Result<String, Error> {
 /// Gets valid token reduction levels as a JSON string
 ///
 /// @return [String] JSON array of valid token reduction levels
-fn get_valid_token_reduction_levels(ruby: &Ruby) -> Result<String, Error> {
+fn get_valid_token_reduction_levels(_ruby: &Ruby) -> Result<String, Error> {
     let ptr = unsafe { kreuzberg_get_valid_token_reduction_levels() };
     if ptr.is_null() {
         return Err(runtime_error("Failed to get valid token reduction levels"));
@@ -3395,7 +3394,7 @@ fn get_error_details_native(ruby: &Ruby) -> Result<Value, Error> {
     // SAFETY: FFI function is thread-safe and returns a struct with allocated C strings
     let details = unsafe { kreuzberg_get_error_details() };
 
-    let hash = RHash::new();
+    let hash = ruby.hash_new();
 
     // Convert C strings to Ruby strings, handling nulls safely
     // SAFETY: All non-null pointers from FFI must be valid C strings

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.0-rc.16'
+  VERSION = '4.0.0-rc.18'
 end

data/vendor/Cargo.toml

@@ -1,8 +1,8 @@
 [workspace]
-members = ["kreuzberg", "kreuzberg-tesseract"]
+members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
 
 [workspace.package]
-version = "4.0.0-rc.16"
+version = "4.0.0-rc.18"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.0.0-rc.16"
+version = "4.0.0-rc.18"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/vendor/kreuzberg/src/extraction/html.rs

@@ -163,6 +163,7 @@ fn convert_html_with_options_large_stack(html: String, options: ConversionOption
 }
 
 #[cfg(not(target_arch = "wasm32"))]
+#[allow(dead_code)]
 fn convert_inline_images_with_large_stack(
     html: String,
     options: ConversionOptions,
@@ -204,20 +205,6 @@ fn extract_panic_reason(panic: &Box<dyn Any + Send + 'static>) -> String {
 }
 
 // WASM implementations skip dedicated stack (not supported) and process inline
-#[cfg(target_arch = "wasm32")]
-fn convert_html_with_options_large_stack(html: String, options: ConversionOptions) -> Result<String> {
-    convert_html_with_options(&html, options)
-}
-
-#[cfg(target_arch = "wasm32")]
-fn convert_inline_images_with_large_stack(
-    html: String,
-    options: ConversionOptions,
-    image_config: LibInlineImageConfig,
-) -> Result<HtmlExtraction> {
-    convert_inline_images_with_options(&html, options, image_config)
-}
-
 /// Convert HTML to markdown with optional configuration.
 ///
 /// Uses sensible defaults if no configuration is provided:

data/vendor/kreuzberg-ffi/Cargo.toml

@@ -0,0 +1,74 @@
+[package]
+name = "kreuzberg-ffi"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+description = "C FFI bindings for Kreuzberg document intelligence library"
+license.workspace = true
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg-ffi"
+readme = "README.md"
+keywords = ["ffi", "bindings", "document", "extraction", "api"]
+categories = ["development-tools::ffi", "text-processing"]
+
+[lib]
+# cdylib: Required by Java (FFM API) and Go (cgo dynamic linking)
+# staticlib: Required by Python (PyO3 static linking) to avoid dylib install_name issues on macOS
+# rlib: Standard Rust library format for workspace dependencies
+crate-type = ["cdylib", "staticlib", "rlib"]
+
+[features]
+# Mirror embeddings feature availability from kreuzberg dependency
+embeddings = []
+# Optional rayon for parallel batch processing
+rayon = ["dep:rayon"]
+# Re-export kreuzberg features for downstream conditional compilation
+pdf = []
+keywords-yake = []
+keywords-rake = []
+
+[dependencies]
+serde_json = { workspace = true }
+serde = { workspace = true }
+async-trait = { workspace = true }
+tokio = { workspace = true }
+html-to-markdown-rs = { version = "2.16.1", default-features = false }
+rayon = { version = "1.11", optional = true }
+
+# On Windows MinGW, disable embeddings/ort since ONNX Runtime is not available
+# in MinGW-compatible form. Use all other features but exclude embeddings.
+[target.'cfg(all(windows, target_env = "gnu"))'.dependencies]
+kreuzberg = { path = "../kreuzberg", features = [
+    "pdf",
+    "excel",
+    "office",
+    "email",
+    "html",
+    "xml",
+    "archives",
+    "ocr",
+    "language-detection",
+    "chunking",
+    "quality",
+    "keywords",
+    "api",
+    "mcp",
+    "otel",
+    "bundled-pdfium",
+] }
+
+[target.'cfg(not(all(windows, target_env = "gnu")))'.dependencies]
+kreuzberg = { path = "../kreuzberg", features = ["full", "bundled-pdfium"] }
+
+[build-dependencies]
+cbindgen = "0.29"
+
+[dev-dependencies]
+tempfile = { workspace = true }
+criterion = { version = "0.8", features = ["html_reports"] }
+
+[[bench]]
+name = "result_view_benchmark"
+harness = false