kreuzberg 4.9.0 → 4.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb7b77bae36a5da34ce209fbf1ea7c0a68aef4b22f8b373b908f9c113f404ef5
-  data.tar.gz: a6c8667aee6ae2c9e11d45fc98fcb355561fec6e4a7d51d852664bd6367af8cc
+  metadata.gz: 559b3104e6e21f2f14a92949d427703b51ea8b35b7a643d8964b6953785aa6e1
+  data.tar.gz: bfce92579c45ecba0da0d8e1f077ecca0bb9dd6a1e96c950e8beb8d0a39b5884
 SHA512:
-  metadata.gz: 7569a4914ab4a4d440a0c74e622a9f26f7189b62bc9c2d05fc5e857a32c8fabde8eb854edef34e94bd95d5357e44137c1573e7ce68db45ed85c26dbe31e6972b
-  data.tar.gz: 9741106549d7bf79cc1ae34a07f686cca1bf6a4c19fcb01b40cd8f1372166c8e9c3a0321e1e26e416ebb98d413ee1c9093d247949292a7c07a85594ea1df508e
+  metadata.gz: 1ea8af57d65eb5008126758041df2bfe07acca9d47ebfbf9c9de79f12b5d5ff2336d55b643268d1ea420db1825eaf9bfef6e5deb7335bd2449e9ccb62800492d
+  data.tar.gz: '019f2abaa7dcaf2b91925f7ed0b5332ce569a41a5ae1f152d698423cce1276ea68e177c2088b4377c369943e235c70584f61a76da5600d6dd8b3bd075bc266ab'

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.0" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.1" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -2916,7 +2916,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-rb"
-version = "4.8.6"
+version = "4.9.1"
 dependencies = [
  "async-trait",
  "html-to-markdown-rs",
@@ -5734,9 +5734,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
 
 [[package]]
 name = "typenum"
-version = "1.19.0"
+version = "1.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
 
 [[package]]
 name = "unicase"
@@ -6186,9 +6186,9 @@ dependencies = [
 
 [[package]]
 name = "web_atoms"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
+checksum = "d7cff6eef815df1834fd250e3a2ff436044d82a9f1bc1980ca1dbdf07effc538"
 dependencies = [
  "phf",
  "phf_codegen",

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-rb"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.9.0'
+  VERSION = '4.9.1'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -32,8 +32,8 @@ html-to-markdown-rs = { version = "3.2.5", default-features = false }
 image = { version = "0.25.10", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
-kreuzberg = { path = "./crates/kreuzberg", version = "4.9.0", default-features = false }
-kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.0" }
+kreuzberg = { path = "./crates/kreuzberg", version = "4.9.1", default-features = false }
+kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.1" }
 lazy_static = "1.5.0"
 libc = "0.2.185"
 liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false }
@@ -45,7 +45,7 @@ num_cpus = "1.17.0"
 once_cell = "1.21.4"
 ort = { version = "2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
 parking_lot = "0.12.5"
-pdf_oxide = { version = "0.3.33", default-features = false }
+pdf_oxide = { version = "0.3.34", default-features = false }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "crates/kreuzberg-pdfium-render", version = "4.3" }
 rayon = "1.12.0"
 reqwest = { version = "0.13.2", default-features = false }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -314,7 +314,7 @@ ort = { version = "2.0.0-rc.12", default-features = false, features = [
 outlook-pst = { version = "1.2.0", optional = true }
 parking_lot = "0.12.5"
 pastey = "0.2"
-pdf_oxide = { version = "0.3.33", default-features = false, optional = true }
+pdf_oxide = { version = "0.3.34", default-features = false, optional = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "../kreuzberg-pdfium-render", features = ["thread_safe", "image_latest"], optional = true }
 pulldown-cmark = { version = "0.13" }
 quick-xml = { version = "0.39.2", features = ["serialize"], optional = true }

data/vendor/kreuzberg/README.md

@@ -18,7 +18,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.9.0 Release**
+> **🚀 Version 4.9.1 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/extraction/pptx/mod.rs

@@ -284,7 +284,7 @@ fn extract_pptx_from_container<R: std::io::Read + std::io::Seek>(
 
     let document = doc_builder
         .map(|b| b.build())
-        .and_then(|d| if d.is_empty() { None } else { Some(d) });
+        .filter(|d| !d.is_empty());
 
     Ok(PptxExtractionResult {
         content,

data/vendor/kreuzberg/src/keywords/yake/preprocessor.rs

@@ -90,28 +90,26 @@ fn find_sentence_end(text: &str, from: usize) -> Option<usize> {
         }
 
         // Look for sentence terminals: . ! ?
-        match memchr3(b'.', b'!', b'?', &bytes[pos..]) {
-            None => return None,
-            Some(offset) => {
-                let terminal_pos = pos + offset;
-                // Consume consecutive terminals (e.g., "..." or "?!")
-                let mut end = terminal_pos + 1;
-                while end < bytes.len() && (bytes[end] == b'.' || bytes[end] == b'!' || bytes[end] == b'?') {
-                    end += 1;
-                }
-
-                // Consume closing quotes/brackets after terminal
-                while end < bytes.len() && matches!(bytes[end], b'"' | b'\'' | b')' | b']' | b'}') {
-                    end += 1;
-                }
+        {
+            let offset = memchr3(b'.', b'!', b'?', &bytes[pos..])?;
+            let terminal_pos = pos + offset;
+            // Consume consecutive terminals (e.g., "..." or "?!")
+            let mut end = terminal_pos + 1;
+            while end < bytes.len() && (bytes[end] == b'.' || bytes[end] == b'!' || bytes[end] == b'?') {
+                end += 1;
+            }
 
-                // Check if this is a real sentence boundary
-                if is_sentence_boundary(text, terminal_pos, end) {
-                    return Some(end);
-                }
+            // Consume closing quotes/brackets after terminal
+            while end < bytes.len() && matches!(bytes[end], b'"' | b'\'' | b')' | b']' | b'}') {
+                end += 1;
+            }
 
-                pos = end;
+            // Check if this is a real sentence boundary
+            if is_sentence_boundary(text, terminal_pos, end) {
+                return Some(end);
             }
+
+            pos = end;
         }
     }
 

data/vendor/kreuzberg/src/pdf/metadata.rs

@@ -320,12 +320,12 @@ pub fn extract_common_metadata_from_document(document: &PdfDocument<'_>) -> Resu
     let authors = metadata_cache[2]
         .as_ref()
         .map(|author_str| parse_authors(author_str))
-        .and_then(|parsed| if !parsed.is_empty() { Some(parsed) } else { None });
+        .filter(|parsed| !parsed.is_empty());
 
     let keywords = metadata_cache[3]
         .as_ref()
         .map(|keywords_str| parse_keywords(keywords_str))
-        .and_then(|parsed| if !parsed.is_empty() { Some(parsed) } else { None });
+        .filter(|parsed| !parsed.is_empty());
 
     let created_at = metadata_cache[4].as_ref().map(|date_str| parse_pdf_date(date_str));
 

data/vendor/kreuzberg/src/pdf/oxide/metadata.rs

@@ -94,11 +94,11 @@ fn extract_common_metadata(doc: &mut OxideDocument) -> Result<CommonPdfMetadata>
 
     let authors = get_info_string(&mut doc.doc, "Author")
         .map(|author_str| parse_authors(&author_str))
-        .and_then(|parsed| if parsed.is_empty() { None } else { Some(parsed) });
+        .filter(|parsed| !parsed.is_empty());
 
     let keywords = get_info_string(&mut doc.doc, "Keywords")
         .map(|kw_str| parse_keywords(&kw_str))
-        .and_then(|parsed| if parsed.is_empty() { None } else { Some(parsed) });
+        .filter(|parsed| !parsed.is_empty());
 
     let created_at = get_info_string(&mut doc.doc, "CreationDate").map(|d| parse_pdf_date(&d));
     let modified_at = get_info_string(&mut doc.doc, "ModDate").map(|d| parse_pdf_date(&d));

data/vendor/kreuzberg/src/pdf/oxide/table.rs

@@ -75,9 +75,7 @@ pub(crate) fn extract_tables_native(doc: &mut OxideDocument) -> Result<Vec<Table
 ///
 /// Maps rows/cells from the native table structure to a 2D `Vec<Vec<String>>`
 /// grid and builds a markdown representation with proper header separators.
-fn convert_extracted_table(
-    table: &pdf_oxide::structure::table_extractor::ExtractedTable,
-) -> (Vec<Vec<String>>, String) {
+fn convert_extracted_table(table: &pdf_oxide::structure::table_extractor::Table) -> (Vec<Vec<String>>, String) {
     let mut cells: Vec<Vec<String>> = Vec::with_capacity(table.rows.len());
     let mut markdown = String::new();
     let mut found_header = false;
@@ -124,7 +122,7 @@ mod tests {
 
     #[test]
     fn test_convert_extracted_table_basic() {
-        use pdf_oxide::structure::table_extractor::{ExtractedTable, TableCell, TableRow};
+        use pdf_oxide::structure::table_extractor::{Table as ExtractedTable, TableCell, TableRow};
 
         let table = ExtractedTable {
             rows: vec![
@@ -191,7 +189,7 @@ mod tests {
 
     #[test]
     fn test_convert_extracted_table_no_header() {
-        use pdf_oxide::structure::table_extractor::{ExtractedTable, TableCell, TableRow};
+        use pdf_oxide::structure::table_extractor::{Table as ExtractedTable, TableCell, TableRow};
 
         let table = ExtractedTable {
             rows: vec![
@@ -233,7 +231,7 @@ mod tests {
 
     #[test]
     fn test_convert_extracted_table_empty() {
-        use pdf_oxide::structure::table_extractor::ExtractedTable;
+        use pdf_oxide::structure::table_extractor::Table as ExtractedTable;
 
         let table = ExtractedTable {
             rows: vec![],

data/vendor/kreuzberg/src/pdf/structure/pipeline.rs

@@ -1465,9 +1465,11 @@ pub(crate) fn extract_document_structure_from_segments(
         .collect();
 
     // Stage 3: Per-page structured extraction.
-    // When the structure tree provides heading roles, skip layout-model heading
-    // overrides — they can demote correctly-tagged headings. The tree is authoritative.
-    let effective_layout_hints = if used_structure_tree { None } else { layout_hints };
+    // Always pass layout hints regardless of structure tree status. Layout hints
+    // provide multi-purpose classification (furniture/header/footer marking, table
+    // regions, list items) beyond just heading overrides. The structure tree's
+    // heading roles are still respected via assigned_role on segments.
+    let effective_layout_hints = layout_hints;
     let page_inputs: Vec<PageInput> = (0..page_count)
         .map(|i| PageInput {
             page_index: i,

data/vendor/kreuzberg-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-ffi"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -28,7 +28,7 @@ ahash = { version = "0.8.12", features = ["serde"] }
 async-trait = "0.1.89"
 ctor = "0.10"
 html-to-markdown-rs = { version = "3.2.5", default-features = false }
-kreuzberg = { path = "../kreuzberg", version = "4.9.0", default-features = false, features = ["bundled-pdfium", "full"] }
+kreuzberg = { path = "../kreuzberg", version = "4.9.1", default-features = false, features = ["bundled-pdfium", "full"] }
 log = "0.4"
 rayon = { version = "1.12.0", optional = true }
 serde = { version = "1.0.228", features = ["derive"] }

data/vendor/kreuzberg-ffi/kreuzberg.h

@@ -9,8 +9,8 @@
 
 #define KREUZBERG_VERSION_MAJOR 4
 #define KREUZBERG_VERSION_MINOR 9
-#define KREUZBERG_VERSION_PATCH 0
-#define KREUZBERG_VERSION "4.9.0"
+#define KREUZBERG_VERSION_PATCH 1
+#define KREUZBERG_VERSION "4.9.1"
 
 
 #include <stdarg.h>

data/vendor/kreuzberg-paddle-ocr/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-paddle-ocr"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-pdfium-render/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-pdfium-render"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-tesseract/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-tesseract"
-version = "4.9.0"
+version = "4.9.1"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-tesseract/build.rs

@@ -613,8 +613,11 @@ mod build_tesseract {
         let eng_traineddata = bundled_tessdata_dir.join("eng.traineddata");
         if !eng_traineddata.exists() {
             fs::create_dir_all(&bundled_tessdata_dir).expect("Failed to create tessdata directory");
-            download_file(
-                "https://github.com/tesseract-ocr/tessdata_fast/raw/main/eng.traineddata",
+            download_file_with_fallback(
+                &[
+                    "https://github.com/tesseract-ocr/tessdata_fast/raw/main/eng.traineddata",
+                    "https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/main/eng.traineddata",
+                ],
                 &eng_traineddata,
                 "eng.traineddata",
             );
@@ -911,49 +914,66 @@ mod build_tesseract {
     }
 
     /// Download a single file to a destination path with retries.
-    fn download_file(url: &str, dest: &Path, label: &str) {
+    /// Download a single file, trying each URL in order. Each URL gets up to
+    /// `max_attempts` retries with exponential backoff before falling through
+    /// to the next URL.
+    fn download_file_with_fallback(urls: &[&str], dest: &Path, label: &str) {
         let client = reqwest::blocking::Client::builder()
-            .timeout(std::time::Duration::from_secs(120))
+            .timeout(std::time::Duration::from_secs(300))
             .http1_only()
             .build()
             .expect("Failed to create HTTP client");
 
-        eprintln!("Downloading {} from {}", label, url);
-        let max_attempts = 3;
-
-        for attempt in 1..=max_attempts {
-            let err_msg = match client.get(url).send() {
-                Ok(resp) => {
-                    if resp.status().is_success() {
-                        match resp.bytes() {
-                            Ok(bytes) => {
-                                fs::write(dest, &bytes).expect("Failed to write downloaded file");
-                                eprintln!("Downloaded {} ({} bytes)", label, bytes.len());
-                                return;
+        let max_attempts: u32 = 5;
+        let mut last_err = String::new();
+
+        for url in urls {
+            eprintln!("Downloading {} from {}", label, url);
+
+            for attempt in 1..=max_attempts {
+                let err_msg = match client.get(*url).send() {
+                    Ok(resp) => {
+                        if resp.status().is_success() {
+                            match resp.bytes() {
+                                Ok(bytes) => {
+                                    fs::write(dest, &bytes).expect("Failed to write downloaded file");
+                                    eprintln!("Downloaded {} ({} bytes)", label, bytes.len());
+                                    return;
+                                }
+                                Err(err) => format!("Failed to read response: {}", err),
                             }
-                            Err(err) => format!("Failed to read response: {}", err),
+                        } else {
+                            format!("HTTP {}", resp.status().as_u16())
                         }
-                    } else {
-                        format!("HTTP {}", resp.status().as_u16())
                     }
+                    Err(err) => err.to_string(),
+                };
+
+                last_err = err_msg.clone();
+
+                if attempt == max_attempts {
+                    println!(
+                        "cargo:warning=All {} attempts for {} exhausted on URL {}",
+                        max_attempts, label, url
+                    );
+                    break;
                 }
-                Err(err) => err.to_string(),
-            };
 
-            if attempt == max_attempts {
-                panic!(
-                    "Failed to download {} after {} attempts: {}",
-                    label, max_attempts, err_msg
+                let backoff = 2u64.pow((attempt - 1).min(4));
+                println!(
+                    "cargo:warning=Download attempt {}/{} for {} failed ({}). Retrying in {}s...",
+                    attempt, max_attempts, label, err_msg, backoff
                 );
+                std::thread::sleep(std::time::Duration::from_secs(backoff));
             }
-
-            let backoff = 2u64.pow((attempt - 1).min(3));
-            println!(
-                "cargo:warning=Download attempt {}/{} for {} failed ({}). Retrying in {}s...",
-                attempt, max_attempts, label, err_msg, backoff
-            );
-            std::thread::sleep(std::time::Duration::from_secs(backoff));
         }
+
+        panic!(
+            "Failed to download {} after trying {} URL(s): {}",
+            label,
+            urls.len(),
+            last_err
+        );
     }
 
     fn normalize_cmake_path(path: &Path) -> String {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzberg
 version: !ruby/object:Gem::Version
-  version: 4.9.0
+  version: 4.9.1
 platform: ruby
 authors:
 - Na'aman Hirschfeld
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-18 00:00:00.000000000 Z
+date: 2026-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys