kreuzberg 4.8.1 → 4.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 051fc6ca84b8545cb7049bd336a4897888b98f125fe40bfa24c6ab2e15232114
-  data.tar.gz: f64b385b1612258e73a6f771d107375ca1f892af7af199cbbaa8b19f184058df
+  metadata.gz: 4ec647109dee8229fd50ce9fe4d87f13bbb769779b22b4862cac321052610d6c
+  data.tar.gz: 18c3cf6df8e339f4286da7fe9b8da84185fa93abf89fcbf339b259c0095a9a5c
 SHA512:
-  metadata.gz: 949b5cdda46928b7981d4178efae64d64568155431785520774343fd27b47d2ebc7432d859fd404ff7f405cc93be7dbfda704e618b2aa079ff7d9522f7dac4c4
-  data.tar.gz: c6ea82f649544a0db19accb6b4eca03af5e9619321a973fe5cac9ad1524014bb766dc7edcec302bb2bef64c7b7ab0deb0ba1760fe80ccf2b5547c1bc11d4ad29
+  metadata.gz: f8c0ab16048bdb9026b55ff15f3ae342af9178a2ab6b1fd85777c3271da35e46474bee025af429745d20584604d520d85a4c8e4ea96bffc49ef5dfca55471b6f
+  data.tar.gz: 46ec0ff10138bd48d7b9ffada23956cd557616d8fdf09090f16562d4daab04381176b2881e08bfdfc1f726a0061e1e25feefa695620fef530444662abb866605

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.8.1" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.8.2" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-rb"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/ext/kreuzberg_rb/native/src/config/types.rs

@@ -1005,6 +1005,16 @@ pub fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<Extra
             config.html_options = Some(parse_html_options(ruby, html_hash)?);
         }
 
+        if let Some(val) = get_kw(ruby, hash, "html_output")
+            && val.equal(ruby.qnil()).ok() != Some(true)
+        {
+            let html_output_json = ruby_value_to_json(val)?;
+            let parsed: kreuzberg::core::config::html_output::HtmlOutputConfig =
+                serde_json::from_value(html_output_json)
+                    .map_err(|e| runtime_error(format!("Invalid html_output: {}", e)))?;
+            config.html_output = Some(parsed);
+        }
+
         if let Some(val) = get_kw(ruby, hash, "pages")
             && val.equal(ruby.qnil()).ok() != Some(true)
         {

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.8.1'
+  VERSION = '4.8.2'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -32,8 +32,8 @@ html-to-markdown-rs = { version = "3.1.0", default-features = false }
 image = { version = "0.25.10", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
-kreuzberg = { path = "./crates/kreuzberg", version = "4.8.1", default-features = false }
-kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.8.1" }
+kreuzberg = { path = "./crates/kreuzberg", version = "4.8.2", default-features = false }
+kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.8.2" }
 lazy_static = "1.5.0"
 libc = "0.2.184"
 liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg/README.md

@@ -18,7 +18,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.8.1 Release**
+> **🚀 Version 4.8.2 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/pdf/structure/pipeline.rs

@@ -1240,7 +1240,10 @@ pub fn extract_document_structure(
     // Deduplicate paragraphs with identical text within each page.
     // Catches bold/shadow rendering artifacts (consecutive duplicates)
     // and table content rendered as both table and body text.
-    deduplicate_paragraphs(&mut all_page_paragraphs);
+    // When strip_repeating_text is disabled, skip dedup to preserve all content.
+    if strip_repeating_text {
+        deduplicate_paragraphs(&mut all_page_paragraphs);
+    }
 
     let total_paragraphs: usize = all_page_paragraphs.iter().map(|p| p.len()).sum();
     tracing::debug!(
@@ -2291,4 +2294,41 @@ mod tests {
         assert!(paras[0].is_page_furniture);
         assert!(paras[1].is_page_furniture);
     }
+
+    #[test]
+    fn test_deduplicate_paragraphs_removes_consecutive_duplicates() {
+        let p1 = para(vec![line(vec![full_line_seg("Brand loses market share")])]);
+        let p2 = para(vec![line(vec![full_line_seg("Brand loses market share")])]);
+        let p3 = para(vec![line(vec![full_line_seg("Different content here")])]);
+        let mut pages = vec![vec![p1, p2, p3]];
+        deduplicate_paragraphs(&mut pages);
+        assert_eq!(pages[0].len(), 2, "consecutive duplicate should be removed");
+    }
+
+    #[test]
+    fn test_deduplicate_paragraphs_removes_non_consecutive_body_duplicates() {
+        let p1 = para(vec![line(vec![full_line_seg("Brand loses market share in volume")])]);
+        let p2 = para(vec![line(vec![full_line_seg("Some intervening paragraph")])]);
+        let p3 = para(vec![line(vec![full_line_seg("Brand loses market share in volume")])]);
+        let mut pages = vec![vec![p1, p2, p3]];
+        deduplicate_paragraphs(&mut pages);
+        assert_eq!(pages[0].len(), 2, "non-consecutive body duplicate should be removed");
+    }
+
+    #[test]
+    fn test_deduplicate_paragraphs_preserves_non_consecutive_headings() {
+        // Pass 2 (non-consecutive dedup) skips headings via is_dedup_candidate.
+        let mut h = para(vec![line(vec![full_line_seg("Brand loses market share in volume")])]);
+        h.heading_level = Some(2);
+        let filler = para(vec![line(vec![full_line_seg("Some other content between them")])]);
+        let mut h2 = para(vec![line(vec![full_line_seg("Brand loses market share in volume")])]);
+        h2.heading_level = Some(2);
+        let mut pages = vec![vec![h, filler, h2]];
+        deduplicate_paragraphs(&mut pages);
+        assert_eq!(
+            pages[0].len(),
+            3,
+            "non-consecutive heading duplicates must be preserved"
+        );
+    }
 }

data/vendor/kreuzberg/tests/api_consistency.rs

@@ -252,6 +252,7 @@ fn test_extraction_config_no_unknown_fields_in_default() {
         "keywords",
         "postprocessor",
         "html_options",
+        "html_output",
         "max_concurrent_extractions",
         "result_format",
         "output_format",

data/vendor/kreuzberg-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-ffi"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -41,7 +41,7 @@ serde_json = { version = "1.0.149" }
 tokio = { version = "1.51.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
 
 [target.'cfg(all(windows, target_env = "gnu"))'.dependencies]
-kreuzberg = { path = "../kreuzberg", version = "4.8.1", default-features = false, features = [
+kreuzberg = { path = "../kreuzberg", version = "4.8.2", default-features = false, features = [
     "pdf",
     "excel",
     "office",
@@ -64,7 +64,7 @@ kreuzberg = { path = "../kreuzberg", version = "4.8.1", default-features = false
 ] }
 
 [target.'cfg(not(all(windows, target_env = "gnu")))'.dependencies]
-kreuzberg = { path = "../kreuzberg", version = "4.8.1", default-features = false, features = ["bundled-pdfium", "full"] }
+kreuzberg = { path = "../kreuzberg", version = "4.8.2", default-features = false, features = ["bundled-pdfium", "full"] }
 
 [build-dependencies]
 cbindgen = "0.29"

data/vendor/kreuzberg-ffi/kreuzberg.h

@@ -9,8 +9,8 @@
 
 #define KREUZBERG_VERSION_MAJOR 4
 #define KREUZBERG_VERSION_MINOR 8
-#define KREUZBERG_VERSION_PATCH 1
-#define KREUZBERG_VERSION "4.8.1"
+#define KREUZBERG_VERSION_PATCH 2
+#define KREUZBERG_VERSION "4.8.2"
 
 
 #include <stdarg.h>
@@ -1269,6 +1269,30 @@ KREUZBERG_EXPORT
 int32_t kreuzberg_config_builder_set_content_filter(struct ConfigBuilder *builder,
                                                     const char *cf_json);
 
+/**
+ * Set HTML output configuration from JSON.
+ *
+ * # Arguments
+ *
+ * * `builder` - Non-null pointer to ConfigBuilder
+ * * `html_output_json` - JSON string for HTML output config
+ *
+ * # Returns
+ *
+ * 0 on success, -1 on error (check kreuzberg_last_error)
+ *
+ * # Safety
+ *
+ * This function is meant to be called from C/FFI code. The caller must ensure:
+ * - `builder` must be a valid, non-null pointer previously returned by `kreuzberg_config_builder_new`
+ * - The pointer must be properly aligned and point to a valid ConfigBuilder instance
+ * - `html_output_json` must be a valid, non-null pointer to a null-terminated UTF-8 string
+ * - The string pointer must remain valid for the duration of the function call
+ */
+KREUZBERG_EXPORT
+int32_t kreuzberg_config_builder_set_html_output(struct ConfigBuilder *builder,
+                                                 const char *html_output_json);
+
 /**
  * Build the final ExtractionConfig and consume the builder.
  *

data/vendor/kreuzberg-ffi/src/config_builder.rs

@@ -14,7 +14,7 @@ use kreuzberg::core::config::LayoutDetectionConfig;
 #[cfg(feature = "tree-sitter")]
 use kreuzberg::core::config::TreeSitterConfig;
 use kreuzberg::core::config::{
-    AccelerationConfig, ChunkingConfig, ContentFilterConfig, ExtractionConfig, ImageExtractionConfig,
+    AccelerationConfig, ChunkingConfig, ContentFilterConfig, ExtractionConfig, HtmlOutputConfig, ImageExtractionConfig,
     LanguageDetectionConfig, OcrConfig, PdfConfig, PostProcessorConfig,
 };
 use std::ffi::{CStr, c_char};
@@ -131,6 +131,13 @@ impl ConfigBuilder {
         Ok(())
     }
 
+    fn set_html_output_from_json(&mut self, json: &str) -> Result<(), String> {
+        let html_output_config: HtmlOutputConfig =
+            serde_json::from_str(json).map_err(|e| format!("Failed to parse HTML output config JSON: {}", e))?;
+        self.config.html_output = Some(html_output_config);
+        Ok(())
+    }
+
     fn build(self) -> ExtractionConfig {
         self.config
     }
@@ -854,6 +861,59 @@ pub unsafe extern "C" fn kreuzberg_config_builder_set_content_filter(
     })
 }
 
+/// Set HTML output configuration from JSON.
+///
+/// # Arguments
+///
+/// * `builder` - Non-null pointer to ConfigBuilder
+/// * `html_output_json` - JSON string for HTML output config
+///
+/// # Returns
+///
+/// 0 on success, -1 on error (check kreuzberg_last_error)
+///
+/// # Safety
+///
+/// This function is meant to be called from C/FFI code. The caller must ensure:
+/// - `builder` must be a valid, non-null pointer previously returned by `kreuzberg_config_builder_new`
+/// - The pointer must be properly aligned and point to a valid ConfigBuilder instance
+/// - `html_output_json` must be a valid, non-null pointer to a null-terminated UTF-8 string
+/// - The string pointer must remain valid for the duration of the function call
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn kreuzberg_config_builder_set_html_output(
+    builder: *mut ConfigBuilder,
+    html_output_json: *const c_char,
+) -> i32 {
+    ffi_panic_guard_i32!("kreuzberg_config_builder_set_html_output", {
+        if builder.is_null() {
+            set_last_error("ConfigBuilder pointer cannot be NULL".to_string());
+            return -1;
+        }
+        if html_output_json.is_null() {
+            set_last_error("HTML output JSON cannot be NULL".to_string());
+            return -1;
+        }
+
+        clear_last_error();
+
+        let json_str = match unsafe { CStr::from_ptr(html_output_json) }.to_str() {
+            Ok(s) => s,
+            Err(e) => {
+                set_last_error(format!("Invalid UTF-8 in HTML output JSON: {}", e));
+                return -1;
+            }
+        };
+
+        match unsafe { (*builder).set_html_output_from_json(json_str) } {
+            Ok(()) => 0,
+            Err(e) => {
+                set_last_error(e);
+                -1
+            }
+        }
+    })
+}
+
 /// Build the final ExtractionConfig and consume the builder.
 ///
 /// After calling this function, the builder pointer is invalid and must not be used.
@@ -1024,6 +1084,32 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_builder_with_html_output() {
+        unsafe {
+            let builder = kreuzberg_config_builder_new();
+            assert!(!builder.is_null());
+
+            let html_json = CString::new(
+                r#"{"theme":"github","class_prefix":"kb-","embed_css":true,"css":".kb-p { color: red; }"}"#,
+            )
+            .unwrap();
+            let result = kreuzberg_config_builder_set_html_output(builder, html_json.as_ptr());
+            assert_eq!(result, 0);
+
+            let config = kreuzberg_config_builder_build(builder);
+            assert!(!config.is_null());
+
+            assert!((*config).html_output.is_some());
+            let ho = (*config).html_output.as_ref().unwrap();
+            assert_eq!(ho.css.as_deref(), Some(".kb-p { color: red; }"));
+            assert!(ho.embed_css);
+
+            // Clean up
+            let _ = Box::from_raw(config);
+        }
+    }
+
     #[test]
     fn test_builder_invalid_json() {
         unsafe {

data/vendor/kreuzberg-paddle-ocr/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-paddle-ocr"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-pdfium-render/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-pdfium-render"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-tesseract/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-tesseract"
-version = "4.8.1"
+version = "4.8.2"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzberg
 version: !ruby/object:Gem::Version
-  version: 4.8.1
+  version: 4.8.2
 platform: ruby
 authors:
 - Na'aman Hirschfeld
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-09 00:00:00.000000000 Z
+date: 2026-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys