kreuzberg 4.9.7 → 4.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24151d9931038d170843ec2b42b0f34b65d78a6f07418683ba71a96c7b2f4f50
-  data.tar.gz: 701aedb58613376393f8e168dbd2ecea211063172cbfdaae7af96590d990e8c1
+  metadata.gz: de92334e109bbca1bdd22469a651f146bf29eee730dfc841c3dcb4703ee3ba5b
+  data.tar.gz: 53140e24511ff0910814325859b3e7382ee9f511e2412181812607f0f3516f33
 SHA512:
-  metadata.gz: b8917f84b8d8f7861c2f71a41248244344da6444942267e65b7feb04123017f099d618b6d72632d710936983f8ee9771e6847447e2119363d6e123caf8cc4f60
-  data.tar.gz: 59fb7bd5a2c079fde4221aa1d578542cdf1b5f3637b28787d735d7a87b6fc86dd0f239e0623d4284c39d1175bed1072f4291a644c435d7242db9e5a3abeb5575
+  metadata.gz: 80b7a6fa716b1adf28d543074581d5f88984aae738669e52116c49d301b1116ed7311c4bcfa90d1853a9d98752b0d2ad13b5c0e14e9f2623217ff319c007eca2
+  data.tar.gz: fab95d4048b382cf3ff4e154d7979bc4963e35e57c802cd5f871950d460ef16c5745c66931fbbf15b02a479f0a991a9e1832a126d445c002c7bf79b92ae143b9

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.7" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.9" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-rb"
-version = "4.9.7"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -49,13 +49,13 @@ kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi" }
 magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
     "rb-sys",
 ] }
-rb-sys = { version = "0.9.126", default-features = false, features = [
+rb-sys = { version = "0.9.128", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
-serde_json = "1.0.149"
+serde_json = "1.0.150"
 toml = "1.1.2"
 serde_yaml_ng = "0.10"
-tokio = { version = "1.52.1", features = [
+tokio = { version = "1.52.3", features = [
     "rt",
     "rt-multi-thread",
     "macros",
@@ -65,7 +65,7 @@ tokio = { version = "1.52.1", features = [
     "time",
     "io-util",
 ] }
-html-to-markdown-rs = { version = "3.3.1", default-features = false }
+html-to-markdown-rs = { version = "3.5.7", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.9.7'
+  VERSION = '4.9.9'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.9.7"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -23,41 +23,40 @@ clap = { version = "4.6", features = ["derive", "color", "suggestions"] }
 comrak = { version = "0.52", default-features = false }
 console_error_panic_hook = "0.1"
 criterion = { version = "0.8", features = ["html_reports"] }
-ctor = "0.10"
+ctor = "1.0"
 dbase = "0.7"
 futures = "0.3"
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.3.1", default-features = false }
+html-to-markdown-rs = { version = "3.5.7", default-features = false }
 image = { version = "0.25.10", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
-kreuzberg = { path = "./crates/kreuzberg", version = "4.9.7", default-features = false }
-kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.7" }
+kreuzberg = { path = "./crates/kreuzberg", version = "4.9.9", default-features = false }
+kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.9" }
 lazy_static = "1.5.0"
-libc = "0.2.185"
-liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false }
+libc = "0.2.186"
+liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false }
 log = "0.4"
-lzma-rust2 = { version = "0.16.2" }
+lzma-rust2 = { version = "0.16.4" }
 memmap2 = "0.9"
 minijinja = "2"
 num_cpus = "1.17.0"
 once_cell = "1.21.4"
 ort = { version = "2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
 parking_lot = "0.12.5"
-pdf_oxide = { version = "0.3.37", default-features = false }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "crates/kreuzberg-pdfium-render", version = "4.3" }
 rayon = "1.12.0"
-reqwest = { version = "0.13.2", default-features = false }
+reqwest = { version = "0.13.4", default-features = false }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.149" }
+serde_json = { version = "1.0.150" }
 serde_toon_format = "0.1"
 tempfile = "3.27.0"
 thiserror = "2.0.18"
-tokio = { version = "1.52.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
+tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
 toml = "1.1.2"
 tracing = "0.1"
-tree-sitter-language-pack = { version = "1.7.0", features = ["serde"], default-features = false }
+tree-sitter-language-pack = { version = "1.8.1", features = ["serde"], default-features = false }
 wasm-bindgen = { version = "0.2", features = ["enable-interning"] }
 wasm-bindgen-futures = "0.4"
 web-sys = { version = "0.3", features = ["Blob", "File", "FileReader", "console", "TextDecoder", "ImageData", "Window", "Response"] }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.9.7"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -39,10 +39,6 @@ simd-utf8 = ["dep:simdutf8"]
 tokio-runtime = ["dep:tokio"]
 
 pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image", "dep:flate2", "html"]
-# Experimental: use pdf_oxide for text extraction (pure Rust, no C++ deps).
-# Provides cleaner word spacing for PDFs with broken font CMaps.
-# Requires 'pdf' feature. Not included in 'full' — opt-in only.
-pdf-oxide = ["pdf", "dep:pdf_oxide"]
 static-pdfium = ["pdf"]
 bundled-pdfium = ["pdf"]
 system-pdfium = ["pdf"]
@@ -61,7 +57,14 @@ office = [
 ]
 hwp = ["dep:cfb", "dep:flate2"]
 iwork = ["dep:zip", "dep:snap"]
-email = ["dep:mail-parser", "dep:cfb", "dep:outlook-pst", "dep:tempfile", "dep:chrono"]
+email = [
+    "dep:mail-parser",
+    "dep:cfb",
+    "dep:outlook-pst",
+    "dep:tempfile",
+    "dep:chrono",
+    "dep:chardetng",
+]
 html = ["dep:html-to-markdown-rs", "dep:v_htmlescape"]
 xml = ["dep:quick-xml", "dep:roxmltree"]
 archives = ["dep:zip", "dep:tar", "dep:sevenz-rust2", "dep:flate2"]
@@ -254,12 +257,12 @@ biblib = { version = "0.4", default-features = false, features = [
 bitvec = "1.0"
 blake3 = "1"
 bytes = { version = "1", features = ["serde"] }
-calamine = { version = "0.34.0", features = ["dates"], optional = true }
+calamine = { version = "0.35.0", features = ["dates"], optional = true }
 cfb = { version = "0.14", optional = true }
 chardetng = { version = "1.0.0", optional = true }
 chrono = { version = "0.4", optional = true }
 comrak = { version = "0.52", default-features = false }
-dashmap = "6.1"
+dashmap = "6.2"
 dbase = { version = "0.7", optional = true }
 dirs = "6"
 encoding_rs = { version = "0.8.35" }
@@ -271,7 +274,7 @@ hayro-jpeg2000 = { version = "0.3", default-features = false, features = [
     "simd",
 ], optional = true }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.3.1", default-features = false, features = [
+html-to-markdown-rs = { version = "3.5.7", default-features = false, features = [
     "inline-images",
     "metadata",
 ], optional = true }
@@ -291,20 +294,20 @@ jotdown = "0.10"
 kamadak-exif = { version = "0.6.1", optional = true }
 
 kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
-libc = "0.2.185"
-liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false, optional = true }
+libc = "0.2.186"
+liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false, optional = true }
 log = "0.4"
-lopdf = { version = "0.40.0", optional = true }
-mail-parser = { version = "0.11.2", optional = true }
-memchr = "2.8.0"
+lopdf = { version = "0.41.0", optional = true }
+mail-parser = { version = "0.11.3", optional = true }
+memchr = "2.8.1"
 memmap2 = "0.9"
 mime_guess = "2.0"
 minijinja = { version = "2", optional = true }
 ndarray = { version = "0.17", optional = true }
 num_cpus = "1.17.0"
 once_cell = "1.21.4"
-opentelemetry = { version = "0.31", features = ["trace"], optional = true }
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
+opentelemetry = { version = "0.32", features = ["trace"], optional = true }
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"], optional = true }
 org = { version = "0.3", optional = true }
 ort = { version = "2.0.0-rc.12", default-features = false, features = [
     "std",
@@ -314,14 +317,13 @@ ort = { version = "2.0.0-rc.12", default-features = false, features = [
 outlook-pst = { version = "1.2.0", optional = true }
 parking_lot = "0.12.5"
 pastey = "0.2"
-pdf_oxide = { version = "0.3.37", default-features = false, optional = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "../kreuzberg-pdfium-render", features = ["thread_safe", "image_latest"], optional = true }
 pulldown-cmark = { version = "0.13" }
-quick-xml = { version = "0.39.2", features = ["serialize"], optional = true }
+quick-xml = { version = "0.40.1", features = ["serialize"], optional = true }
 rake = { version = "0.3.6", optional = true }
 rayon = "1.12.0"
 regex = "1.12.3"
-rmcp = { version = "1.5.0", features = [
+rmcp = { version = "1.7.0", features = [
     "server",
     "macros",
     "base64",
@@ -333,32 +335,36 @@ rmp-serde = "1.3"
 
 roxmltree = { version = "0.21.1", optional = true }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.149" }
+serde_json = { version = "1.0.150" }
 serde_toon_format = "0.1"
 serde_yaml_ng = "0.10.0"
 sevenz-rust2 = { version = "0.20.2", optional = true }
 sha2 = { version = "0.11", optional = true }
 simdutf8 = { version = "0.1", optional = true }
 snap = { version = "1.1", optional = true }
-tar = { version = "0.4.45", optional = true }
+tar = { version = "0.4.46", optional = true }
 tempfile = { version = "3.27.0", optional = true }
-text-splitter = { version = "0.30.1", features = ["markdown"], optional = true }
+text-splitter = { version = "0.31.0", features = ["markdown"], optional = true }
 thiserror = "2.0.18"
 tiff = { version = "0.11", optional = true }
-tokenizers = { version = "0.22", optional = true, default-features = false, features = [
+# Keep aligned with text-splitter's optional tokenizers integration so ChunkSizer
+# is implemented for the same Tokenizer type used by Kreuzberg.
+tokenizers = { version = "0.23.1", optional = true, default-features = false, features = [
     "http",
     "fancy-regex",
 ] }
-tokio = { version = "1.52.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"], optional = true }
+tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"], optional = true }
 toml = "1.1.2"
 tower = { version = "0.5", features = ["timeout", "limit", "util"], optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit", "catch-panic", "request-id", "sensitive-headers", "compression-full"], optional = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.32", optional = true }
+tracing-opentelemetry = { version = "0.33", optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 urlencoding = "2"
-utoipa = { version = "5.4", features = ["axum_extras"], optional = true }
-v_htmlescape = { version = "0.15", optional = true }
+utoipa = { version = "5.5", features = ["axum_extras"], optional = true }
+# Pinned to 0.15 — v_htmlescape 0.17 renamed `escape` fn to an `Escape` struct.
+# Update call sites in src/rendering/html_styled.rs before bumping.
+v_htmlescape = { version = "=0.15.8", optional = true }
 whatlang = { version = "0.18.0", optional = true }
 zip = { version = ">=7.0.0", optional = true, default-features = false, features = [
     "deflate-flate2",
@@ -392,7 +398,7 @@ optional = true
 # Override getrandom to enable js feature for WASM targets
 # This is needed because ring/rustls (via ureq) depend on getrandom without js feature
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
-tree-sitter-language-pack = { version = "1.7.0", features = ["serde"], default-features = false, optional = true }
+tree-sitter-language-pack = { version = "1.8.1", features = ["serde"], default-features = false, optional = true }
 wasm-bindgen-rayon = { version = "1.3", optional = true }
 
 [build-dependencies]
@@ -406,10 +412,10 @@ dotenvy = "0.15"
 filetime = "0.2"
 image = { version = "0.25.10", default-features = false, features = ["png"] }
 jsonschema = "0.46"
-serial_test = "3.4.0"
-tar = "0.4.45"
+serial_test = "3.5.0"
+tar = "0.4.46"
 tempfile = "3.27.0"
-tokio = { version = "1.52.1", features = ["macros", "time"] }
+tokio = { version = "1.52.3", features = ["macros", "time"] }
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zip = { version = ">=7.0.0, <8.6.0", default-features = false, features = ["deflate-flate2"] }

data/vendor/kreuzberg/README.md

@@ -18,7 +18,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.9.7 Release**
+> **🚀 Version 4.9.9 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/core/config/pdf.rs

@@ -8,17 +8,14 @@ use serde::{Deserialize, Serialize};
 /// PDF extraction backend selection.
 ///
 /// Controls which PDF library is used for text extraction:
-/// - `Pdfium`: pdfium-render (default, C++ based, mature)
-/// - `PdfOxide`: pdf_oxide (pure Rust, faster, requires `pdf-oxide` feature)
-/// - `Auto`: automatically select based on available features
+/// - `Pdfium`: pdfium-render (default, mature)
+/// - `Auto`: automatically select the default available backend
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum PdfBackend {
     /// Use pdfium-render backend (default).
     #[default]
     Pdfium,
-    /// Use pdf_oxide backend (pure Rust). Requires `pdf-oxide` feature.
-    PdfOxide,
     /// Automatically select the best available backend.
     Auto,
 }

data/vendor/kreuzberg/src/core/config/tree_sitter.rs

@@ -155,7 +155,6 @@ impl From<&TreeSitterProcessConfig> for tree_sitter_language_pack::ProcessConfig
             symbols: p.symbols,
             diagnostics: p.diagnostics,
             chunk_max_size: p.chunk_max_size,
-            extractions: None,
         }
     }
 }

data/vendor/kreuzberg/src/core/extractor/bytes.rs

@@ -128,7 +128,12 @@ pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionC
 
     #[cfg(not(feature = "tokio-runtime"))]
     let result = {
-        let _ = config.extraction_timeout_secs;
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
         extraction_future.await
     };
 

data/vendor/kreuzberg/src/core/extractor/file.rs

@@ -142,7 +142,12 @@ pub async fn extract_file(
 
     #[cfg(not(feature = "tokio-runtime"))]
     let result = {
-        let _ = config.extraction_timeout_secs;
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
         extraction_future.await
     };
 

data/vendor/kreuzberg/src/core/extractor/legacy.rs

@@ -35,6 +35,13 @@ pub(super) fn extract_bytes_sync_impl(
     let cfg = config.cloned().unwrap_or_default();
     let cfg = cfg.normalized().into_owned();
 
+    if cfg.extraction_timeout_secs.is_some() {
+        return Err(crate::KreuzbergError::Validation {
+            message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+            source: None,
+        });
+    }
+
     let validated_mime = if let Some(mime) = mime_type {
         if mime == "application/octet-stream" {
             mime::detect_mime_type_from_bytes(content)?

data/vendor/kreuzberg/src/core/pipeline/features.rs

@@ -37,10 +37,19 @@ fn recompute_boundaries_from_pages(content: &str, pages: &[crate::types::PageCon
             continue;
         }
 
-        // Try exact match first
-        if let Some(pos) = content[search_offset..].find(&page.content) {
+        let normalized: String = page
+            .content
+            .split("\n\n")
+            .map(str::trim)
+            .filter(|s| !s.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n\n");
+
+        // Try normalized exact match first. PDF page text can contain trailing
+        // spaces that render_plain strips before chunking.
+        if let Some(pos) = content[search_offset..].find(normalized.as_str()) {
             let byte_start = search_offset + pos;
-            let byte_end = content.floor_char_boundary(byte_start + page.content.len());
+            let byte_end = content.floor_char_boundary(byte_start + normalized.len());
             boundaries.push(PageBoundary {
                 page_number: page.page_number,
                 byte_start,
@@ -50,12 +59,12 @@ fn recompute_boundaries_from_pages(content: &str, pages: &[crate::types::PageCon
             continue;
         }
 
-        // Fallback: search for first non-empty line of page content
+        // Fallback: search for first non-empty line of page content.
         if let Some(line) = page.content.lines().find(|l| !l.trim().is_empty()).map(|l| l.trim())
             && let Some(pos) = content[search_offset..].find(line)
         {
             let byte_start = search_offset + pos;
-            let raw_end = (byte_start + page.content.len()).min(content.len());
+            let raw_end = (byte_start + normalized.len()).min(content.len());
             let byte_end = content.floor_char_boundary(raw_end);
             boundaries.push(PageBoundary {
                 page_number: page.page_number,
@@ -176,25 +185,27 @@ pub(super) fn execute_chunking(result: &mut ExtractionResult, config: &Extractio
         let resolved_config = chunking_config.resolve_preset();
         let chunking_config = &resolved_config;
 
-        // Recompute page boundaries against `result.content` (rendered by `render_plain`)
-        // if per-page content is available.  The boundaries stored in
-        // `result.metadata.pages.boundaries` were computed against the raw extractor text
-        // and may have different byte offsets than the rendered content (fix for #636).
+        let (chunk_input, heading_source) = if config.output_format != crate::core::config::OutputFormat::Plain {
+            (
+                result.formatted_content.as_deref().unwrap_or(result.content.as_str()),
+                None,
+            )
+        } else {
+            (result.content.as_str(), result.formatted_content.as_deref())
+        };
+
         let recomputed_boundaries: Option<Vec<PageBoundary>> = result
             .pages
             .as_deref()
-            .map(|pages| recompute_boundaries_from_pages(&result.content, pages));
+            .map(|pages| recompute_boundaries_from_pages(chunk_input, pages))
+            .filter(|boundaries| !boundaries.is_empty());
 
         let page_boundaries: Option<&[PageBoundary]> = recomputed_boundaries
             .as_deref()
             .or_else(|| result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref()));
 
-        // Pass formatted_content (markdown) for heading context resolution when available.
-        // Plain-text rendering strips heading markers, but the markdown chunker needs them
-        // to build the heading hierarchy for chunk metadata.
-        let heading_source = result.formatted_content.as_deref();
         match crate::chunking::chunk_text_with_heading_source(
-            &result.content,
+            chunk_input,
             chunking_config,
             page_boundaries,
             heading_source,
@@ -314,3 +325,92 @@ pub(super) fn execute_token_reduction(result: &mut ExtractionResult, config: &Ex
 
     Ok(())
 }
+
+#[cfg(test)]
+#[cfg(feature = "chunking")]
+mod tests {
+    use super::*;
+    use crate::core::config::{ChunkerType, ChunkingConfig, OutputFormat};
+    use crate::types::PageContent;
+
+    fn make_page(page_number: usize, content: &str) -> PageContent {
+        PageContent {
+            page_number,
+            content: content.to_string(),
+            tables: Vec::new(),
+            images: Vec::new(),
+            hierarchy: None,
+            is_blank: None,
+            layout_regions: None,
+        }
+    }
+
+    fn markdown_chunking_config() -> ExtractionConfig {
+        ExtractionConfig {
+            output_format: OutputFormat::Markdown,
+            chunking: Some(ChunkingConfig {
+                max_characters: 2000,
+                overlap: 0,
+                trim: true,
+                chunker_type: ChunkerType::Markdown,
+                ..Default::default()
+            }),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn chunks_content_is_markdown_when_output_format_is_markdown() {
+        let mut result = ExtractionResult {
+            content: "SH-001 Luca Bianchi Common Germany 3500000".to_string(),
+            formatted_content: Some("| SH-001 | Luca Bianchi | Common | Germany | 3,500,000 |".to_string()),
+            mime_type: Cow::Borrowed("application/pdf"),
+            ..Default::default()
+        };
+
+        execute_chunking(&mut result, &markdown_chunking_config()).unwrap();
+
+        let chunks = result.chunks.expect("chunks must be populated");
+        assert!(!chunks.is_empty());
+        assert!(chunks.iter().any(|chunk| chunk.content.contains('|')));
+        assert!(chunks.iter().all(|chunk| !chunk.content.starts_with("SH-001 Luca")));
+        assert!(result.formatted_content.is_some());
+    }
+
+    #[test]
+    fn markdown_chunks_preserve_page_metadata_when_formatted_pages_match() {
+        let mut result = ExtractionResult {
+            content: "Page one text\n\nPage two text".to_string(),
+            formatted_content: Some("# Page one\n\nPage one text\n\n# Page two\n\nPage two text".to_string()),
+            pages: Some(vec![make_page(1, "Page one text"), make_page(2, "Page two text")]),
+            mime_type: Cow::Borrowed("application/pdf"),
+            ..Default::default()
+        };
+
+        execute_chunking(&mut result, &markdown_chunking_config()).unwrap();
+
+        let chunks = result.chunks.expect("chunks must be populated");
+        assert!(!chunks.is_empty());
+        assert!(chunks.iter().any(|chunk| chunk.metadata.first_page.is_some()));
+        assert!(chunks.iter().any(|chunk| chunk.metadata.last_page.is_some()));
+    }
+
+    #[test]
+    fn recompute_boundaries_trailing_space_pages_all_resolve() {
+        let p1_raw = "Heading \n\nBody paragraph one. ";
+        let p2_raw = "Second heading \n\nBody paragraph two. ";
+        let p3_raw = "Conclusion. ";
+        let p1_norm = "Heading\n\nBody paragraph one.";
+        let p2_norm = "Second heading\n\nBody paragraph two.";
+        let p3_norm = "Conclusion.";
+        let content = format!("{p1_norm}\n\n{p2_norm}\n\n{p3_norm}");
+
+        let pages = vec![make_page(1, p1_raw), make_page(2, p2_raw), make_page(3, p3_raw)];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        assert_eq!(boundaries.len(), 3);
+        assert_eq!(&content[boundaries[0].byte_start..boundaries[0].byte_end], p1_norm);
+        assert_eq!(&content[boundaries[1].byte_start..boundaries[1].byte_end], p2_norm);
+        assert_eq!(&content[boundaries[2].byte_start..boundaries[2].byte_end], p3_norm);
+    }
+}

data/vendor/kreuzberg/src/embeddings/mod.rs

@@ -270,11 +270,13 @@ fn load_tokenizer(
     {
         for (_, value) in &map {
             if let Some(content) = value.as_str() {
-                tokenizer.add_special_tokens(&[AddedToken {
-                    content: content.to_string(),
-                    special: true,
-                    ..Default::default()
-                }]);
+                tokenizer
+                    .add_special_tokens([AddedToken {
+                        content: content.to_string(),
+                        special: true,
+                        ..Default::default()
+                    }])
+                    .map_err(|e| crate::KreuzbergError::embedding(format!("Failed to add special token: {e}")))?;
             } else if value.is_object()
                 && let (Some(content), Some(single_word), Some(lstrip), Some(rstrip), Some(normalized)) = (
                     value["content"].as_str(),
@@ -284,14 +286,16 @@ fn load_tokenizer(
                     value["normalized"].as_bool(),
                 )
             {
-                tokenizer.add_special_tokens(&[AddedToken {
-                    content: content.to_string(),
-                    special: true,
-                    single_word,
-                    lstrip,
-                    rstrip,
-                    normalized,
-                }]);
+                tokenizer
+                    .add_special_tokens([AddedToken {
+                        content: content.to_string(),
+                        special: true,
+                        single_word,
+                        lstrip,
+                        rstrip,
+                        normalized,
+                    }])
+                    .map_err(|e| crate::KreuzbergError::embedding(format!("Failed to add special token: {e}")))?;
             }
         }
     }