kreuzberg 4.9.8 → 4.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b634075425816167cdb132080aa31a6c5c561badbd154de17896866bdf88ba6e
-  data.tar.gz: 4a8d041bad2da842a676b2e2358ba44d3269323e820728bdb568ebc9171adee1
+  metadata.gz: de92334e109bbca1bdd22469a651f146bf29eee730dfc841c3dcb4703ee3ba5b
+  data.tar.gz: 53140e24511ff0910814325859b3e7382ee9f511e2412181812607f0f3516f33
 SHA512:
-  metadata.gz: 60c018a882054c23b629ee1d0692493c25c78a52e106fc7f25e3c22d7cdcfd36341f5aaeebea3d6be48d1425550914124daaeaeeb02bbd84bbb64cd6afedad59
-  data.tar.gz: 62d942f23dae32120184cd5b011275a2556b3e112a866128bd290a67ddbbd94c523ec2eaecd24589911264609ae612fd22eb51f3efc2e9eec4b1d08f8bd6823c
+  metadata.gz: 80b7a6fa716b1adf28d543074581d5f88984aae738669e52116c49d301b1116ed7311c4bcfa90d1853a9d98752b0d2ad13b5c0e14e9f2623217ff319c007eca2
+  data.tar.gz: fab95d4048b382cf3ff4e154d7979bc4963e35e57c802cd5f871950d460ef16c5745c66931fbbf15b02a479f0a991a9e1832a126d445c002c7bf79b92ae143b9

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.8" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.9" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-rb"
-version = "4.9.8"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -52,7 +52,7 @@ magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb51742
 rb-sys = { version = "0.9.128", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
-serde_json = "1.0.149"
+serde_json = "1.0.150"
 toml = "1.1.2"
 serde_yaml_ng = "0.10"
 tokio = { version = "1.52.3", features = [
@@ -65,7 +65,7 @@ tokio = { version = "1.52.3", features = [
     "time",
     "io-util",
 ] }
-html-to-markdown-rs = { version = "3.4.1", default-features = false }
+html-to-markdown-rs = { version = "3.5.7", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.9.8'
+  VERSION = '4.9.9'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.9.8"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -28,29 +28,28 @@ dbase = "0.7"
 futures = "0.3"
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.4.1", default-features = false }
+html-to-markdown-rs = { version = "3.5.7", default-features = false }
 image = { version = "0.25.10", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
-kreuzberg = { path = "./crates/kreuzberg", version = "4.9.8", default-features = false }
-kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.8" }
+kreuzberg = { path = "./crates/kreuzberg", version = "4.9.9", default-features = false }
+kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.9" }
 lazy_static = "1.5.0"
 libc = "0.2.186"
 liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false }
 log = "0.4"
-lzma-rust2 = { version = "0.16.2" }
+lzma-rust2 = { version = "0.16.4" }
 memmap2 = "0.9"
 minijinja = "2"
 num_cpus = "1.17.0"
 once_cell = "1.21.4"
 ort = { version = "2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
 parking_lot = "0.12.5"
-pdf_oxide = { version = "0.3.49", default-features = false }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "crates/kreuzberg-pdfium-render", version = "4.3" }
 rayon = "1.12.0"
-reqwest = { version = "0.13.3", default-features = false }
+reqwest = { version = "0.13.4", default-features = false }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.149" }
+serde_json = { version = "1.0.150" }
 serde_toon_format = "0.1"
 tempfile = "3.27.0"
 thiserror = "2.0.18"

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.9.8"
+version = "4.9.9"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -39,10 +39,6 @@ simd-utf8 = ["dep:simdutf8"]
 tokio-runtime = ["dep:tokio"]
 
 pdf = ["dep:pdfium-render", "dep:lopdf", "dep:image", "dep:flate2", "html"]
-# Experimental: use pdf_oxide for text extraction (pure Rust, no C++ deps).
-# Provides cleaner word spacing for PDFs with broken font CMaps.
-# Requires 'pdf' feature. Not included in 'full' — opt-in only.
-pdf-oxide = ["pdf", "dep:pdf_oxide"]
 static-pdfium = ["pdf"]
 bundled-pdfium = ["pdf"]
 system-pdfium = ["pdf"]
@@ -61,7 +57,14 @@ office = [
 ]
 hwp = ["dep:cfb", "dep:flate2"]
 iwork = ["dep:zip", "dep:snap"]
-email = ["dep:mail-parser", "dep:cfb", "dep:outlook-pst", "dep:tempfile", "dep:chrono"]
+email = [
+    "dep:mail-parser",
+    "dep:cfb",
+    "dep:outlook-pst",
+    "dep:tempfile",
+    "dep:chrono",
+    "dep:chardetng",
+]
 html = ["dep:html-to-markdown-rs", "dep:v_htmlescape"]
 xml = ["dep:quick-xml", "dep:roxmltree"]
 archives = ["dep:zip", "dep:tar", "dep:sevenz-rust2", "dep:flate2"]
@@ -259,7 +262,7 @@ cfb = { version = "0.14", optional = true }
 chardetng = { version = "1.0.0", optional = true }
 chrono = { version = "0.4", optional = true }
 comrak = { version = "0.52", default-features = false }
-dashmap = "6.1"
+dashmap = "6.2"
 dbase = { version = "0.7", optional = true }
 dirs = "6"
 encoding_rs = { version = "0.8.35" }
@@ -271,7 +274,7 @@ hayro-jpeg2000 = { version = "0.3", default-features = false, features = [
     "simd",
 ], optional = true }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.4.1", default-features = false, features = [
+html-to-markdown-rs = { version = "3.5.7", default-features = false, features = [
     "inline-images",
     "metadata",
 ], optional = true }
@@ -294,9 +297,9 @@ kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
 libc = "0.2.186"
 liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false, optional = true }
 log = "0.4"
-lopdf = { version = "0.40.0", optional = true }
+lopdf = { version = "0.41.0", optional = true }
 mail-parser = { version = "0.11.3", optional = true }
-memchr = "2.8.0"
+memchr = "2.8.1"
 memmap2 = "0.9"
 mime_guess = "2.0"
 minijinja = { version = "2", optional = true }
@@ -314,7 +317,6 @@ ort = { version = "2.0.0-rc.12", default-features = false, features = [
 outlook-pst = { version = "1.2.0", optional = true }
 parking_lot = "0.12.5"
 pastey = "0.2"
-pdf_oxide = { version = "0.3.49", default-features = false, optional = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "../kreuzberg-pdfium-render", features = ["thread_safe", "image_latest"], optional = true }
 pulldown-cmark = { version = "0.13" }
 quick-xml = { version = "0.40.1", features = ["serialize"], optional = true }
@@ -333,22 +335,21 @@ rmp-serde = "1.3"
 
 roxmltree = { version = "0.21.1", optional = true }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.149" }
+serde_json = { version = "1.0.150" }
 serde_toon_format = "0.1"
 serde_yaml_ng = "0.10.0"
 sevenz-rust2 = { version = "0.20.2", optional = true }
 sha2 = { version = "0.11", optional = true }
 simdutf8 = { version = "0.1", optional = true }
 snap = { version = "1.1", optional = true }
-tar = { version = "0.4.45", optional = true }
+tar = { version = "0.4.46", optional = true }
 tempfile = { version = "3.27.0", optional = true }
-text-splitter = { version = "0.30.1", features = ["markdown"], optional = true }
+text-splitter = { version = "0.31.0", features = ["markdown"], optional = true }
 thiserror = "2.0.18"
 tiff = { version = "0.11", optional = true }
-# Pinned to 0.22 — text-splitter 0.30.1 ChunkSizer impl + embeddings/add_special_tokens
-# break against tokenizers 0.23. Bump deliberately on the next minor with a coordinated
-# text-splitter upgrade. Tracked under issue #991 / 4.9.8 release.
-tokenizers = { version = "=0.22.2", optional = true, default-features = false, features = [
+# Keep aligned with text-splitter's optional tokenizers integration so ChunkSizer
+# is implemented for the same Tokenizer type used by Kreuzberg.
+tokenizers = { version = "0.23.1", optional = true, default-features = false, features = [
     "http",
     "fancy-regex",
 ] }
@@ -357,7 +358,7 @@ toml = "1.1.2"
 tower = { version = "0.5", features = ["timeout", "limit", "util"], optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit", "catch-panic", "request-id", "sensitive-headers", "compression-full"], optional = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.32", optional = true }
+tracing-opentelemetry = { version = "0.33", optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 urlencoding = "2"
 utoipa = { version = "5.5", features = ["axum_extras"], optional = true }
@@ -411,8 +412,8 @@ dotenvy = "0.15"
 filetime = "0.2"
 image = { version = "0.25.10", default-features = false, features = ["png"] }
 jsonschema = "0.46"
-serial_test = "3.4.0"
-tar = "0.4.45"
+serial_test = "3.5.0"
+tar = "0.4.46"
 tempfile = "3.27.0"
 tokio = { version = "1.52.3", features = ["macros", "time"] }
 tokio-test = "0.4"

data/vendor/kreuzberg/README.md

@@ -18,7 +18,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.9.8 Release**
+> **🚀 Version 4.9.9 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/core/config/pdf.rs

@@ -8,17 +8,14 @@ use serde::{Deserialize, Serialize};
 /// PDF extraction backend selection.
 ///
 /// Controls which PDF library is used for text extraction:
-/// - `Pdfium`: pdfium-render (default, C++ based, mature)
-/// - `PdfOxide`: pdf_oxide (pure Rust, faster, requires `pdf-oxide` feature)
-/// - `Auto`: automatically select based on available features
+/// - `Pdfium`: pdfium-render (default, mature)
+/// - `Auto`: automatically select the default available backend
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum PdfBackend {
     /// Use pdfium-render backend (default).
     #[default]
     Pdfium,
-    /// Use pdf_oxide backend (pure Rust). Requires `pdf-oxide` feature.
-    PdfOxide,
     /// Automatically select the best available backend.
     Auto,
 }

data/vendor/kreuzberg/src/core/extractor/bytes.rs

@@ -128,7 +128,12 @@ pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionC
 
     #[cfg(not(feature = "tokio-runtime"))]
     let result = {
-        let _ = config.extraction_timeout_secs;
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
         extraction_future.await
     };
 

data/vendor/kreuzberg/src/core/extractor/file.rs

@@ -142,7 +142,12 @@ pub async fn extract_file(
 
     #[cfg(not(feature = "tokio-runtime"))]
     let result = {
-        let _ = config.extraction_timeout_secs;
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
         extraction_future.await
     };
 

data/vendor/kreuzberg/src/core/extractor/legacy.rs

@@ -35,6 +35,13 @@ pub(super) fn extract_bytes_sync_impl(
     let cfg = config.cloned().unwrap_or_default();
     let cfg = cfg.normalized().into_owned();
 
+    if cfg.extraction_timeout_secs.is_some() {
+        return Err(crate::KreuzbergError::Validation {
+            message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+            source: None,
+        });
+    }
+
     let validated_mime = if let Some(mime) = mime_type {
         if mime == "application/octet-stream" {
             mime::detect_mime_type_from_bytes(content)?

data/vendor/kreuzberg/src/core/pipeline/features.rs

@@ -37,10 +37,19 @@ fn recompute_boundaries_from_pages(content: &str, pages: &[crate::types::PageCon
             continue;
         }
 
-        // Try exact match first
-        if let Some(pos) = content[search_offset..].find(&page.content) {
+        let normalized: String = page
+            .content
+            .split("\n\n")
+            .map(str::trim)
+            .filter(|s| !s.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n\n");
+
+        // Try normalized exact match first. PDF page text can contain trailing
+        // spaces that render_plain strips before chunking.
+        if let Some(pos) = content[search_offset..].find(normalized.as_str()) {
             let byte_start = search_offset + pos;
-            let byte_end = content.floor_char_boundary(byte_start + page.content.len());
+            let byte_end = content.floor_char_boundary(byte_start + normalized.len());
             boundaries.push(PageBoundary {
                 page_number: page.page_number,
                 byte_start,
@@ -50,12 +59,12 @@ fn recompute_boundaries_from_pages(content: &str, pages: &[crate::types::PageCon
             continue;
         }
 
-        // Fallback: search for first non-empty line of page content
+        // Fallback: search for first non-empty line of page content.
         if let Some(line) = page.content.lines().find(|l| !l.trim().is_empty()).map(|l| l.trim())
             && let Some(pos) = content[search_offset..].find(line)
         {
             let byte_start = search_offset + pos;
-            let raw_end = (byte_start + page.content.len()).min(content.len());
+            let raw_end = (byte_start + normalized.len()).min(content.len());
             let byte_end = content.floor_char_boundary(raw_end);
             boundaries.push(PageBoundary {
                 page_number: page.page_number,
@@ -176,25 +185,27 @@ pub(super) fn execute_chunking(result: &mut ExtractionResult, config: &Extractio
         let resolved_config = chunking_config.resolve_preset();
         let chunking_config = &resolved_config;
 
-        // Recompute page boundaries against `result.content` (rendered by `render_plain`)
-        // if per-page content is available.  The boundaries stored in
-        // `result.metadata.pages.boundaries` were computed against the raw extractor text
-        // and may have different byte offsets than the rendered content (fix for #636).
+        let (chunk_input, heading_source) = if config.output_format != crate::core::config::OutputFormat::Plain {
+            (
+                result.formatted_content.as_deref().unwrap_or(result.content.as_str()),
+                None,
+            )
+        } else {
+            (result.content.as_str(), result.formatted_content.as_deref())
+        };
+
         let recomputed_boundaries: Option<Vec<PageBoundary>> = result
             .pages
             .as_deref()
-            .map(|pages| recompute_boundaries_from_pages(&result.content, pages));
+            .map(|pages| recompute_boundaries_from_pages(chunk_input, pages))
+            .filter(|boundaries| !boundaries.is_empty());
 
         let page_boundaries: Option<&[PageBoundary]> = recomputed_boundaries
             .as_deref()
             .or_else(|| result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref()));
 
-        // Pass formatted_content (markdown) for heading context resolution when available.
-        // Plain-text rendering strips heading markers, but the markdown chunker needs them
-        // to build the heading hierarchy for chunk metadata.
-        let heading_source = result.formatted_content.as_deref();
         match crate::chunking::chunk_text_with_heading_source(
-            &result.content,
+            chunk_input,
             chunking_config,
             page_boundaries,
             heading_source,
@@ -314,3 +325,92 @@ pub(super) fn execute_token_reduction(result: &mut ExtractionResult, config: &Ex
 
     Ok(())
 }
+
+#[cfg(test)]
+#[cfg(feature = "chunking")]
+mod tests {
+    use super::*;
+    use crate::core::config::{ChunkerType, ChunkingConfig, OutputFormat};
+    use crate::types::PageContent;
+
+    fn make_page(page_number: usize, content: &str) -> PageContent {
+        PageContent {
+            page_number,
+            content: content.to_string(),
+            tables: Vec::new(),
+            images: Vec::new(),
+            hierarchy: None,
+            is_blank: None,
+            layout_regions: None,
+        }
+    }
+
+    fn markdown_chunking_config() -> ExtractionConfig {
+        ExtractionConfig {
+            output_format: OutputFormat::Markdown,
+            chunking: Some(ChunkingConfig {
+                max_characters: 2000,
+                overlap: 0,
+                trim: true,
+                chunker_type: ChunkerType::Markdown,
+                ..Default::default()
+            }),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn chunks_content_is_markdown_when_output_format_is_markdown() {
+        let mut result = ExtractionResult {
+            content: "SH-001 Luca Bianchi Common Germany 3500000".to_string(),
+            formatted_content: Some("| SH-001 | Luca Bianchi | Common | Germany | 3,500,000 |".to_string()),
+            mime_type: Cow::Borrowed("application/pdf"),
+            ..Default::default()
+        };
+
+        execute_chunking(&mut result, &markdown_chunking_config()).unwrap();
+
+        let chunks = result.chunks.expect("chunks must be populated");
+        assert!(!chunks.is_empty());
+        assert!(chunks.iter().any(|chunk| chunk.content.contains('|')));
+        assert!(chunks.iter().all(|chunk| !chunk.content.starts_with("SH-001 Luca")));
+        assert!(result.formatted_content.is_some());
+    }
+
+    #[test]
+    fn markdown_chunks_preserve_page_metadata_when_formatted_pages_match() {
+        let mut result = ExtractionResult {
+            content: "Page one text\n\nPage two text".to_string(),
+            formatted_content: Some("# Page one\n\nPage one text\n\n# Page two\n\nPage two text".to_string()),
+            pages: Some(vec![make_page(1, "Page one text"), make_page(2, "Page two text")]),
+            mime_type: Cow::Borrowed("application/pdf"),
+            ..Default::default()
+        };
+
+        execute_chunking(&mut result, &markdown_chunking_config()).unwrap();
+
+        let chunks = result.chunks.expect("chunks must be populated");
+        assert!(!chunks.is_empty());
+        assert!(chunks.iter().any(|chunk| chunk.metadata.first_page.is_some()));
+        assert!(chunks.iter().any(|chunk| chunk.metadata.last_page.is_some()));
+    }
+
+    #[test]
+    fn recompute_boundaries_trailing_space_pages_all_resolve() {
+        let p1_raw = "Heading \n\nBody paragraph one. ";
+        let p2_raw = "Second heading \n\nBody paragraph two. ";
+        let p3_raw = "Conclusion. ";
+        let p1_norm = "Heading\n\nBody paragraph one.";
+        let p2_norm = "Second heading\n\nBody paragraph two.";
+        let p3_norm = "Conclusion.";
+        let content = format!("{p1_norm}\n\n{p2_norm}\n\n{p3_norm}");
+
+        let pages = vec![make_page(1, p1_raw), make_page(2, p2_raw), make_page(3, p3_raw)];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        assert_eq!(boundaries.len(), 3);
+        assert_eq!(&content[boundaries[0].byte_start..boundaries[0].byte_end], p1_norm);
+        assert_eq!(&content[boundaries[1].byte_start..boundaries[1].byte_end], p2_norm);
+        assert_eq!(&content[boundaries[2].byte_start..boundaries[2].byte_end], p3_norm);
+    }
+}

data/vendor/kreuzberg/src/embeddings/mod.rs

@@ -270,11 +270,13 @@ fn load_tokenizer(
     {
         for (_, value) in &map {
             if let Some(content) = value.as_str() {
-                tokenizer.add_special_tokens(&[AddedToken {
-                    content: content.to_string(),
-                    special: true,
-                    ..Default::default()
-                }]);
+                tokenizer
+                    .add_special_tokens([AddedToken {
+                        content: content.to_string(),
+                        special: true,
+                        ..Default::default()
+                    }])
+                    .map_err(|e| crate::KreuzbergError::embedding(format!("Failed to add special token: {e}")))?;
             } else if value.is_object()
                 && let (Some(content), Some(single_word), Some(lstrip), Some(rstrip), Some(normalized)) = (
                     value["content"].as_str(),
@@ -284,14 +286,16 @@ fn load_tokenizer(
                     value["normalized"].as_bool(),
                 )
             {
-                tokenizer.add_special_tokens(&[AddedToken {
-                    content: content.to_string(),
-                    special: true,
-                    single_word,
-                    lstrip,
-                    rstrip,
-                    normalized,
-                }]);
+                tokenizer
+                    .add_special_tokens([AddedToken {
+                        content: content.to_string(),
+                        special: true,
+                        single_word,
+                        lstrip,
+                        rstrip,
+                        normalized,
+                    }])
+                    .map_err(|e| crate::KreuzbergError::embedding(format!("Failed to add special token: {e}")))?;
             }
         }
     }

data/vendor/kreuzberg/src/extraction/email.rs

@@ -74,12 +74,23 @@ fn maybe_transcode_utf16(data: &[u8]) -> Option<Vec<u8>> {
         (true, 2)
     } else if data[0] == 0xFE && data[1] == 0xFF {
         (false, 2)
-    } else if data[1] == 0x00 && data[3] == 0x00 && data[0] != 0x00 && data[2] != 0x00 {
-        // No BOM, but looks like UTF-16 LE (e.g. "M\0I\0M\0E\0")
-        (true, 0)
-    } else if data[0] == 0x00 && data[2] == 0x00 && data[1] != 0x00 && data[3] != 0x00 {
-        // No BOM, but looks like UTF-16 BE (e.g. "\0M\0I\0M\0E")
-        (false, 0)
+    } else if data.len() >= 16 {
+        let is_le_heuristic = data[1] == 0x00 && data[3] == 0x00 && data[5] == 0x00 && data[7] == 0x00;
+        let is_be_heuristic = data[0] == 0x00 && data[2] == 0x00 && data[4] == 0x00 && data[6] == 0x00;
+
+        if is_le_heuristic || is_be_heuristic {
+            let mut detector = chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow);
+            detector.feed(data, true);
+            let guess = detector.guess(None, chardetng::Utf8Detection::Allow);
+
+            if guess.name() == "UTF-8" || guess.name() == "windows-1252" {
+                (is_le_heuristic, 0)
+            } else {
+                return None;
+            }
+        } else {
+            return None;
+        }
     } else {
         return None;
     };
@@ -553,6 +564,8 @@ Courier{\\colortbl\\red0\\green0\\blue0\r\n\\par \\pard\\plain\\f0\\fs20\\b\\i\\
 \\scaps\\outline\\shadow\\imprint\\emboss\\lang1024\\sbasedon1033\\fcharset0 {\\*\\cs10 \\additive \
 Default Paragraph Font}";
 
+const MAX_RTF_DECOMPRESSED_CAPACITY: usize = 16 * 1024 * 1024;
+
 /// Decompress a PR_RTF_COMPRESSED stream per the MS-OXRTFCP specification.
 ///
 /// Returns `None` when the data is too short, has a bad magic number, or
@@ -585,7 +598,7 @@ fn decompress_rtf_compressed(data: &[u8]) -> Option<Vec<u8>> {
     // comp_size includes the 12 bytes after the first u32, so input length should be comp_size - 12.
     let end = (comp_size.saturating_sub(12)).min(input.len());
 
-    let mut output = Vec::with_capacity(raw_size as usize);
+    let mut output = Vec::with_capacity((raw_size as usize).min(MAX_RTF_DECOMPRESSED_CAPACITY));
     let mut pos = 0usize;
 
     while pos < end {
@@ -2105,6 +2118,44 @@ mod tests {
         assert_eq!(headers.get("user_agent").unwrap(), "MyAgent/1.0");
     }
 
+    #[test]
+    fn test_maybe_transcode_utf16_short_binary_does_not_trigger_heuristic() {
+        assert!(maybe_transcode_utf16(&[b'M', 0, b'I', 0]).is_none());
+    }
+
+    #[test]
+    fn test_decompress_rtf_compressed_crafted_raw_size_does_not_over_allocate() {
+        let mut data = Vec::with_capacity(20);
+        data.extend_from_slice(&16u32.to_le_bytes());
+        data.extend_from_slice(&0xFFFF_FFFFu32.to_le_bytes());
+        data.extend_from_slice(&0x75465a4cu32.to_le_bytes());
+        data.extend_from_slice(&0u32.to_le_bytes());
+        data.extend_from_slice(&[0x00, b'A', b'B', b'C']);
+
+        let out = decompress_rtf_compressed(&data).expect("crafted size should not force OOM");
+        assert!(out.len() < 16, "output should stay tiny");
+    }
+
+    #[test]
+    fn test_decompress_rtf_compressed_cap_is_hint_only() {
+        let payload: &[u8] = &[
+            0x00, b'A', b'B', b'C', b'D', b'E', b'F', b'G', b'H', 0x00, b'I', b'J', b'K', b'L', b'M', b'N', b'O', b'P',
+            0x00, b'Q', b'R', b'S', b'T', b'U', b'V', b'W', b'X',
+        ];
+        let comp_size = (12 + payload.len()) as u32;
+        let raw_size = 1u32;
+        let mut data = Vec::new();
+        data.extend_from_slice(&comp_size.to_le_bytes());
+        data.extend_from_slice(&raw_size.to_le_bytes());
+        data.extend_from_slice(&0x75465a4cu32.to_le_bytes());
+        data.extend_from_slice(&0u32.to_le_bytes());
+        data.extend_from_slice(payload);
+
+        let out = decompress_rtf_compressed(&data).expect("should decompress");
+        assert_eq!(out.len(), 24);
+        assert_eq!(&out[..8], b"ABCDEFGH");
+    }
+
     #[test]
     fn test_decompress_rtf_compressed_too_short() {
         assert!(decompress_rtf_compressed(&[0u8; 10]).is_none());