kreuzberg 4.9.7 → 4.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24151d9931038d170843ec2b42b0f34b65d78a6f07418683ba71a96c7b2f4f50
-  data.tar.gz: 701aedb58613376393f8e168dbd2ecea211063172cbfdaae7af96590d990e8c1
+  metadata.gz: b634075425816167cdb132080aa31a6c5c561badbd154de17896866bdf88ba6e
+  data.tar.gz: 4a8d041bad2da842a676b2e2358ba44d3269323e820728bdb568ebc9171adee1
 SHA512:
-  metadata.gz: b8917f84b8d8f7861c2f71a41248244344da6444942267e65b7feb04123017f099d618b6d72632d710936983f8ee9771e6847447e2119363d6e123caf8cc4f60
-  data.tar.gz: 59fb7bd5a2c079fde4221aa1d578542cdf1b5f3637b28787d735d7a87b6fc86dd0f239e0623d4284c39d1175bed1072f4291a644c435d7242db9e5a3abeb5575
+  metadata.gz: 60c018a882054c23b629ee1d0692493c25c78a52e106fc7f25e3c22d7cdcfd36341f5aaeebea3d6be48d1425550914124daaeaeeb02bbd84bbb64cd6afedad59
+  data.tar.gz: 62d942f23dae32120184cd5b011275a2556b3e112a866128bd290a67ddbbd94c523ec2eaecd24589911264609ae612fd22eb51f3efc2e9eec4b1d08f8bd6823c

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.7" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.9.8" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-rb"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -49,13 +49,13 @@ kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi" }
 magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
     "rb-sys",
 ] }
-rb-sys = { version = "0.9.126", default-features = false, features = [
+rb-sys = { version = "0.9.128", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
 serde_json = "1.0.149"
 toml = "1.1.2"
 serde_yaml_ng = "0.10"
-tokio = { version = "1.52.1", features = [
+tokio = { version = "1.52.3", features = [
     "rt",
     "rt-multi-thread",
     "macros",
@@ -65,7 +65,7 @@ tokio = { version = "1.52.1", features = [
     "time",
     "io-util",
 ] }
-html-to-markdown-rs = { version = "3.3.1", default-features = false }
+html-to-markdown-rs = { version = "3.4.1", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.9.7'
+  VERSION = '4.9.8'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -23,20 +23,20 @@ clap = { version = "4.6", features = ["derive", "color", "suggestions"] }
 comrak = { version = "0.52", default-features = false }
 console_error_panic_hook = "0.1"
 criterion = { version = "0.8", features = ["html_reports"] }
-ctor = "0.10"
+ctor = "1.0"
 dbase = "0.7"
 futures = "0.3"
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.3.1", default-features = false }
+html-to-markdown-rs = { version = "3.4.1", default-features = false }
 image = { version = "0.25.10", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
-kreuzberg = { path = "./crates/kreuzberg", version = "4.9.7", default-features = false }
-kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.7" }
+kreuzberg = { path = "./crates/kreuzberg", version = "4.9.8", default-features = false }
+kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.9.8" }
 lazy_static = "1.5.0"
-libc = "0.2.185"
-liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false }
+libc = "0.2.186"
+liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false }
 log = "0.4"
 lzma-rust2 = { version = "0.16.2" }
 memmap2 = "0.9"
@@ -45,19 +45,19 @@ num_cpus = "1.17.0"
 once_cell = "1.21.4"
 ort = { version = "2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
 parking_lot = "0.12.5"
-pdf_oxide = { version = "0.3.37", default-features = false }
+pdf_oxide = { version = "0.3.49", default-features = false }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "crates/kreuzberg-pdfium-render", version = "4.3" }
 rayon = "1.12.0"
-reqwest = { version = "0.13.2", default-features = false }
+reqwest = { version = "0.13.3", default-features = false }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149" }
 serde_toon_format = "0.1"
 tempfile = "3.27.0"
 thiserror = "2.0.18"
-tokio = { version = "1.52.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
+tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
 toml = "1.1.2"
 tracing = "0.1"
-tree-sitter-language-pack = { version = "1.7.0", features = ["serde"], default-features = false }
+tree-sitter-language-pack = { version = "1.8.1", features = ["serde"], default-features = false }
 wasm-bindgen = { version = "0.2", features = ["enable-interning"] }
 wasm-bindgen-futures = "0.4"
 web-sys = { version = "0.3", features = ["Blob", "File", "FileReader", "console", "TextDecoder", "ImageData", "Window", "Response"] }

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -254,7 +254,7 @@ biblib = { version = "0.4", default-features = false, features = [
 bitvec = "1.0"
 blake3 = "1"
 bytes = { version = "1", features = ["serde"] }
-calamine = { version = "0.34.0", features = ["dates"], optional = true }
+calamine = { version = "0.35.0", features = ["dates"], optional = true }
 cfb = { version = "0.14", optional = true }
 chardetng = { version = "1.0.0", optional = true }
 chrono = { version = "0.4", optional = true }
@@ -271,7 +271,7 @@ hayro-jpeg2000 = { version = "0.3", default-features = false, features = [
     "simd",
 ], optional = true }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "3.3.1", default-features = false, features = [
+html-to-markdown-rs = { version = "3.4.1", default-features = false, features = [
     "inline-images",
     "metadata",
 ], optional = true }
@@ -291,11 +291,11 @@ jotdown = "0.10"
 kamadak-exif = { version = "0.6.1", optional = true }
 
 kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
-libc = "0.2.185"
-liter-llm = { version = "1.2", features = ["native-http", "tracing"], default-features = false, optional = true }
+libc = "0.2.186"
+liter-llm = { version = "1.3", features = ["native-http", "tracing"], default-features = false, optional = true }
 log = "0.4"
 lopdf = { version = "0.40.0", optional = true }
-mail-parser = { version = "0.11.2", optional = true }
+mail-parser = { version = "0.11.3", optional = true }
 memchr = "2.8.0"
 memmap2 = "0.9"
 mime_guess = "2.0"
@@ -303,8 +303,8 @@ minijinja = { version = "2", optional = true }
 ndarray = { version = "0.17", optional = true }
 num_cpus = "1.17.0"
 once_cell = "1.21.4"
-opentelemetry = { version = "0.31", features = ["trace"], optional = true }
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"], optional = true }
+opentelemetry = { version = "0.32", features = ["trace"], optional = true }
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"], optional = true }
 org = { version = "0.3", optional = true }
 ort = { version = "2.0.0-rc.12", default-features = false, features = [
     "std",
@@ -314,14 +314,14 @@ ort = { version = "2.0.0-rc.12", default-features = false, features = [
 outlook-pst = { version = "1.2.0", optional = true }
 parking_lot = "0.12.5"
 pastey = "0.2"
-pdf_oxide = { version = "0.3.37", default-features = false, optional = true }
+pdf_oxide = { version = "0.3.49", default-features = false, optional = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", path = "../kreuzberg-pdfium-render", features = ["thread_safe", "image_latest"], optional = true }
 pulldown-cmark = { version = "0.13" }
-quick-xml = { version = "0.39.2", features = ["serialize"], optional = true }
+quick-xml = { version = "0.40.1", features = ["serialize"], optional = true }
 rake = { version = "0.3.6", optional = true }
 rayon = "1.12.0"
 regex = "1.12.3"
-rmcp = { version = "1.5.0", features = [
+rmcp = { version = "1.7.0", features = [
     "server",
     "macros",
     "base64",
@@ -345,11 +345,14 @@ tempfile = { version = "3.27.0", optional = true }
 text-splitter = { version = "0.30.1", features = ["markdown"], optional = true }
 thiserror = "2.0.18"
 tiff = { version = "0.11", optional = true }
-tokenizers = { version = "0.22", optional = true, default-features = false, features = [
+# Pinned to 0.22 — text-splitter 0.30.1 ChunkSizer impl + embeddings/add_special_tokens
+# break against tokenizers 0.23. Bump deliberately on the next minor with a coordinated
+# text-splitter upgrade. Tracked under issue #991 / 4.9.8 release.
+tokenizers = { version = "=0.22.2", optional = true, default-features = false, features = [
     "http",
     "fancy-regex",
 ] }
-tokio = { version = "1.52.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"], optional = true }
+tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"], optional = true }
 toml = "1.1.2"
 tower = { version = "0.5", features = ["timeout", "limit", "util"], optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit", "catch-panic", "request-id", "sensitive-headers", "compression-full"], optional = true }
@@ -357,8 +360,10 @@ tracing = "0.1"
 tracing-opentelemetry = { version = "0.32", optional = true }
 unicode-normalization = { version = "0.1.25", optional = true }
 urlencoding = "2"
-utoipa = { version = "5.4", features = ["axum_extras"], optional = true }
-v_htmlescape = { version = "0.15", optional = true }
+utoipa = { version = "5.5", features = ["axum_extras"], optional = true }
+# Pinned to 0.15 — v_htmlescape 0.17 renamed `escape` fn to an `Escape` struct.
+# Update call sites in src/rendering/html_styled.rs before bumping.
+v_htmlescape = { version = "=0.15.8", optional = true }
 whatlang = { version = "0.18.0", optional = true }
 zip = { version = ">=7.0.0", optional = true, default-features = false, features = [
     "deflate-flate2",
@@ -392,7 +397,7 @@ optional = true
 # Override getrandom to enable js feature for WASM targets
 # This is needed because ring/rustls (via ureq) depend on getrandom without js feature
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
-tree-sitter-language-pack = { version = "1.7.0", features = ["serde"], default-features = false, optional = true }
+tree-sitter-language-pack = { version = "1.8.1", features = ["serde"], default-features = false, optional = true }
 wasm-bindgen-rayon = { version = "1.3", optional = true }
 
 [build-dependencies]
@@ -409,7 +414,7 @@ jsonschema = "0.46"
 serial_test = "3.4.0"
 tar = "0.4.45"
 tempfile = "3.27.0"
-tokio = { version = "1.52.1", features = ["macros", "time"] }
+tokio = { version = "1.52.3", features = ["macros", "time"] }
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zip = { version = ">=7.0.0, <8.6.0", default-features = false, features = ["deflate-flate2"] }

data/vendor/kreuzberg/README.md

@@ -18,7 +18,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.9.7 Release**
+> **🚀 Version 4.9.8 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/core/config/tree_sitter.rs

@@ -155,7 +155,6 @@ impl From<&TreeSitterProcessConfig> for tree_sitter_language_pack::ProcessConfig
             symbols: p.symbols,
             diagnostics: p.diagnostics,
             chunk_max_size: p.chunk_max_size,
-            extractions: None,
         }
     }
 }

data/vendor/kreuzberg/src/extractors/rtf/encoding.rs

@@ -1,6 +1,8 @@
 //! Character encoding utilities for RTF parsing.
 //!
-//! Provides hex byte parsing and Windows-1252 character mapping for the 0x80-0x9F range.
+//! Provides hex byte parsing and legacy Windows codepage decoding for RTF byte escapes.
+
+use encoding_rs::Encoding;
 
 /// Convert a hex digit character to its numeric value.
 ///
@@ -69,6 +71,48 @@ pub fn decode_windows_1252(byte: u8) -> char {
     }
 }
 
+/// Map a Windows codepage number to an `encoding_rs` encoding.
+///
+/// Unknown values fall back to Windows-1252, the RTF default ANSI codepage.
+#[inline]
+pub(crate) fn encoding_for_windows_codepage(codepage: u32) -> &'static Encoding {
+    let label: &[u8] = match codepage {
+        65001 => b"utf-8",
+        20127 => b"us-ascii",
+        1250 => b"windows-1250",
+        1251 => b"windows-1251",
+        1252 => b"windows-1252",
+        1253 => b"windows-1253",
+        1254 => b"windows-1254",
+        1255 => b"windows-1255",
+        1256 => b"windows-1256",
+        1257 => b"windows-1257",
+        1258 => b"windows-1258",
+        932 | 10001 => b"shift_jis",
+        936 | 10008 => b"gbk",
+        949 | 10003 => b"euc-kr",
+        950 | 10002 => b"big5",
+        28591 => b"iso-8859-1",
+        28592 => b"iso-8859-2",
+        28595 => b"iso-8859-5",
+        28597 => b"iso-8859-7",
+        28599 => b"iso-8859-9",
+        _ => b"windows-1252",
+    };
+    Encoding::for_label(label).unwrap_or(encoding_rs::WINDOWS_1252)
+}
+
+/// Decode RTF hex escape bytes using the active ANSI codepage.
+#[inline]
+pub(crate) fn decode_ansi_bytes(bytes: &[u8], codepage: u32) -> String {
+    if codepage == 1252 {
+        return bytes.iter().map(|&byte| decode_windows_1252(byte)).collect();
+    }
+
+    let (decoded, _, _) = encoding_for_windows_codepage(codepage).decode(bytes);
+    decoded.into_owned()
+}
+
 /// Parse an RTF control word and extract its value.
 ///
 /// Returns a tuple of (control_word, optional_numeric_value).

data/vendor/kreuzberg/src/extractors/rtf/mod.rs

@@ -486,6 +486,19 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_rtf_ansicpg1251_hex_escape_extraction() {
+        let rtf_content =
+            r#"{\rtf1\ansi\ansicpg1251\deff0{\fonttbl{\f0\fnil\fcharset204 Arial;}}\f0 \'cf\'f0\'e8\'e2\'e5\'f2}"#;
+        let (text, _, _, _, _) = extract_text_from_rtf(rtf_content, true);
+
+        assert!(text.contains("Привет"), "expected readable Cyrillic, got: {text:?}");
+        assert!(
+            !text.contains("Ïðèâåò"),
+            "should not decode CP1251 bytes as Windows-1252"
+        );
+    }
+
     #[tokio::test]
     async fn test_rtf_document_structure_with_annotations() {
         let rtf_content = r#"{\rtf1 Normal text\par {\b Bold paragraph}\par More normal text}"#;

data/vendor/kreuzberg/src/extractors/rtf/parser.rs

@@ -1,6 +1,6 @@
 //! Core RTF parsing logic.
 
-use crate::extractors::rtf::encoding::{decode_windows_1252, parse_hex_byte, parse_rtf_control_word};
+use crate::extractors::rtf::encoding::{decode_ansi_bytes, parse_hex_byte, parse_rtf_control_word};
 use crate::extractors::rtf::formatting::{map_offset, normalize_whitespace_with_mapping};
 use crate::extractors::rtf::images::{RtfImage, extract_pict_image};
 use crate::extractors::rtf::tables::TableState;
@@ -944,6 +944,9 @@ pub fn extract_text_from_rtf(
     // Unicode skip count (\ucN): how many replacement bytes follow \uN.
     // Scoped per group — push on '{', pop on '}'.
     let mut uc_stack: Vec<u8> = vec![1]; // default \uc1
+    // ANSI codepage for \'hh escapes. RTF defaults to Windows-1252 unless
+    // overridden by \ansicpgNNNN. Scoped like other document properties.
+    let mut ansi_codepage_stack: Vec<u32> = vec![1252];
 
     // Hyperlink field tracking for \field{\*\fldinst HYPERLINK "url"}{\fldrslt text}
     let mut in_fldinst = false;
@@ -1009,6 +1012,8 @@ pub fn extract_text_from_rtf(
                 // Inherit current uc value into new group scope
                 let current_uc = uc_stack.last().copied().unwrap_or(1);
                 uc_stack.push(current_uc);
+                let current_codepage = ansi_codepage_stack.last().copied().unwrap_or(1252);
+                ansi_codepage_stack.push(current_codepage);
                 // Inherit hidden state into new group scope
                 let current_hidden = hidden_stack.last().copied().unwrap_or(false);
                 hidden_stack.push(current_hidden);
@@ -1028,6 +1033,9 @@ pub fn extract_text_from_rtf(
                 if uc_stack.len() > 1 {
                     uc_stack.pop();
                 }
+                if ansi_codepage_stack.len() > 1 {
+                    ansi_codepage_stack.pop();
+                }
                 if hidden_stack.len() > 1 {
                     hidden_stack.pop();
                 }
@@ -1143,6 +1151,7 @@ pub fn extract_text_from_rtf(
                                 &mut para_metas,
                                 &mut para_meta_emitted,
                                 &mut uc_stack,
+                                &mut ansi_codepage_stack,
                                 &mut footnote_count,
                                 in_footnote,
                                 &mut footnote_buf,
@@ -1188,12 +1197,21 @@ pub fn extract_text_from_rtf(
                             expect_destination = false;
                             let hex1 = chars.next();
                             let hex2 = chars.next();
-                            // Capture hex-encoded chars in footnote buffer even when skipping
-                            if in_footnote
-                                && let (Some(h1), Some(h2)) = (hex1, hex2)
+                            let bytes = if let (Some(h1), Some(h2)) = (hex1, hex2)
                                 && let Some(byte) = parse_hex_byte(h1, h2)
                             {
-                                footnote_buf.push(decode_windows_1252(byte));
+                                let mut bytes = vec![byte];
+                                while let Some(next_bytes) = consume_adjacent_hex_escape(&mut chars) {
+                                    bytes.push(next_bytes);
+                                }
+                                Some(bytes)
+                            } else {
+                                None
+                            };
+
+                            if in_footnote && let Some(bytes) = bytes.as_deref() {
+                                let codepage = ansi_codepage_stack.last().copied().unwrap_or(1252);
+                                footnote_buf.push_str(&decode_ansi_bytes(bytes, codepage));
                             }
                             if skip_depth > 0 {
                                 continue;
@@ -1202,14 +1220,13 @@ pub fn extract_text_from_rtf(
                             if hidden_stack.last().copied().unwrap_or(false) {
                                 continue;
                             }
-                            if let (Some(h1), Some(h2)) = (hex1, hex2)
-                                && let Some(byte) = parse_hex_byte(h1, h2)
-                            {
-                                let decoded = decode_windows_1252(byte);
+                            if let Some(bytes) = bytes.as_deref() {
+                                let codepage = ansi_codepage_stack.last().copied().unwrap_or(1252);
+                                let decoded = decode_ansi_bytes(bytes, codepage);
                                 if let Some(state) = table_state.as_mut()
                                     && state.in_row
                                 {
-                                    state.current_cell.push(decoded);
+                                    state.current_cell.push_str(&decoded);
                                 } else {
                                     // Flush deferred boundary space
                                     if pending_boundary_space
@@ -1221,7 +1238,7 @@ pub fn extract_text_from_rtf(
                                     }
                                     pending_boundary_space = false;
                                     para_meta_emitted = false;
-                                    result.push(decoded);
+                                    result.push_str(&decoded);
                                     if let Some(flag) = group_has_text.last_mut() {
                                         *flag = true;
                                     }
@@ -1331,6 +1348,13 @@ pub fn extract_text_from_rtf(
                                 {
                                     *uc = val.max(0) as u8;
                                 }
+                                if control_word == "ansicpg"
+                                    && let Some(val) = _param
+                                    && val > 0
+                                    && let Some(codepage) = ansi_codepage_stack.last_mut()
+                                {
+                                    *codepage = val as u32;
+                                }
                                 // Capture unicode chars inside footnote buffers
                                 if in_footnote
                                     && control_word == "u"
@@ -1382,6 +1406,7 @@ pub fn extract_text_from_rtf(
                                 &mut para_metas,
                                 &mut para_meta_emitted,
                                 &mut uc_stack,
+                                &mut ansi_codepage_stack,
                                 &mut footnote_count,
                                 in_footnote,
                                 &mut footnote_buf,
@@ -1528,6 +1553,23 @@ pub fn extract_text_from_rtf(
     (final_result, tables, images, para_metas, formatting_data)
 }
 
+fn consume_adjacent_hex_escape(chars: &mut std::iter::Peekable<std::str::Chars>) -> Option<u8> {
+    let mut lookahead = chars.clone();
+    if lookahead.next()? != '\\' || lookahead.next()? != '\'' {
+        return None;
+    }
+    let h1 = lookahead.next()?;
+    let h2 = lookahead.next()?;
+    let byte = parse_hex_byte(h1, h2)?;
+
+    chars.next();
+    chars.next();
+    chars.next();
+    chars.next();
+
+    Some(byte)
+}
+
 /// Handle an RTF control word during parsing.
 #[allow(clippy::too_many_arguments, clippy::ptr_arg)]
 fn handle_control_word(
@@ -1549,6 +1591,7 @@ fn handle_control_word(
     para_metas: &mut Vec<ParagraphMeta>,
     para_meta_emitted: &mut bool,
     uc_stack: &mut Vec<u8>,
+    ansi_codepage_stack: &mut [u32],
     footnote_count: &mut usize,
     _in_footnote: bool,
     _footnote_buf: &mut String,
@@ -1617,6 +1660,14 @@ fn handle_control_word(
                 *uc = val.max(0) as u8;
             }
         }
+        "ansicpg" => {
+            if let Some(val) = param
+                && val > 0
+                && let Some(codepage) = ansi_codepage_stack.last_mut()
+            {
+                *codepage = val as u32;
+            }
+        }
         // Unicode escape: \u1234 (signed integer)
         "u" => {
             if let Some(code_num) = param {

data/vendor/kreuzberg/src/llm/structured.rs

@@ -92,23 +92,28 @@ pub async fn extract_structured(
     let sanitized_schema = sanitize_schema_for_provider(&config.schema, &config.llm.model);
 
     // Build chat request with JSON schema response format.
-    // Use field assignment because `stream` is pub(crate) in liter-llm.
-    let mut request = liter_llm::ChatCompletionRequest::default();
-    request.model = config.llm.model.clone();
-    request.messages = vec![liter_llm::Message::User(liter_llm::UserMessage {
-        content: liter_llm::UserContent::Text(prompt),
-        name: None,
-    })];
-    request.temperature = config.llm.temperature;
-    request.max_tokens = config.llm.max_tokens;
-    request.response_format = Some(liter_llm::ResponseFormat::JsonSchema {
-        json_schema: liter_llm::JsonSchemaFormat {
-            name: config.schema_name.clone(),
-            description: config.schema_description.clone(),
-            schema: sanitized_schema,
-            strict: Some(config.strict),
-        },
-    });
+    // Use field assignment because `stream` is pub(crate) in liter-llm; struct-init
+    // syntax with `..Default::default()` won't compile across the crate boundary.
+    #[allow(clippy::field_reassign_with_default)]
+    let request = {
+        let mut req = liter_llm::ChatCompletionRequest::default();
+        req.model = config.llm.model.clone();
+        req.messages = vec![liter_llm::Message::User(liter_llm::UserMessage {
+            content: liter_llm::UserContent::Text(prompt),
+            name: None,
+        })];
+        req.temperature = config.llm.temperature;
+        req.max_tokens = config.llm.max_tokens;
+        req.response_format = Some(liter_llm::ResponseFormat::JsonSchema {
+            json_schema: liter_llm::JsonSchemaFormat {
+                name: config.schema_name.clone(),
+                description: config.schema_description.clone(),
+                schema: sanitized_schema,
+                strict: Some(config.strict),
+            },
+        });
+        req
+    };
 
     let response = client
         .chat(request)

data/vendor/kreuzberg/src/llm/vlm_ocr.rs

@@ -136,12 +136,17 @@ pub async fn vlm_ocr(
         name: None,
     });
 
-    // Use mutable default because `stream` is pub(crate) in liter-llm.
-    let mut request = ChatCompletionRequest::default();
-    request.model = config.model.clone();
-    request.messages = vec![message];
-    request.temperature = config.temperature;
-    request.max_tokens = config.max_tokens;
+    // Use mutable default because `stream` is pub(crate) in liter-llm; struct-init
+    // syntax with `..Default::default()` won't compile across the crate boundary.
+    #[allow(clippy::field_reassign_with_default)]
+    let request = {
+        let mut req = ChatCompletionRequest::default();
+        req.model = config.model.clone();
+        req.messages = vec![message];
+        req.temperature = config.temperature;
+        req.max_tokens = config.max_tokens;
+        req
+    };
 
     let response = client.chat(request).await.map_err(|e| {
         crate::KreuzbergError::ocr(format!(

data/vendor/kreuzberg/src/pdf/oxide_text.rs

@@ -36,6 +36,7 @@ pub(crate) fn current_pdf_path() -> Option<PathBuf> {
 /// Returns segments per page (indexed by page number, 0-based).
 /// Returns `None` if pdf_oxide fails to open or extract the document.
 #[cfg(feature = "pdf")]
+#[allow(unused_mut)] // pdf is mutated under feature-gated paths only
 pub(crate) fn extract_segments_with_oxide(page_count: usize) -> Option<Vec<Vec<SegmentData>>> {
     let file_path = match current_pdf_path() {
         Some(p) => {

data/vendor/kreuzberg/tests/rtf_extractor_tests.rs

@@ -98,6 +98,33 @@ async fn test_rtf_accent_extraction() {
     );
 }
 
+/// Test extraction of RTF file with CP1251 hex byte escapes.
+///
+/// File: ansicpg1251.rtf
+/// Content: Cyrillic text encoded as `\'hh` bytes with `\ansicpg1251`
+/// Expected: Decodes byte escapes with the declared ANSI codepage
+#[tokio::test]
+async fn test_rtf_ansicpg1251_extraction() {
+    let config = ExtractionConfig::default();
+    let path = get_rtf_path("ansicpg1251.rtf");
+
+    let result = extract_file(&path, Some("application/rtf"), &config).await;
+
+    assert!(result.is_ok(), "RTF extraction should succeed for ansicpg1251.rtf");
+    let extraction = result.expect("Operation failed");
+
+    assert_eq!(extraction.mime_type, "application/rtf");
+    assert!(
+        extraction.content.contains("Привет, мир!"),
+        "Should decode CP1251 hex escapes as Cyrillic text (found: {})",
+        extraction.content
+    );
+    assert!(
+        !extraction.content.contains("Ïðèâåò"),
+        "Should not decode CP1251 bytes as Windows-1252 mojibake"
+    );
+}
+
 /// Test extraction of RTF file with bookmarks (internal anchors/references).
 ///
 /// File: bookmark.rtf
@@ -531,6 +558,7 @@ async fn test_rtf_no_critical_content_loss() {
 
     let must_extract = vec![
         "unicode.rtf",
+        "ansicpg1251.rtf",
         "accent.rtf",
         "heading.rtf",
         "list_simple.rtf",
@@ -574,7 +602,13 @@ async fn test_rtf_no_critical_content_loss() {
 async fn test_rtf_mime_type_preservation() {
     let config = ExtractionConfig::default();
 
-    let test_files = vec!["unicode.rtf", "accent.rtf", "heading.rtf", "list_simple.rtf"];
+    let test_files = vec![
+        "unicode.rtf",
+        "ansicpg1251.rtf",
+        "accent.rtf",
+        "heading.rtf",
+        "list_simple.rtf",
+    ];
 
     for filename in test_files {
         let path = get_rtf_path(filename);

data/vendor/kreuzberg-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-ffi"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -28,14 +28,14 @@ tree-sitter = ["kreuzberg/tree-sitter"]
 [dependencies]
 ahash = { version = "0.8.12", features = ["serde"] }
 async-trait = "0.1.89"
-ctor = "0.10"
-html-to-markdown-rs = { version = "3.3.1", default-features = false }
-kreuzberg = { path = "../kreuzberg", version = "4.9.7", default-features = false, features = ["bundled-pdfium", "full"] }
+ctor = "1.0"
+html-to-markdown-rs = { version = "3.4.1", default-features = false }
+kreuzberg = { path = "../kreuzberg", version = "4.9.8", default-features = false, features = ["bundled-pdfium", "full"] }
 log = "0.4"
 rayon = { version = "1.12.0", optional = true }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149" }
-tokio = { version = "1.52.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
+tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
 
 [build-dependencies]
 cbindgen = "0.29"

data/vendor/kreuzberg-ffi/kreuzberg.h

@@ -9,8 +9,8 @@
 
 #define KREUZBERG_VERSION_MAJOR 4
 #define KREUZBERG_VERSION_MINOR 9
-#define KREUZBERG_VERSION_PATCH 7
-#define KREUZBERG_VERSION "4.9.7"
+#define KREUZBERG_VERSION_PATCH 8
+#define KREUZBERG_VERSION "4.9.8"
 
 
 #include <stdarg.h>

data/vendor/kreuzberg-ffi/src/lib.rs

@@ -108,7 +108,7 @@ pub use types::*;
 pub use util::{kreuzberg_last_error, kreuzberg_last_error_code, kreuzberg_last_panic_context, kreuzberg_version};
 pub use validation::*;
 
-#[ctor::ctor]
+#[ctor::ctor(unsafe)]
 fn setup_onnx_runtime_path() {
     kreuzberg::ort_discovery::ensure_ort_available();
 }

data/vendor/kreuzberg-paddle-ocr/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-paddle-ocr"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-pdfium-render/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-pdfium-render"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/kreuzberg-tesseract/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg-tesseract"
-version = "4.9.7"
+version = "4.9.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -38,21 +38,21 @@ dynamic-linking = []
 thiserror = "2.0.18"
 
 [build-dependencies]
-cc = { version = "^1.2.60", optional = true }
+cc = { version = "^1.2.62", optional = true }
 cmake = { version = "0.1.58", optional = true }
 zip = { version = ">=7.0.0", optional = true, default-features = false, features = [
     "deflate-flate2-zlib-rs",
 ] }
 
 [target.'cfg(not(target_os = "windows"))'.build-dependencies]
-reqwest = { version = "0.13.2", default-features = false, features = [
+reqwest = { version = "0.13.3", default-features = false, features = [
     "blocking",
     "rustls",
 ], optional = true }
 
 # Use native-tls on Windows to avoid aws-lc-sys CMake build issues with MinGW
 [target.'cfg(target_os = "windows")'.build-dependencies]
-reqwest = { version = "0.13.2", default-features = false, features = [
+reqwest = { version = "0.13.3", default-features = false, features = [
     "blocking",
     "native-tls",
 ], optional = true }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzberg
 version: !ruby/object:Gem::Version
-  version: 4.9.7
+  version: 4.9.8
 platform: ruby
 authors:
 - Na'aman Hirschfeld
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-08 00:00:00.000000000 Z
+date: 2026-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys