kreuzberg 4.4.2 → 4.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b65c17f6cf4332ce366d8da0197077c2e222a262a2da29b0681c1139918527bf
-  data.tar.gz: 1082caefd179e5eeaa44b69ad93f02458dae2e6cc2edf5aaf1dfc1eee838276c
+  metadata.gz: 879027155ef841eb494ce23bdd9817664d93cc86acb59cf7d04408b6b717cff5
+  data.tar.gz: be26ecbf4ad1f07cfa7f02a197ef355ed534e4a708361b8a2fcf4feb6bce514a
 SHA512:
-  metadata.gz: '01496d04b7671527a8e861ac3d155a4c1ed96c75b6898d419ad021edba646e896fbd3fa46920c0a8ebf8d7eba79ab3ad83edce44261b813b16a1e6d28161fc61'
-  data.tar.gz: 64aa2e0654a3300c06948c65ed1d944f2c05f08f9989fcd27f5ab38e20772292d5913edea6beed522da4056ba0288f4416bed1ee6f90438155e3ec112d6ef4cb
+  metadata.gz: 980236a2633bfdc3b835aa8bf4b215aa623be2d61d74758bc6b668d5bcc48fecd7ed9dd97db5cb3a65dce3b2aae9aee8b15f2adb5df89bd48f7179fe0daab506
+  data.tar.gz: 57a7ed2f02591dbe8c7a79078504e2130953bb177151d246b8dffab40c9db1b260b526ac4abd0780b88835ea3378f835583a9400af7ae32481ad9c90ac4e86eb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.4.2)
+    kreuzberg (4.4.3)
       rb_sys (~> 0.9.119)
       sorbet-runtime (~> 0.5)
 
@@ -110,7 +110,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
-    rubocop (1.85.0)
+    rubocop (1.85.1)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -134,7 +134,7 @@ GEM
       rubocop (~> 1.81)
     ruby-progressbar (1.13.0)
     securerandom (0.4.1)
-    sorbet-runtime (0.6.12984)
+    sorbet-runtime (0.6.12993)
     steep (1.10.0)
       activesupport (>= 5.1)
       concurrent-ruby (>= 1.1.10)
@@ -222,7 +222,7 @@ CHECKSUMS
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.1) sha256=fe112755501b8d0466b5ada6cf50c8c3f41e897fa128ac5d263ec09eedc9f986
   json-schema (6.1.0) sha256=6bf70a2cfb6dfd5a06da28093fa8190f324c88eabd36a7f47097f227321dc702
-  kreuzberg (4.4.2)
+  kreuzberg (4.4.3)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2
@@ -253,13 +253,13 @@ CHECKSUMS
   rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
   rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47
   rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
-  rubocop (1.85.0) sha256=317407feb681a07d54f64d2f9e1d6b6af1ce7678e51cd658e3ad8bd66da48c01
+  rubocop (1.85.1) sha256=3dbcf9e961baa4c376eeeb2a03913dca5e3987033b04d38fa538aa1e7406cc77
   rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd
   rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834
   rubocop-rspec (3.9.0) sha256=8fa70a3619408237d789aeecfb9beef40576acc855173e60939d63332fdb55e2
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
-  sorbet-runtime (0.6.12984) sha256=3fff20a5b147a2e191210563d61886ac121fc1cd8b5e0faf6bc18873139e0fe4
+  sorbet-runtime (0.6.12993) sha256=5720d6e70063ed39528ddb18248c13a8072cf6991cf7d6652dcc0b8e9bc6b4ac
   steep (1.10.0) sha256=1b295b55f9aaff1b8d3ee42453ee55bc2a1078fda0268f288edb2dc014f4d7d1
   strscan (3.1.7) sha256=5f76462b94a3ea50b44973225b7d75b2cb96d4e1bee9ef1319b99ca117b72c8c
   terminal-table (4.0.0) sha256=f504793203f8251b2ea7c7068333053f0beeea26093ec9962e62ea79f94301d2

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.4.2" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.4.3" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -1390,9 +1390,9 @@ dependencies = [
 
 [[package]]
 name = "fastembed"
-version = "5.11.0"
+version = "5.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4339d45a80579ab8305616a501eacdbf18fb0f7def7fa6e4c0b75941416d5b0"
+checksum = "e7b01c79c5cb8ab3ce31c3d52916fda278e14cac027ff3a9cb66c419ed7288f8"
 dependencies = [
  "anyhow",
  "hf-hub 0.4.3",
@@ -1977,9 +1977,9 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "2.27.2"
+version = "2.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06f53f29c48f234830ccc312f652770723419fe9b3529fa5e0b027a0fedb7869"
+checksum = "842656c6f85a8e7b35fd5656646ffa0d6644cdcfe429634c7d9923a5626e330b"
 dependencies = [
  "ahash",
  "astral-tl",
@@ -2661,7 +2661,7 @@ dependencies = [
  "thiserror 2.0.18",
  "tiff 0.11.3",
  "tokio",
- "toml 1.0.3+spec-1.1.0",
+ "toml 1.0.4+spec-1.1.0",
  "tower",
  "tower-http",
  "tracing",
@@ -2729,7 +2729,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-rb"
-version = "4.4.1"
+version = "4.4.3"
 dependencies = [
  "async-trait",
  "html-to-markdown-rs",
@@ -2741,7 +2741,7 @@ dependencies = [
  "serde_json",
  "serde_yaml_ng",
  "tokio",
- "toml 1.0.3+spec-1.1.0",
+ "toml 1.0.4+spec-1.1.0",
 ]
 
 [[package]]
@@ -5294,9 +5294,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "1.0.3+spec-1.1.0"
+version = "1.0.4+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7614eaf19ad818347db24addfa201729cf2a9b6fdfd9eb0ab870fcacc606c0c"
+checksum = "c94c3321114413476740df133f0d8862c61d87c8d26f04c6841e033c8c80db47"
 dependencies = [
  "indexmap",
  "serde_core",
@@ -5700,9 +5700,9 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.21.0"
+version = "1.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",
@@ -6314,9 +6314,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
 [[package]]
 name = "winnow"
-version = "0.7.14"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
+checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
 
 [[package]]
 name = "wit-bindgen"

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -25,10 +25,10 @@ hex = "0.4.3"
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
 parking_lot = "0.12.5"
-html-to-markdown-rs = { version = "2.27.2", default-features = false }
+html-to-markdown-rs = { version = "2.28.0", default-features = false }
 reqwest = { version = "0.13.2", default-features = false }
 image = { version = "0.25.9", default-features = false }
-toml = "1.0.3"
+toml = "1.0.4"
 tempfile = "3.26.0"
 lzma-rust2 = { version = "0.16.2" }
 log = "0.4"
@@ -45,7 +45,7 @@ collapsible_if = "allow"
 
 [package]
 name = "kreuzberg-rb"
-version = "4.4.2"
+version = "4.4.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -99,7 +99,7 @@ rb-sys = { version = "0.9.124", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
 serde_json = "1.0.149"
-toml = "1.0.3"
+toml = "1.0.4"
 serde_yaml_ng = "0.10"
 tokio = { version = "1.50.0", features = [
     "rt",
@@ -111,7 +111,7 @@ tokio = { version = "1.50.0", features = [
     "time",
     "io-util",
 ] }
-html-to-markdown-rs = { version = "2.27.2", default-features = false }
+html-to-markdown-rs = { version = "2.28.0", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

data/ext/kreuzberg_rb/native/src/config/types.rs

@@ -317,10 +317,17 @@ pub fn parse_image_extraction_config(ruby: &Ruby, hash: RHash) -> Result<ImageEx
         600
     };
 
+    let inject_placeholders = if let Some(val) = get_kw(ruby, hash, "inject_placeholders") {
+        bool::try_convert(val)?
+    } else {
+        true
+    };
+
     let config = ImageExtractionConfig {
         extract_images,
         target_dpi,
         max_image_dimension,
+        inject_placeholders,
         auto_adjust_dpi,
         min_dpi,
         max_dpi,

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.4.2'
+  VERSION = '4.4.3'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.4.2"
+version = "4.4.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -23,7 +23,7 @@ criterion = { version = "0.8", features = ["html_reports"] }
 ctor = "0.6"
 getrandom = { version = "0.4.2", features = ["wasm_js"] }
 hex = "0.4.3"
-html-to-markdown-rs = { version = "2.27.2", default-features = false }
+html-to-markdown-rs = { version = "2.28.0", default-features = false }
 image = { version = "0.25.9", default-features = false }
 itertools = "0.14"
 js-sys = "0.3"
@@ -33,6 +33,7 @@ log = "0.4"
 lzma-rust2 = { version = "0.16.2" }
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
+ort = { version = "=2.0.0-rc.11", default-features = false }
 parking_lot = "0.12.5"
 pdfium-render = { package = "kreuzberg-pdfium-render", version = "4.3" }
 rayon = "1.11.0"
@@ -42,7 +43,7 @@ serde_json = { version = "1.0.149" }
 tempfile = "3.26.0"
 thiserror = "2.0.18"
 tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
-toml = "1.0.3"
+toml = "1.0.4"
 tracing = "0.1"
 wasm-bindgen = { version = "0.2", features = ["enable-interning"] }
 wasm-bindgen-futures = "0.4"

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.4.2"
+version = "4.4.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -153,7 +153,7 @@ serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149" }
 serde_yaml_ng = "0.10.0"
 jotdown = "0.9"
-toml = "1.0.3"
+toml = "1.0.4"
 mime_guess = "2.0"
 rmp-serde = "1.3"
 thiserror = "2.0.18"
@@ -169,8 +169,8 @@ zip = { version = "8.2.0", optional = true, default-features = false, features =
     "deflate-flate2",
 ] }
 mail-parser = { version = "0.11.2", optional = true }
-html-to-markdown-rs = { version = "2.27.2", default-features = false , features = [
-    "inline-images", "metadata", ], optional = true }
+html-to-markdown-rs = { version = "2.28.0", default-features = false , features = [
+    "inline-images", "metadata", "visitor", ], optional = true }
 cfb = { version = "0.14.0", optional = true }
 quick-xml = { version = "0.39.2", features = ["serialize"], optional = true }
 tar = { version = "0.4.44", optional = true }
@@ -244,7 +244,7 @@ kreuzberg-paddle-ocr = { path = "../kreuzberg-paddle-ocr", version = "4.3", opti
 reqwest = { version = "0.13.2", default-features = false , features = [
     "json", "rustls", "blocking", ], optional = true }
 # Use rustls-tls for fastembed on non-Windows platforms
-fastembed = { version = "5.11", default-features = false, features = [
+fastembed = { version = "5.12", default-features = false, features = [
     "hf-hub-rustls-tls",
     "ort-load-dynamic",
 ], optional = true }
@@ -260,7 +260,7 @@ kreuzberg-paddle-ocr = { path = "../kreuzberg-paddle-ocr", version = "4.3", opti
 reqwest = { version = "0.13.2", default-features = false , features = [
     "json", "native-tls", "blocking", ], optional = true }
 # Use native-tls for fastembed on Windows
-fastembed = { version = "5.11", default-features = false, features = [
+fastembed = { version = "5.12", default-features = false, features = [
     "hf-hub-native-tls",
     "ort-load-dynamic",
 ], optional = true }

data/vendor/kreuzberg/README.md

@@ -17,7 +17,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.4.2 Release**
+> **🚀 Version 4.4.3 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/chunking/builder.rs

@@ -35,14 +35,15 @@ pub fn build_chunk_config(max_characters: usize, overlap: usize, trim: bool) ->
 ///
 /// This function takes a collection of text segments (produced by a text splitter)
 /// and constructs Chunk objects with proper metadata, including:
-/// - Byte offsets accounting for overlap
+/// - Byte offsets derived from the chunk's position in the source text
 /// - Chunk indices and total count
 /// - Page boundary information (if provided)
 ///
 /// # Arguments
 ///
+/// * `source_text` - The original text that the chunks were split from. Chunk
+///   slices must borrow from this text (as `text-splitter` guarantees).
 /// * `text_chunks` - Iterator of text segments to convert into chunks
-/// * `overlap` - Number of characters to overlap between chunks
 /// * `page_boundaries` - Optional page boundary markers for mapping chunks to pages
 ///
 /// # Returns
@@ -53,8 +54,8 @@ pub fn build_chunk_config(max_characters: usize, overlap: usize, trim: bool) ->
 ///
 /// Returns an error if page boundary calculation fails.
 pub fn build_chunks<'a, I>(
+    source_text: &'a str,
     text_chunks: I,
-    overlap: usize,
     page_boundaries: Option<&[PageBoundary]>,
 ) -> Result<Vec<Chunk>>
 where
@@ -62,86 +63,37 @@ where
 {
     let chunks_vec: Vec<&str> = text_chunks.into_iter().collect();
     let total_chunks = chunks_vec.len();
-    let mut byte_offset = 0;
+    let source_start = source_text.as_ptr() as usize;
     let mut chunks = Vec::with_capacity(total_chunks);
 
     for (index, chunk_text) in chunks_vec.into_iter().enumerate() {
-        let chunk = build_single_chunk(
-            chunk_text,
-            index,
-            total_chunks,
-            &mut byte_offset,
-            overlap,
-            page_boundaries,
-        )?;
-        chunks.push(chunk);
+        let byte_start = chunk_text.as_ptr() as usize - source_start;
+        let byte_end = byte_start + chunk_text.len();
+
+        let (first_page, last_page) = if let Some(boundaries) = page_boundaries {
+            calculate_page_range(byte_start, byte_end, boundaries)?
+        } else {
+            (None, None)
+        };
+
+        chunks.push(Chunk {
+            content: chunk_text.to_string(),
+            embedding: None,
+            metadata: ChunkMetadata {
+                byte_start,
+                byte_end,
+                token_count: None,
+                chunk_index: index,
+                total_chunks,
+                first_page,
+                last_page,
+            },
+        });
     }
 
     Ok(chunks)
 }
 
-/// Build a single chunk with metadata.
-///
-/// # Arguments
-///
-/// * `chunk_text` - The text content for this chunk
-/// * `index` - Zero-based index of this chunk
-/// * `total_chunks` - Total number of chunks in the collection
-/// * `byte_offset` - Mutable reference to current byte offset (will be updated)
-/// * `overlap` - Number of characters to overlap between chunks
-/// * `page_boundaries` - Optional page boundary markers
-///
-/// # Returns
-///
-/// A complete Chunk object with all metadata filled in.
-///
-/// # Errors
-///
-/// Returns an error if page boundary calculation fails.
-fn build_single_chunk(
-    chunk_text: &str,
-    index: usize,
-    total_chunks: usize,
-    byte_offset: &mut usize,
-    overlap: usize,
-    page_boundaries: Option<&[PageBoundary]>,
-) -> Result<Chunk> {
-    let byte_start = *byte_offset;
-    let chunk_length = chunk_text.len();
-    let byte_end = byte_start + chunk_length;
-
-    // Calculate overlap for next chunk (not applicable to last chunk)
-    let overlap_chars = if index < total_chunks - 1 {
-        overlap.min(chunk_length)
-    } else {
-        0
-    };
-
-    // Update offset for next chunk, accounting for overlap
-    *byte_offset = byte_end - overlap_chars;
-
-    // Calculate page range if boundaries are provided
-    let (first_page, last_page) = if let Some(boundaries) = page_boundaries {
-        calculate_page_range(byte_start, byte_end, boundaries)?
-    } else {
-        (None, None)
-    };
-
-    Ok(Chunk {
-        content: chunk_text.to_string(),
-        embedding: None,
-        metadata: ChunkMetadata {
-            byte_start,
-            byte_end,
-            token_count: None,
-            chunk_index: index,
-            total_chunks,
-            first_page,
-            last_page,
-        },
-    })
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -162,15 +114,16 @@ mod tests {
 
     #[test]
     fn test_build_chunks_empty() {
+        let source = "";
         let text_chunks: Vec<&str> = vec![];
-        let result = build_chunks(text_chunks, 5, None).unwrap();
+        let result = build_chunks(source, text_chunks, None).unwrap();
         assert_eq!(result.len(), 0);
     }
 
     #[test]
     fn test_build_chunks_single() {
-        let text_chunks = vec!["Single chunk"];
-        let result = build_chunks(text_chunks, 5, None).unwrap();
+        let source = "Single chunk";
+        let result = build_chunks(source, vec![source], None).unwrap();
         assert_eq!(result.len(), 1);
         assert_eq!(result[0].content, "Single chunk");
         assert_eq!(result[0].metadata.chunk_index, 0);
@@ -179,29 +132,10 @@ mod tests {
         assert_eq!(result[0].metadata.byte_end, 12);
     }
 
-    #[test]
-    fn test_build_chunks_multiple_with_overlap() {
-        let text_chunks = vec!["First chunk here", "Second chunk here", "Third chunk here"];
-        let overlap = 5;
-        let result = build_chunks(text_chunks, overlap, None).unwrap();
-
-        assert_eq!(result.len(), 3);
-
-        // First chunk
-        assert_eq!(result[0].content, "First chunk here");
-        assert_eq!(result[0].metadata.byte_start, 0);
-        assert_eq!(result[0].metadata.byte_end, 16);
-
-        // Second chunk should start before first ends (overlap)
-        assert!(result[1].metadata.byte_start < result[0].metadata.byte_end);
-
-        // Third chunk should start before second ends (overlap)
-        assert!(result[2].metadata.byte_start < result[1].metadata.byte_end);
-    }
-
     #[test]
     fn test_build_chunks_with_page_boundaries() {
-        let text_chunks = vec!["First chunk", "Second chunk"];
+        let source = "First chunkSecond chunk";
+        let text_chunks = vec![&source[0..11], &source[11..23]];
         let boundaries = vec![
             PageBoundary {
                 byte_start: 0,
@@ -215,7 +149,7 @@ mod tests {
             },
         ];
 
-        let result = build_chunks(text_chunks, 0, Some(&boundaries)).unwrap();
+        let result = build_chunks(source, text_chunks, Some(&boundaries)).unwrap();
 
         assert_eq!(result.len(), 2);
         assert_eq!(result[0].metadata.first_page, Some(1));
@@ -223,65 +157,32 @@ mod tests {
     }
 
     #[test]
-    fn test_build_chunks_offset_tracking() {
-        let text_chunks = vec!["AAAAA", "BBBBB", "CCCCC"];
-        let overlap = 2;
-        let result = build_chunks(text_chunks, overlap, None).unwrap();
+    fn test_build_chunks_offset_from_source() {
+        let source = "AAAAABBBBBCCCCC";
+        // Overlapping slices from source
+        let text_chunks = vec![&source[0..5], &source[3..8], &source[6..11]];
+        let result = build_chunks(source, text_chunks, None).unwrap();
 
         assert_eq!(result.len(), 3);
 
-        // First chunk: 0-5
         assert_eq!(result[0].metadata.byte_start, 0);
         assert_eq!(result[0].metadata.byte_end, 5);
 
-        // Second chunk: 3-8 (overlap of 2)
         assert_eq!(result[1].metadata.byte_start, 3);
         assert_eq!(result[1].metadata.byte_end, 8);
 
-        // Third chunk: 6-11 (overlap of 2, but last chunk so no further adjustment)
         assert_eq!(result[2].metadata.byte_start, 6);
         assert_eq!(result[2].metadata.byte_end, 11);
     }
 
-    #[test]
-    fn test_build_single_chunk_metadata() {
-        let mut offset = 0;
-        let chunk = build_single_chunk("Test content", 0, 1, &mut offset, 5, None).unwrap();
-
-        assert_eq!(chunk.content, "Test content");
-        assert_eq!(chunk.metadata.byte_start, 0);
-        assert_eq!(chunk.metadata.byte_end, 12);
-        assert_eq!(chunk.metadata.chunk_index, 0);
-        assert_eq!(chunk.metadata.total_chunks, 1);
-        assert_eq!(chunk.metadata.first_page, None);
-        assert_eq!(chunk.metadata.last_page, None);
-    }
-
-    #[test]
-    fn test_build_single_chunk_with_overlap() {
-        let mut offset = 0;
-
-        // First chunk
-        let chunk1 = build_single_chunk("0123456789", 0, 2, &mut offset, 3, None).unwrap();
-        assert_eq!(chunk1.metadata.byte_start, 0);
-        assert_eq!(chunk1.metadata.byte_end, 10);
-        assert_eq!(offset, 7); // 10 - 3 (overlap)
-
-        // Second chunk
-        let chunk2 = build_single_chunk("ABCDEFGHIJ", 1, 2, &mut offset, 3, None).unwrap();
-        assert_eq!(chunk2.metadata.byte_start, 7);
-        assert_eq!(chunk2.metadata.byte_end, 17);
-        assert_eq!(offset, 17); // Last chunk, no overlap subtracted
-    }
-
     #[test]
     fn test_build_chunks_no_overlap() {
-        let text_chunks = vec!["AAAAA", "BBBBB", "CCCCC"];
-        let result = build_chunks(text_chunks, 0, None).unwrap();
+        let source = "AAAAABBBBBCCCCC";
+        let text_chunks = vec![&source[0..5], &source[5..10], &source[10..15]];
+        let result = build_chunks(source, text_chunks, None).unwrap();
 
         assert_eq!(result.len(), 3);
 
-        // Chunks should be contiguous with no overlap
         assert_eq!(result[0].metadata.byte_start, 0);
         assert_eq!(result[0].metadata.byte_end, 5);