kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/src/core/pipeline.rs

@@ -47,6 +47,24 @@ pub async fn run_pipeline(mut result: ExtractionResult, config: &ExtractionConfi
             let _ = crate::keywords::ensure_initialized();
         }
 
+        #[cfg(feature = "language-detection")]
+        {
+            let _ = crate::language_detection::ensure_initialized();
+        }
+
+        #[cfg(feature = "chunking")]
+        {
+            let _ = crate::chunking::ensure_initialized();
+        }
+
+        #[cfg(feature = "quality")]
+        {
+            let registry = crate::plugins::registry::get_post_processor_registry();
+            if let Ok(mut reg) = registry.write() {
+                let _ = reg.register(std::sync::Arc::new(crate::text::QualityProcessor), 30);
+            }
+        }
+
         let processor_registry = crate::plugins::registry::get_post_processor_registry();
 
         for stage in [ProcessingStage::Early, ProcessingStage::Middle, ProcessingStage::Late] {
@@ -130,7 +148,9 @@ pub async fn run_pipeline(mut result: ExtractionResult, config: &ExtractionConfi
             chunker_type: crate::chunking::ChunkerType::Text,
         };
 
-        match crate::chunking::chunk_text(&result.content, &chunk_config) {
+        let page_boundaries = result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref());
+
+        match crate::chunking::chunk_text(&result.content, &chunk_config, page_boundaries) {
             Ok(chunking_result) => {
                 result.chunks = Some(chunking_result.chunks);
 
@@ -228,6 +248,157 @@ pub async fn run_pipeline(mut result: ExtractionResult, config: &ExtractionConfi
     Ok(result)
 }
 
+/// Run the post-processing pipeline synchronously (WASM-compatible version).
+///
+/// This is a synchronous implementation for WASM and non-async contexts.
+/// It performs a subset of the full async pipeline, excluding async post-processors
+/// and validators.
+///
+/// # Arguments
+///
+/// * `result` - The extraction result to process
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// The processed extraction result.
+///
+/// # Notes
+///
+/// This function is only available when the `tokio-runtime` feature is disabled.
+/// It handles:
+/// - Quality processing (if enabled)
+/// - Chunking (if enabled)
+/// - Language detection (if enabled)
+///
+/// It does NOT handle:
+/// - Async post-processors
+/// - Async validators
+#[cfg(not(feature = "tokio-runtime"))]
+pub fn run_pipeline_sync(mut result: ExtractionResult, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    // Quality processing
+    #[cfg(feature = "quality")]
+    if config.enable_quality_processing {
+        let quality_score = crate::text::quality::calculate_quality_score(
+            &result.content,
+            Some(
+                &result
+                    .metadata
+                    .additional
+                    .iter()
+                    .map(|(k, v)| (k.clone(), v.to_string()))
+                    .collect(),
+            ),
+        );
+        result.metadata.additional.insert(
+            "quality_score".to_string(),
+            serde_json::Value::Number(
+                serde_json::Number::from_f64(quality_score).unwrap_or(serde_json::Number::from(0)),
+            ),
+        );
+    }
+
+    #[cfg(not(feature = "quality"))]
+    if config.enable_quality_processing {
+        result.metadata.additional.insert(
+            "quality_processing_error".to_string(),
+            serde_json::Value::String("Quality processing feature not enabled".to_string()),
+        );
+    }
+
+    // Chunking
+    #[cfg(feature = "chunking")]
+    if let Some(ref chunking_config) = config.chunking {
+        let chunk_config = crate::chunking::ChunkingConfig {
+            max_characters: chunking_config.max_chars,
+            overlap: chunking_config.max_overlap,
+            trim: true,
+            chunker_type: crate::chunking::ChunkerType::Text,
+        };
+
+        match crate::chunking::chunk_text(&result.content, &chunk_config, None) {
+            Ok(chunking_result) => {
+                result.chunks = Some(chunking_result.chunks);
+
+                if let Some(ref chunks) = result.chunks {
+                    result.metadata.additional.insert(
+                        "chunk_count".to_string(),
+                        serde_json::Value::Number(serde_json::Number::from(chunks.len())),
+                    );
+                }
+
+                #[cfg(feature = "embeddings")]
+                if let Some(ref embedding_config) = chunking_config.embedding
+                    && let Some(ref mut chunks) = result.chunks
+                {
+                    match crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config) {
+                        Ok(()) => {
+                            result
+                                .metadata
+                                .additional
+                                .insert("embeddings_generated".to_string(), serde_json::Value::Bool(true));
+                        }
+                        Err(e) => {
+                            result
+                                .metadata
+                                .additional
+                                .insert("embedding_error".to_string(), serde_json::Value::String(e.to_string()));
+                        }
+                    }
+                }
+
+                #[cfg(not(feature = "embeddings"))]
+                if chunking_config.embedding.is_some() {
+                    result.metadata.additional.insert(
+                        "embedding_error".to_string(),
+                        serde_json::Value::String("Embeddings feature not enabled".to_string()),
+                    );
+                }
+            }
+            Err(e) => {
+                result
+                    .metadata
+                    .additional
+                    .insert("chunking_error".to_string(), serde_json::Value::String(e.to_string()));
+            }
+        }
+    }
+
+    #[cfg(not(feature = "chunking"))]
+    if config.chunking.is_some() {
+        result.metadata.additional.insert(
+            "chunking_error".to_string(),
+            serde_json::Value::String("Chunking feature not enabled".to_string()),
+        );
+    }
+
+    // Language detection
+    #[cfg(feature = "language-detection")]
+    if let Some(ref lang_config) = config.language_detection {
+        match crate::language_detection::detect_languages(&result.content, lang_config) {
+            Ok(detected) => {
+                result.detected_languages = detected;
+            }
+            Err(e) => {
+                result.metadata.additional.insert(
+                    "language_detection_error".to_string(),
+                    serde_json::Value::String(e.to_string()),
+                );
+            }
+        }
+    }
+
+    #[cfg(not(feature = "language-detection"))]
+    if config.language_detection.is_some() {
+        result.metadata.additional.insert(
+            "language_detection_error".to_string(),
+            serde_json::Value::String("Language detection feature not enabled".to_string()),
+        );
+    }
+
+    Ok(result)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -235,6 +406,7 @@ mod tests {
     use lazy_static::lazy_static;
 
     const VALIDATION_MARKER_KEY: &str = "registry_validation_marker";
+    #[cfg(feature = "quality")]
     const QUALITY_VALIDATION_MARKER: &str = "quality_validation_test";
     const POSTPROCESSOR_VALIDATION_MARKER: &str = "postprocessor_validation_test";
     const ORDER_VALIDATION_MARKER: &str = "order_validation_test";
@@ -253,6 +425,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         result.metadata.additional.insert(
             VALIDATION_MARKER_KEY.to_string(),
@@ -275,6 +448,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig {
             enable_quality_processing: true,
@@ -295,6 +469,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig {
             enable_quality_processing: false,
@@ -316,6 +491,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig {
             chunking: Some(crate::ChunkingConfig {
@@ -343,6 +519,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig {
             chunking: None,
@@ -367,6 +544,7 @@ mod tests {
                 additional,
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -403,6 +581,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig::default();
 
@@ -432,6 +611,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig::default();
 
@@ -452,6 +632,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         let config = ExtractionConfig {
             enable_quality_processing: true,
@@ -488,6 +669,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         #[cfg(feature = "keywords-yake")]
@@ -529,6 +711,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig {
@@ -564,6 +747,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         #[cfg(feature = "keywords-yake")]
@@ -693,6 +877,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         result.metadata.additional.insert(
             VALIDATION_MARKER_KEY.to_string(),
@@ -778,6 +963,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
         result.metadata.additional.insert(
             VALIDATION_MARKER_KEY.to_string(),
@@ -969,6 +1155,7 @@ Natural language processing enables computers to understand human language.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();

data/vendor/kreuzberg/src/extraction/docx.rs

@@ -2,8 +2,13 @@
 //!
 //! This module provides high-performance text extraction from DOCX files using the docx-lite
 //! library, which uses streaming XML parsing for efficiency.
+//!
+//! Page break detection is best-effort, detecting only explicit page breaks (`<w:br w:type="page"/>`)
+//! in the document XML. This does not account for automatic pagination based on content reflowing.
 
 use crate::error::{KreuzbergError, Result};
+use crate::types::PageBoundary;
+use std::io::Cursor;
 
 /// Extract text from DOCX bytes using docx-lite.
 ///
@@ -22,6 +27,163 @@ pub fn extract_text(bytes: &[u8]) -> Result<String> {
         .map_err(|e| KreuzbergError::parsing(format!("DOCX text extraction failed: {}", e)))
 }
 
+/// Extract text and page boundaries from DOCX bytes.
+///
+/// Detects explicit page breaks (`<w:br w:type="page"/>`) in the document XML and maps them to
+/// character offsets in the extracted text. This is a best-effort approach that only detects
+/// explicit page breaks, not automatic pagination.
+///
+/// # Arguments
+/// * `bytes` - The DOCX file contents as bytes
+///
+/// # Returns
+/// * `Ok((String, Option<Vec<PageBoundary>>))` - Extracted text and optional page boundaries
+/// * `Err(KreuzbergError)` - If extraction fails
+///
+/// # Limitations
+/// - Only detects explicit page breaks, not reflowed content
+/// - Page numbers are estimates, not guaranteed accurate
+/// - Word's pagination may differ from detected breaks
+/// - No page dimensions available (would require layout engine)
+///
+/// # Performance
+/// Performs two passes: one with docx-lite for text extraction and one for page break detection.
+pub fn extract_text_with_page_breaks(bytes: &[u8]) -> Result<(String, Option<Vec<PageBoundary>>)> {
+    let text = extract_text(bytes)?;
+
+    let page_breaks = detect_page_breaks(bytes)?;
+
+    if page_breaks.is_empty() {
+        return Ok((text, None));
+    }
+
+    let boundaries = map_page_breaks_to_boundaries(&text, page_breaks)?;
+
+    Ok((text, Some(boundaries)))
+}
+
+/// Detect explicit page break positions in document.xml and extract full text with page boundaries.
+///
+/// This is a convenience function for the extractor that combines text extraction with page
+/// break detection. It returns the extracted text along with page boundaries.
+///
+/// # Arguments
+/// * `bytes` - The DOCX file contents (ZIP archive)
+///
+/// # Returns
+/// * `Ok(Option<Vec<PageBoundary>>)` - Optional page boundaries
+/// * `Err(KreuzbergError)` - If extraction fails
+///
+/// # Limitations
+/// - Only detects explicit page breaks, not reflowed content
+/// - Page numbers are estimates based on detected breaks
+pub fn detect_page_breaks_from_docx(bytes: &[u8]) -> Result<Option<Vec<PageBoundary>>> {
+    match extract_text_with_page_breaks(bytes) {
+        Ok((_, boundaries)) => Ok(boundaries),
+        Err(e) => {
+            tracing::debug!("Page break detection failed: {}", e);
+            Ok(None)
+        }
+    }
+}
+
+/// Detect explicit page break positions in document.xml.
+///
+/// Returns a vector of byte offsets within the document.xml content where page breaks occur.
+/// These offsets will later be mapped to character positions in the extracted text.
+///
+/// # Arguments
+/// * `bytes` - The DOCX file contents (ZIP archive)
+///
+/// # Returns
+/// * `Ok(Vec<usize>)` - Vector of detected page break byte offsets (empty if none found)
+/// * `Err(KreuzbergError)` - If ZIP/XML parsing fails
+fn detect_page_breaks(bytes: &[u8]) -> Result<Vec<usize>> {
+    use zip::ZipArchive;
+
+    let cursor = Cursor::new(bytes);
+    let mut archive =
+        ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to open DOCX as ZIP: {}", e)))?;
+
+    let document_xml = match archive.by_name("word/document.xml") {
+        Ok(mut file) => {
+            let mut content = String::new();
+            std::io::Read::read_to_string(&mut file, &mut content)
+                .map_err(|e| KreuzbergError::parsing(format!("Failed to read document.xml: {}", e)))?;
+            content
+        }
+        Err(_) => return Ok(Vec::new()),
+    };
+
+    let mut breaks = Vec::new();
+    let search_pattern = r#"<w:br w:type="page"/>"#;
+
+    for (idx, _) in document_xml.match_indices(search_pattern) {
+        breaks.push(idx);
+    }
+
+    Ok(breaks)
+}
+
+/// Map detected page break positions to byte boundaries in extracted text.
+///
+/// Since we don't have a precise mapping between document.xml byte positions and final text
+/// character positions, we use a heuristic: divide the text roughly equally between detected breaks.
+/// This is best-effort and may not perfectly match Word's pagination.
+///
+/// # LIMITATION
+/// This is a best-effort heuristic that distributes content evenly across detected page breaks.
+/// It does not account for actual page layout, varying page sizes, or Word's pagination logic.
+/// Use with caution. The function correctly handles multibyte UTF-8 characters (emoji, CJK, etc.)
+/// by working with character indices rather than byte indices.
+///
+/// # Arguments
+/// * `text` - The extracted document text
+/// * `page_breaks` - Vector of detected page break positions (unused, but kept for extension)
+///
+/// # Returns
+/// * `Ok(Vec<PageBoundary>)` - Byte boundaries for each page
+fn map_page_breaks_to_boundaries(text: &str, page_breaks: Vec<usize>) -> Result<Vec<PageBoundary>> {
+    if page_breaks.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let page_count = page_breaks.len() + 1;
+
+    let char_count = text.chars().count();
+    let chars_per_page = char_count / page_count;
+
+    let mut boundaries = Vec::new();
+    let mut byte_offset = 0;
+
+    for page_num in 1..=page_count {
+        let start = byte_offset;
+
+        let end = if page_num == page_count {
+            text.len()
+        } else {
+            let remaining = &text[byte_offset..];
+            let chars_to_skip = chars_per_page;
+            byte_offset
+                + remaining
+                    .chars()
+                    .take(chars_to_skip)
+                    .map(|c| c.len_utf8())
+                    .sum::<usize>()
+        };
+
+        byte_offset = end;
+
+        boundaries.push(PageBoundary {
+            byte_start: start,
+            byte_end: end,
+            page_number: page_num,
+        });
+    }
+
+    Ok(boundaries)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -37,4 +199,200 @@ mod tests {
         let result = extract_text(b"not a docx file");
         assert!(result.is_err());
     }
+
+    #[test]
+    fn test_map_page_breaks_to_boundaries_empty() {
+        let result = map_page_breaks_to_boundaries("test text", Vec::new()).unwrap();
+        assert!(result.is_empty());
+    }
+
+    #[test]
+    fn test_map_page_breaks_to_boundaries_single_break() {
+        let text = "Page 1 content here with some text.Page 2 content here with more text.";
+        let breaks = vec![0];
+
+        let result = map_page_breaks_to_boundaries(text, breaks).unwrap();
+
+        assert_eq!(result.len(), 2);
+        assert_eq!(result[0].page_number, 1);
+        assert_eq!(result[0].byte_start, 0);
+        assert!(result[0].byte_end > 0);
+        assert!(result[0].byte_end < text.len());
+
+        assert_eq!(result[1].page_number, 2);
+        assert_eq!(result[1].byte_start, result[0].byte_end);
+        assert_eq!(result[1].byte_end, text.len());
+    }
+
+    #[test]
+    fn test_map_page_breaks_to_boundaries_multiple_breaks() {
+        let text = "A".repeat(300);
+        let breaks = vec![0, 0, 0];
+
+        let result = map_page_breaks_to_boundaries(&text, breaks).unwrap();
+
+        assert_eq!(result.len(), 4);
+        assert_eq!(result[0].page_number, 1);
+        assert_eq!(result[3].page_number, 4);
+        assert_eq!(result[3].byte_end, text.len());
+
+        for i in 0..result.len() - 1 {
+            assert_eq!(result[i].byte_end, result[i + 1].byte_start);
+        }
+    }
+
+    #[test]
+    fn test_map_page_breaks_to_boundaries_utf8_boundary() {
+        let text = "Hello world! こんにちは世界！ More text here.";
+        let breaks = vec![0];
+
+        let result = map_page_breaks_to_boundaries(text, breaks).unwrap();
+
+        assert_eq!(result.len(), 2);
+        assert!(text.is_char_boundary(result[0].byte_start));
+        assert!(text.is_char_boundary(result[0].byte_end));
+        assert!(text.is_char_boundary(result[1].byte_start));
+        assert!(text.is_char_boundary(result[1].byte_end));
+    }
+
+    #[test]
+    fn test_docx_page_breaks_with_emoji() {
+        let text = "Hello 😀 World 🌍 Foo 🎉 Bar";
+        let breaks = vec![0, 0];
+
+        let result = map_page_breaks_to_boundaries(text, breaks).unwrap();
+
+        assert_eq!(result.len(), 3);
+        assert_eq!(result[0].page_number, 1);
+        assert_eq!(result[1].page_number, 2);
+        assert_eq!(result[2].page_number, 3);
+
+        for boundary in &result {
+            assert!(
+                text.is_char_boundary(boundary.byte_start),
+                "byte_start {} is not a valid UTF-8 boundary",
+                boundary.byte_start
+            );
+            assert!(
+                text.is_char_boundary(boundary.byte_end),
+                "byte_end {} is not a valid UTF-8 boundary",
+                boundary.byte_end
+            );
+        }
+
+        assert_eq!(result[0].byte_start, 0);
+        assert_eq!(result[0].byte_end, result[1].byte_start);
+        assert_eq!(result[1].byte_end, result[2].byte_start);
+        assert_eq!(result[2].byte_end, text.len());
+
+        let reconstructed = format!(
+            "{}{}{}",
+            &text[result[0].byte_start..result[0].byte_end],
+            &text[result[1].byte_start..result[1].byte_end],
+            &text[result[2].byte_start..result[2].byte_end]
+        );
+        assert_eq!(reconstructed, text);
+    }
+
+    #[test]
+    fn test_docx_page_breaks_with_cjk() {
+        let text = "你好世界你好世界你好世界你好世界";
+        let breaks = vec![0];
+
+        let result = map_page_breaks_to_boundaries(text, breaks).unwrap();
+
+        assert_eq!(result.len(), 2);
+        assert_eq!(result[0].page_number, 1);
+        assert_eq!(result[1].page_number, 2);
+
+        for boundary in &result {
+            assert!(
+                text.is_char_boundary(boundary.byte_start),
+                "byte_start {} is not a valid UTF-8 boundary",
+                boundary.byte_start
+            );
+            assert!(
+                text.is_char_boundary(boundary.byte_end),
+                "byte_end {} is not a valid UTF-8 boundary",
+                boundary.byte_end
+            );
+        }
+
+        assert_eq!(result[0].byte_start, 0);
+        assert_eq!(result[0].byte_end, result[1].byte_start);
+        assert_eq!(result[1].byte_end, text.len());
+
+        let reconstructed = format!(
+            "{}{}",
+            &text[result[0].byte_start..result[0].byte_end],
+            &text[result[1].byte_start..result[1].byte_end]
+        );
+        assert_eq!(reconstructed, text);
+    }
+
+    #[test]
+    fn test_docx_page_breaks_multibyte_utf8() {
+        let text = "ASCII 😀 中文 hello 🎉 world 日本語";
+        let breaks = vec![0, 0];
+
+        let result = map_page_breaks_to_boundaries(text, breaks).unwrap();
+
+        assert_eq!(result.len(), 3);
+
+        for boundary in &result {
+            assert!(
+                text.is_char_boundary(boundary.byte_start),
+                "byte_start {} is not a valid UTF-8 boundary",
+                boundary.byte_start
+            );
+            assert!(
+                text.is_char_boundary(boundary.byte_end),
+                "byte_end {} is not a valid UTF-8 boundary",
+                boundary.byte_end
+            );
+        }
+
+        assert_eq!(result[0].byte_start, 0);
+        for i in 0..result.len() - 1 {
+            assert_eq!(
+                result[i].byte_end,
+                result[i + 1].byte_start,
+                "Gap or overlap between page {} and {}",
+                i + 1,
+                i + 2
+            );
+        }
+        assert_eq!(
+            result[result.len() - 1].byte_end,
+            text.len(),
+            "Last page does not end at text boundary"
+        );
+
+        let mut reconstructed = String::new();
+        for boundary in &result {
+            reconstructed.push_str(&text[boundary.byte_start..boundary.byte_end]);
+        }
+        assert_eq!(reconstructed, text);
+    }
+
+    #[test]
+    fn test_detect_page_breaks_no_feature() {
+        let result = detect_page_breaks(b"invalid");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_extract_text_with_page_breaks_no_breaks() {
+        let docx_path =
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/documents/lorem_ipsum.docx");
+        if let Ok(bytes) = std::fs::read(docx_path) {
+            let result = extract_text_with_page_breaks(&bytes);
+            if let Ok((text, boundaries)) = result {
+                assert!(!text.is_empty());
+                if let Some(b) = boundaries {
+                    assert!(!b.is_empty());
+                }
+            }
+        }
+    }
 }