kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/src/pdf/text.rs

@@ -1,14 +1,19 @@
 use super::error::{PdfError, Result};
+use crate::core::config::PageConfig;
+use crate::types::{PageBoundary, PageContent};
 use pdfium_render::prelude::*;
 
+/// Result type for PDF text extraction with optional page tracking.
+#[allow(dead_code)]
+type PdfTextExtractionResult = (String, Option<Vec<PageBoundary>>, Option<Vec<PageContent>>);
+
 pub struct PdfTextExtractor {
     pdfium: Pdfium,
 }
 
 impl PdfTextExtractor {
     pub fn new() -> Result<Self> {
-        let binding = Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path("./"))
-            .or_else(|_| Pdfium::bind_to_system_library())
+        let binding = Pdfium::bind_to_system_library()
             .map_err(|e| PdfError::TextExtractionFailed(format!("Failed to initialize Pdfium: {}", e)))?;
 
         let pdfium = Pdfium::new(binding);
@@ -31,7 +36,8 @@ impl PdfTextExtractor {
             }
         })?;
 
-        extract_text_from_pdf_document(&document)
+        let (content, _, _) = extract_text_from_pdf_document(&document, None)?;
+        Ok(content)
     }
 
     pub fn extract_text_with_passwords(&self, pdf_bytes: &[u8], passwords: &[&str]) -> Result<String> {
@@ -89,28 +95,104 @@ pub fn extract_text_from_pdf_with_passwords(pdf_bytes: &[u8], passwords: &[&str]
     extractor.extract_text_with_passwords(pdf_bytes, passwords)
 }
 
-pub fn extract_text_from_pdf_document(document: &PdfDocument<'_>) -> Result<String> {
+/// Extract text from PDF document with optional page boundary tracking.
+///
+/// # Arguments
+///
+/// * `document` - The PDF document to extract text from
+/// * `page_config` - Optional page configuration for boundary tracking and page markers
+///
+/// # Returns
+///
+/// A tuple containing:
+/// - The extracted text content (String)
+/// - Optional page boundaries when page tracking is enabled (Vec<PageBoundary>)
+/// - Optional per-page content when extract_pages is enabled (Vec<PageContent>)
+///
+/// # Implementation Details
+///
+/// When page_config is None, returns fast path with (content, None, None).
+/// When page_config is Some, tracks byte offsets using .len() for O(1) performance (UTF-8 valid boundaries).
+pub fn extract_text_from_pdf_document(
+    document: &PdfDocument<'_>,
+    page_config: Option<&PageConfig>,
+) -> Result<PdfTextExtractionResult> {
     let page_count = document.pages().len() as usize;
 
+    if page_config.is_none() {
+        let estimated_size = page_count * 2048;
+        let mut content = String::with_capacity(estimated_size);
+
+        for page in document.pages().iter() {
+            let text = page
+                .text()
+                .map_err(|e| PdfError::TextExtractionFailed(format!("Page text extraction failed: {}", e)))?;
+
+            let page_text = text.all();
+
+            if !content.is_empty() {
+                content.push_str("\n\n");
+            }
+            content.push_str(&page_text);
+        }
+
+        content.shrink_to_fit();
+        return Ok((content, None, None));
+    }
+
+    let config = page_config.unwrap();
     let estimated_size = page_count * 2048;
     let mut content = String::with_capacity(estimated_size);
+    let mut boundaries = Vec::with_capacity(page_count);
+    let mut page_contents = if config.extract_pages {
+        Some(Vec::with_capacity(page_count))
+    } else {
+        None
+    };
+
+    for (page_idx, page) in document.pages().iter().enumerate() {
+        let page_number = page_idx + 1;
 
-    for page in document.pages().iter() {
         let text = page
             .text()
             .map_err(|e| PdfError::TextExtractionFailed(format!("Page text extraction failed: {}", e)))?;
 
         let page_text = text.all();
 
-        if !content.is_empty() {
+        if page_number > 1 && config.insert_page_markers {
+            let marker = config.marker_format.replace("{page_num}", &page_number.to_string());
+            content.push_str(&marker);
+        }
+
+        if page_number > 1 && !config.insert_page_markers && !content.is_empty() {
             content.push_str("\n\n");
         }
+
+        let byte_start = content.len();
+
         content.push_str(&page_text);
+
+        let byte_end = content.len();
+
+        boundaries.push(PageBoundary {
+            byte_start,
+            byte_end,
+            page_number,
+        });
+
+        if let Some(ref mut pages) = page_contents {
+            pages.push(PageContent {
+                page_number,
+                content: page_text,
+                tables: Vec::new(),
+                images: Vec::new(),
+            });
+        }
     }
 
     content.shrink_to_fit();
 
-    Ok(content)
+    Ok((content, Some(boundaries), page_contents))
 }
 
 #[cfg(test)]

data/vendor/kreuzberg/src/plugins/extractor.rs

@@ -10,6 +10,9 @@ use async_trait::async_trait;
 use std::path::Path;
 use std::sync::Arc;
 
+#[cfg(not(feature = "tokio-runtime"))]
+use crate::KreuzbergError;
+
 /// Trait for document extractor plugins.
 ///
 /// Implement this trait to add support for new document formats or to override
@@ -61,6 +64,7 @@ use std::sync::Arc;
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
+///             pages: None,
 ///         })
 ///     }
 ///
@@ -139,6 +143,7 @@ pub trait DocumentExtractor: Plugin {
     ///         detected_languages: None,
     ///         chunks: None,
     ///         images: None,
+    ///         pages: None,
     ///     })
     /// }
     /// # }
@@ -209,14 +214,27 @@ pub trait DocumentExtractor: Plugin {
     ///         detected_languages: None,
     ///         chunks: None,
     ///         images: None,
+    ///         pages: None,
     ///     })
     /// }
     /// # }
     /// ```
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
-        use crate::core::io;
-        let bytes = io::read_file_async(path).await?;
-        self.extract_bytes(&bytes, mime_type, config).await
+        #[cfg(feature = "tokio-runtime")]
+        {
+            use crate::core::io;
+            let bytes = io::read_file_async(path).await?;
+            self.extract_bytes(&bytes, mime_type, config).await
+        }
+        #[cfg(not(feature = "tokio-runtime"))]
+        {
+            let _ = (path, mime_type, config);
+            // For WASM and non-tokio environments, file extraction is not supported
+            // through the default implementation. Implementations must provide their own.
+            Err(KreuzbergError::Other(
+                "File-based extraction requires the tokio-runtime feature".to_string(),
+            ))
+        }
     }
 
     /// Get the list of MIME types supported by this extractor.
@@ -359,6 +377,14 @@ pub trait DocumentExtractor: Plugin {
     fn can_handle(&self, _path: &Path, _mime_type: &str) -> bool {
         true
     }
+
+    /// Attempt to get a reference to this extractor as a SyncExtractor.
+    ///
+    /// Returns None if the extractor doesn't support synchronous extraction.
+    /// This is used for WASM and other sync-only environments.
+    fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
+        None
+    }
 }
 
 /// Register a document extractor with the global registry.
@@ -412,6 +438,7 @@ pub trait DocumentExtractor: Plugin {
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
+///             pages: None,
 ///         })
 ///     }
 ///
@@ -577,6 +604,7 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
 
@@ -749,6 +777,7 @@ mod tests {
                     detected_languages: None,
                     chunks: None,
                     images: None,
+                    pages: None,
                 })
             }
 
@@ -953,6 +982,7 @@ mod tests {
                     detected_languages: None,
                     chunks: None,
                     images: None,
+                    pages: None,
                 })
             }
 
@@ -998,6 +1028,7 @@ mod tests {
                     detected_languages: None,
                     chunks: None,
                     images: None,
+                    pages: None,
                 })
             }
 

data/vendor/kreuzberg/src/plugins/mod.rs

@@ -47,6 +47,7 @@
 //! #             detected_languages: None,
 //! #             chunks: None,
 //! #             images: None,
+//! #             pages: None,
 //! #         })
 //! #     }
 //! #     async fn extract_file(&self, _: &std::path::Path, _: &str, _: &kreuzberg::ExtractionConfig)
@@ -59,6 +60,7 @@
 //! #             detected_languages: None,
 //! #             chunks: None,
 //! #             images: None,
+//! #             pages: None,
 //! #         })
 //! #     }
 //! #     fn supported_mime_types(&self) -> &[&str] { &[] }
@@ -120,6 +122,7 @@
 //!             detected_languages: None,
 //!             chunks: None,
 //!             images: None,
+//!             pages: None,
 //!         })
 //!     }
 //!

data/vendor/kreuzberg/src/plugins/ocr.rs

@@ -10,6 +10,9 @@ use async_trait::async_trait;
 use std::path::Path;
 use std::sync::Arc;
 
+#[cfg(not(feature = "tokio-runtime"))]
+use crate::KreuzbergError;
+
 /// OCR backend types.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum OcrBackendType {
@@ -64,6 +67,7 @@ pub enum OcrBackendType {
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
+///             pages: None,
 ///         })
 ///     }
 ///
@@ -142,6 +146,7 @@ pub trait OcrBackend: Plugin {
     ///         detected_languages: None,
     ///         chunks: None,
     ///         images: None,
+    ///         pages: None,
     ///     })
     /// }
     /// # }
@@ -162,9 +167,21 @@ pub trait OcrBackend: Plugin {
     ///
     /// Same as `process_image`, plus file I/O errors.
     async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
-        use crate::core::io;
-        let bytes = io::read_file_async(path).await?;
-        self.process_image(&bytes, config).await
+        #[cfg(feature = "tokio-runtime")]
+        {
+            use crate::core::io;
+            let bytes = io::read_file_async(path).await?;
+            self.process_image(&bytes, config).await
+        }
+        #[cfg(not(feature = "tokio-runtime"))]
+        {
+            let _ = (path, config);
+            // For WASM and non-tokio environments, file-based OCR is not supported
+            // through the default implementation. Implementations must provide their own.
+            Err(KreuzbergError::Other(
+                "File-based OCR processing requires the tokio-runtime feature".to_string(),
+            ))
+        }
     }
 
     /// Check if this backend supports a given language code.
@@ -302,6 +319,7 @@ pub trait OcrBackend: Plugin {
 ///             detected_languages: None,
 ///             chunks: None,
 ///             images: None,
+///             pages: None,
 ///         })
 ///     }
 ///     fn supports_language(&self, _: &str) -> bool { true }
@@ -462,6 +480,7 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
 

data/vendor/kreuzberg/src/plugins/processor.rs

@@ -373,6 +373,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -422,6 +423,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -488,6 +490,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -513,6 +516,7 @@ mod tests {
                 additional,
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -543,6 +547,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         assert_eq!(processor.estimated_duration_ms(&result), 0);
@@ -593,6 +598,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let txt_result = ExtractionResult {
@@ -603,6 +609,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         assert!(processor.should_process(&pdf_result, &config));
@@ -631,6 +638,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();

data/vendor/kreuzberg/src/plugins/registry.rs

@@ -661,6 +661,7 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
 
@@ -705,6 +706,7 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
 

data/vendor/kreuzberg/src/plugins/validator.rs

@@ -489,6 +489,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -507,6 +508,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -527,6 +529,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -562,6 +565,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -609,6 +613,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let txt_result = ExtractionResult {
@@ -619,6 +624,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         assert!(validator.should_validate(&pdf_result, &config));
@@ -702,6 +708,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -729,6 +736,7 @@ mod tests {
                 additional,
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -759,6 +767,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();
@@ -787,6 +796,7 @@ mod tests {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             };
 
             assert!(validator.validate(&result, &config).await.is_ok());
@@ -805,6 +815,7 @@ mod tests {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config = ExtractionConfig::default();

data/vendor/kreuzberg/src/text/mod.rs

@@ -7,9 +7,15 @@ pub mod string_utils;
 #[cfg(feature = "quality")]
 pub mod token_reduction;
 
+#[cfg(feature = "quality")]
+pub mod quality_processor;
+
 #[cfg(feature = "quality")]
 pub use quality::{calculate_quality_score, clean_extracted_text, normalize_spaces};
 
+#[cfg(feature = "quality")]
+pub use quality_processor::QualityProcessor;
+
 #[cfg(feature = "quality")]
 pub use string_utils::{calculate_text_confidence, fix_mojibake, get_encoding_cache_key, safe_decode};