kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/src/extractors/pdf.rs

@@ -3,7 +3,7 @@
 use crate::Result;
 use crate::core::config::ExtractionConfig;
 use crate::plugins::{DocumentExtractor, Plugin};
-use crate::types::{ExtractionResult, Metadata};
+use crate::types::{ExtractionResult, Metadata, PageContent};
 use async_trait::async_trait;
 use std::path::Path;
 
@@ -140,7 +140,7 @@ fn evaluate_native_text_for_ocr(native_text: &str, page_count: Option<usize>) ->
 #[cfg(all(feature = "pdf", feature = "ocr"))]
 fn extract_tables_from_document(
     document: &PdfDocument,
-    _metadata: &crate::pdf::metadata::PdfMetadata,
+    _metadata: &crate::pdf::metadata::PdfExtractionMetadata,
 ) -> Result<Vec<Table>> {
     use crate::ocr::table::{reconstruct_table, table_to_markdown};
     use crate::pdf::table::extract_words_from_page;
@@ -177,11 +177,41 @@ fn extract_tables_from_document(
 #[cfg(all(feature = "pdf", not(feature = "ocr")))]
 fn extract_tables_from_document(
     _document: &PdfDocument,
-    _metadata: &crate::pdf::metadata::PdfMetadata,
+    _metadata: &crate::pdf::metadata::PdfExtractionMetadata,
 ) -> Result<Vec<crate::types::Table>> {
     Ok(vec![])
 }
 
+/// Helper function to assign tables and images to pages.
+///
+/// If page_contents is None, returns None (no per-page tracking enabled).
+/// Otherwise, iterates through tables and images, assigning them to pages based on page_number.
+fn assign_tables_and_images_to_pages(
+    mut page_contents: Option<Vec<PageContent>>,
+    tables: &[crate::types::Table],
+    images: &[crate::types::ExtractedImage],
+) -> Option<Vec<PageContent>> {
+    let pages = page_contents.take()?;
+
+    let mut updated_pages = pages;
+
+    for table in tables {
+        if let Some(page) = updated_pages.iter_mut().find(|p| p.page_number == table.page_number) {
+            page.tables.push(table.clone());
+        }
+    }
+
+    for image in images {
+        if let Some(page_num) = image.page_number
+            && let Some(page) = updated_pages.iter_mut().find(|p| p.page_number == page_num)
+        {
+            page.images.push(image.clone());
+        }
+    }
+
+    Some(updated_pages)
+}
+
 /// PDF document extractor using pypdfium2 and playa-pdf.
 pub struct PdfExtractor;
 
@@ -295,9 +325,10 @@ impl DocumentExtractor for PdfExtractor {
         config: &ExtractionConfig,
     ) -> Result<ExtractionResult> {
         #[cfg(feature = "pdf")]
-        let (pdf_metadata, native_text, tables) = if crate::core::batch_mode::is_batch_mode() {
+        let (pdf_metadata, native_text, tables, page_contents) = if crate::core::batch_mode::is_batch_mode() {
             let content_owned = content.to_vec();
             let span = tracing::Span::current();
+            let pages_config = config.pages.clone();
             tokio::task::spawn_blocking(move || {
                 let _guard = span.entered();
                 let bindings = Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path("./"))
@@ -315,12 +346,25 @@ impl DocumentExtractor for PdfExtractor {
                     }
                 })?;
 
-                let metadata = crate::pdf::metadata::extract_metadata_from_document(&document)?;
-                let native_text = crate::pdf::text::extract_text_from_pdf_document(&document)?;
+                let (native_text, boundaries, page_contents) =
+                    crate::pdf::text::extract_text_from_pdf_document(&document, pages_config.as_ref())?;
 
-                let tables = extract_tables_from_document(&document, &metadata)?;
+                let pdf_metadata =
+                    crate::pdf::metadata::extract_metadata_from_document(&document, boundaries.as_deref())?;
 
-                Ok::<_, crate::error::KreuzbergError>((metadata, native_text, tables))
+                let tables = extract_tables_from_document(&document, &pdf_metadata)?;
+
+                if let Some(ref page_cfg) = pages_config
+                    && page_cfg.extract_pages
+                    && page_contents.is_none()
+                {
+                    return Err(PdfError::ExtractionFailed(
+                        "Page extraction was configured but no page data was extracted in batch mode".to_string(),
+                    )
+                    .into());
+                }
+
+                Ok::<_, crate::error::KreuzbergError>((pdf_metadata, native_text, tables, page_contents))
             })
             .await
             .map_err(|e| crate::error::KreuzbergError::Other(format!("PDF extraction task failed: {}", e)))??
@@ -340,12 +384,14 @@ impl DocumentExtractor for PdfExtractor {
                 }
             })?;
 
-            let metadata = crate::pdf::metadata::extract_metadata_from_document(&document)?;
-            let native_text = crate::pdf::text::extract_text_from_pdf_document(&document)?;
+            let (native_text, boundaries, page_contents) =
+                crate::pdf::text::extract_text_from_pdf_document(&document, config.pages.as_ref())?;
+
+            let pdf_metadata = crate::pdf::metadata::extract_metadata_from_document(&document, boundaries.as_deref())?;
 
-            let tables = extract_tables_from_document(&document, &metadata)?;
+            let tables = extract_tables_from_document(&document, &pdf_metadata)?;
 
-            (metadata, native_text, tables)
+            (pdf_metadata, native_text, tables, page_contents)
         };
 
         #[cfg(feature = "ocr")]
@@ -356,20 +402,19 @@ impl DocumentExtractor for PdfExtractor {
                 native_text
             }
         } else if config.ocr.is_some() {
-            let decision = evaluate_native_text_for_ocr(&native_text, pdf_metadata.page_count);
+            let decision = evaluate_native_text_for_ocr(&native_text, None);
 
             if std::env::var("KREUZBERG_DEBUG_OCR").is_ok() {
                 eprintln!(
                     "[kreuzberg::pdf::ocr] fallback={} non_whitespace={} alnum={} meaningful_words={} \
-                     avg_non_whitespace={:.2} avg_alnum={:.2} alnum_ratio={:.3} pages={}",
+                     avg_non_whitespace={:.2} avg_alnum={:.2} alnum_ratio={:.3}",
                     decision.fallback,
                     decision.stats.non_whitespace,
                     decision.stats.alnum,
                     decision.stats.meaningful_words,
                     decision.avg_non_whitespace,
                     decision.avg_alnum,
-                    decision.stats.alnum_ratio,
-                    pdf_metadata.page_count.unwrap_or(0)
+                    decision.stats.alnum_ratio
                 );
             }
 
@@ -385,6 +430,20 @@ impl DocumentExtractor for PdfExtractor {
         #[cfg(not(feature = "ocr"))]
         let text = native_text;
 
+        #[cfg(feature = "pdf")]
+        if let Some(ref page_cfg) = config.pages
+            && page_cfg.insert_page_markers
+        {
+            let marker_placeholder = page_cfg.marker_format.replace("{page_num}", "");
+            if !marker_placeholder.is_empty() && !text.contains(&marker_placeholder) {
+                #[cfg(feature = "otel")]
+                tracing::warn!(
+                    "Page markers were configured but none found in extracted content. \
+                     This may indicate very short documents or incomplete extraction."
+                );
+            }
+        }
+
         let images = if config.images.is_some() {
             match crate::pdf::images::extract_images_from_pdf(content) {
                 Ok(pdf_images) => Some(
@@ -415,14 +474,33 @@ impl DocumentExtractor for PdfExtractor {
             None
         };
 
+        let final_pages = assign_tables_and_images_to_pages(page_contents, &tables, images.as_deref().unwrap_or(&[]));
+
         Ok(ExtractionResult {
             content: text,
             mime_type: mime_type.to_string(),
             metadata: Metadata {
                 #[cfg(feature = "pdf")]
-                format: Some(crate::types::FormatMetadata::Pdf(pdf_metadata)),
+                title: pdf_metadata.title.clone(),
+                #[cfg(feature = "pdf")]
+                subject: pdf_metadata.subject.clone(),
+                #[cfg(feature = "pdf")]
+                authors: pdf_metadata.authors.clone(),
+                #[cfg(feature = "pdf")]
+                keywords: pdf_metadata.keywords.clone(),
+                #[cfg(feature = "pdf")]
+                created_at: pdf_metadata.created_at.clone(),
+                #[cfg(feature = "pdf")]
+                modified_at: pdf_metadata.modified_at.clone(),
+                #[cfg(feature = "pdf")]
+                created_by: pdf_metadata.created_by.clone(),
+                #[cfg(feature = "pdf")]
+                pages: pdf_metadata.page_structure.clone(),
+                #[cfg(feature = "pdf")]
+                format: Some(crate::types::FormatMetadata::Pdf(pdf_metadata.pdf_specific)),
                 ..Default::default()
             },
+            pages: final_pages,
             tables,
             detected_languages: None,
             chunks: None,
@@ -490,4 +568,106 @@ mod tests {
         let sample = " . , ; : -- -- ";
         assert!(evaluate_native_text_for_ocr(sample, Some(2)).fallback);
     }
+
+    #[tokio::test]
+    #[cfg(feature = "pdf")]
+    async fn test_pdf_batch_mode_validates_page_config_enabled() {
+        use crate::core::config::PageConfig;
+
+        let extractor = PdfExtractor::new();
+
+        let config = ExtractionConfig {
+            pages: Some(PageConfig {
+                extract_pages: true,
+                insert_page_markers: false,
+                marker_format: "<!-- PAGE {page_num} -->".to_string(),
+            }),
+            ..Default::default()
+        };
+
+        let pdf_path =
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/pdfs/google_doc_document.pdf");
+        if let Ok(content) = std::fs::read(pdf_path) {
+            let result = extractor.extract_bytes(&content, "application/pdf", &config).await;
+            assert!(
+                result.is_ok(),
+                "Failed to extract PDF with page config: {:?}",
+                result.err()
+            );
+
+            let extraction_result = result.unwrap();
+            assert!(
+                extraction_result.pages.is_some(),
+                "Pages should be extracted when extract_pages is true"
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[cfg(feature = "pdf")]
+    async fn test_pdf_batch_mode_validates_page_config_disabled() {
+        let extractor = PdfExtractor::new();
+        let config = ExtractionConfig::default();
+
+        let pdf_path =
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/pdfs/google_doc_document.pdf");
+        if let Ok(content) = std::fs::read(pdf_path) {
+            let result = extractor.extract_bytes(&content, "application/pdf", &config).await;
+            assert!(
+                result.is_ok(),
+                "Failed to extract PDF without page config: {:?}",
+                result.err()
+            );
+
+            let extraction_result = result.unwrap();
+            assert!(
+                extraction_result.pages.is_none(),
+                "Pages should not be extracted when pages config is None"
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[cfg(feature = "pdf")]
+    async fn test_pdf_page_marker_validation() {
+        use crate::core::config::PageConfig;
+
+        let extractor = PdfExtractor::new();
+
+        let config = ExtractionConfig {
+            pages: Some(PageConfig {
+                extract_pages: true,
+                insert_page_markers: true,
+                marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
+            }),
+            ..Default::default()
+        };
+
+        let pdf_path =
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/pdfs/multi_page.pdf");
+        if let Ok(content) = std::fs::read(pdf_path) {
+            let result = extractor.extract_bytes(&content, "application/pdf", &config).await;
+            assert!(
+                result.is_ok(),
+                "Failed to extract PDF with page markers: {:?}",
+                result.err()
+            );
+
+            let extraction_result = result.unwrap();
+            let marker_placeholder = "<!-- PAGE ";
+            if extraction_result.content.len() > 100 {
+                assert!(
+                    extraction_result.content.contains(marker_placeholder),
+                    "Page markers should be inserted when configured and document has multiple pages"
+                );
+            }
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_pdf_extractor_without_feature_pdf() {
+        let extractor = PdfExtractor::new();
+        assert_eq!(extractor.name(), "pdf-extractor");
+    }
 }

data/vendor/kreuzberg/src/extractors/pptx.rs

@@ -1,3 +1,5 @@
+#![cfg(all(feature = "tokio-runtime", feature = "office"))]
+
 //! PowerPoint presentation extractor.
 
 use crate::Result;
@@ -69,6 +71,7 @@ impl PptxExtractor {
                         detected_languages: None,
                         chunks: None,
                         images: None,
+                        pages: None,
                     };
                     image.ocr_result = Some(Box::new(extraction_result));
                 }
@@ -117,17 +120,18 @@ impl DocumentExtractor for PptxExtractor {
     ) -> Result<ExtractionResult> {
         let extract_images = config.images.as_ref().is_some_and(|img| img.extract_images);
 
+        let pages_config = config.pages.clone();
         let pptx_result = if crate::core::batch_mode::is_batch_mode() {
             let content_owned = content.to_vec();
             let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || {
                 let _guard = span.entered();
-                crate::extraction::pptx::extract_pptx_from_bytes(&content_owned, extract_images)
+                crate::extraction::pptx::extract_pptx_from_bytes(&content_owned, extract_images, pages_config.as_ref())
             })
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("PPTX extraction task failed: {}", e)))??
         } else {
-            crate::extraction::pptx::extract_pptx_from_bytes(content, extract_images)?
+            crate::extraction::pptx::extract_pptx_from_bytes(content, extract_images, config.pages.as_ref())?
         };
 
         let mut additional = std::collections::HashMap::new();
@@ -149,14 +153,21 @@ impl DocumentExtractor for PptxExtractor {
             None
         };
 
+        let mut metadata = Metadata {
+            format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
+            additional,
+            ..Default::default()
+        };
+
+        if let Some(page_structure) = pptx_result.page_structure {
+            metadata.pages = Some(page_structure);
+        }
+
         Ok(ExtractionResult {
             content: pptx_result.content,
             mime_type: mime_type.to_string(),
-            metadata: Metadata {
-                format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
-                additional,
-                ..Default::default()
-            },
+            metadata,
+            pages: pptx_result.page_contents,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -177,7 +188,8 @@ impl DocumentExtractor for PptxExtractor {
 
         let extract_images = config.images.as_ref().is_some_and(|img| img.extract_images);
 
-        let pptx_result = crate::extraction::pptx::extract_pptx_from_path(path_str, extract_images)?;
+        let pptx_result =
+            crate::extraction::pptx::extract_pptx_from_path(path_str, extract_images, config.pages.as_ref())?;
 
         let mut additional = std::collections::HashMap::new();
         additional.insert("slide_count".to_string(), serde_json::json!(pptx_result.slide_count));
@@ -198,14 +210,21 @@ impl DocumentExtractor for PptxExtractor {
             None
         };
 
+        let mut metadata = Metadata {
+            format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
+            additional,
+            ..Default::default()
+        };
+
+        if let Some(page_structure) = pptx_result.page_structure {
+            metadata.pages = Some(page_structure);
+        }
+
         Ok(ExtractionResult {
             content: pptx_result.content,
             mime_type: mime_type.to_string(),
-            metadata: Metadata {
-                format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
-                additional,
-                ..Default::default()
-            },
+            metadata,
+            pages: pptx_result.page_contents,
             tables: vec![],
             detected_languages: None,
             chunks: None,

data/vendor/kreuzberg/src/extractors/rst.rs

@@ -453,6 +453,7 @@ impl DocumentExtractor for RstExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/rtf.rs

@@ -391,9 +391,7 @@ fn extract_text_from_rtf(content: &str) -> (String, Vec<Table>) {
                                     if let Some(state) = table_state.as_ref()
                                         && !state.in_row
                                         && !state.rows.is_empty()
-                                    {
-                                        // We'll finalize once we see content outside the table
-                                    }
+                                    {}
                                 }
                                 _ => {}
                             }
@@ -571,7 +569,7 @@ fn extract_rtf_metadata(rtf_content: &str, extracted_text: &str) -> HashMap<Stri
             };
 
             let mut chars = cleaned_segment.chars().peekable();
-            chars.next(); // consume the leading backslash
+            chars.next();
             let (keyword, numeric) = parse_rtf_control_word(&mut chars);
             let remaining: String = chars.collect();
             let trimmed = remaining.trim();
@@ -771,6 +769,7 @@ impl DocumentExtractor for RtfExtractor {
                 additional: metadata_map,
                 ..Default::default()
             },
+            pages: None,
             tables,
             detected_languages: None,
             chunks: None,

data/vendor/kreuzberg/src/extractors/structured.rs

@@ -5,6 +5,7 @@ use crate::core::config::ExtractionConfig;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::types::{ExtractionResult, Metadata};
 use async_trait::async_trait;
+#[cfg(feature = "tokio-runtime")]
 use std::path::Path;
 
 /// Structured data extractor supporting JSON, YAML, and TOML.
@@ -80,6 +81,7 @@ impl DocumentExtractor for StructuredExtractor {
                 additional,
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,

data/vendor/kreuzberg/src/extractors/text.rs

@@ -86,6 +86,7 @@ impl DocumentExtractor for PlainTextExtractor {
                 })),
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -94,7 +95,7 @@ impl DocumentExtractor for PlainTextExtractor {
     }
 
     fn supported_mime_types(&self) -> &[&str] {
-        &["text/plain"]
+        &["text/plain", "text/csv", "text/tab-separated-values"]
     }
 
     fn priority(&self) -> i32 {
@@ -178,6 +179,7 @@ impl DocumentExtractor for MarkdownExtractor {
                 })),
                 ..Default::default()
             },
+            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,
@@ -245,7 +247,10 @@ mod tests {
         let extractor = PlainTextExtractor::new();
         assert_eq!(extractor.name(), "plain-text-extractor");
         assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
-        assert_eq!(extractor.supported_mime_types(), &["text/plain"]);
+        assert_eq!(
+            extractor.supported_mime_types(),
+            &["text/plain", "text/csv", "text/tab-separated-values"]
+        );
         assert_eq!(extractor.priority(), 50);
     }
 

data/vendor/kreuzberg/src/extractors/typst.rs

@@ -112,6 +112,7 @@ impl DocumentExtractor for TypstExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 

data/vendor/kreuzberg/src/extractors/xml.rs

@@ -3,6 +3,7 @@
 use crate::Result;
 use crate::core::config::ExtractionConfig;
 use crate::extraction::xml::parse_xml;
+use crate::extractors::SyncExtractor;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::types::ExtractionResult;
 use async_trait::async_trait;
@@ -51,21 +52,8 @@ impl Plugin for XmlExtractor {
     }
 }
 
-#[async_trait]
-impl DocumentExtractor for XmlExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
-    async fn extract_bytes(
-        &self,
-        content: &[u8],
-        mime_type: &str,
-        _config: &ExtractionConfig,
-    ) -> Result<ExtractionResult> {
+impl SyncExtractor for XmlExtractor {
+    fn extract_sync(&self, content: &[u8], mime_type: &str, _config: &ExtractionConfig) -> Result<ExtractionResult> {
         let xml_result = parse_xml(content, false)?;
 
         Ok(ExtractionResult {
@@ -82,8 +70,28 @@ impl DocumentExtractor for XmlExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
+}
+
+#[async_trait]
+impl DocumentExtractor for XmlExtractor {
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, content, config),
+        fields(
+            extractor.name = self.name(),
+            content.size_bytes = content.len(),
+        )
+    ))]
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        self.extract_sync(content, mime_type, config)
+    }
 
     fn supported_mime_types(&self) -> &[&str] {
         &["application/xml", "text/xml", "image/svg+xml"]
@@ -92,6 +100,10 @@ impl DocumentExtractor for XmlExtractor {
     fn priority(&self) -> i32 {
         50
     }
+
+    fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
+        Some(self)
+    }
 }
 
 #[cfg(test)]

data/vendor/kreuzberg/src/keywords/processor.rs

@@ -45,7 +45,8 @@ impl Plugin for KeywordExtractor {
     }
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl PostProcessor for KeywordExtractor {
     async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
         let keyword_config = match &config.keywords {
@@ -112,6 +113,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -140,6 +142,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -164,6 +167,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -188,6 +192,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         processor.process(&mut result, &config).await.unwrap();
@@ -223,6 +228,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let config_with_keywords = ExtractionConfig {
@@ -247,6 +253,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let long_result = ExtractionResult {
@@ -257,6 +264,7 @@ machine learning that uses neural networks with multiple layers.
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         };
 
         let short_duration = processor.estimated_duration_ms(&short_result);