kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/extraction/structured.rs

@@ -31,7 +31,6 @@
 //! # }
 //! ```
 use crate::error::{KreuzbergError, Result};
-use crate::text::utf8_validation;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
@@ -240,8 +239,8 @@ fn is_text_field(key: &str, custom_patterns: &[String]) -> bool {
 }
 
 pub fn parse_yaml(data: &[u8]) -> Result<StructuredDataResult> {
-    let yaml_str = utf8_validation::from_utf8(data)
-        .map_err(|e| KreuzbergError::parsing(format!("Invalid UTF-8 in YAML: {}", e)))?;
+    let yaml_str =
+        std::str::from_utf8(data).map_err(|e| KreuzbergError::parsing(format!("Invalid UTF-8 in YAML: {}", e)))?;
 
     let value: serde_json::Value = serde_yaml_ng::from_str(yaml_str)
         .map_err(|e| KreuzbergError::parsing(format!("Failed to parse YAML: {}", e)))?;
@@ -312,8 +311,8 @@ fn extract_from_value(
 }
 
 pub fn parse_toml(data: &[u8]) -> Result<StructuredDataResult> {
-    let toml_str = utf8_validation::from_utf8(data)
-        .map_err(|e| KreuzbergError::parsing(format!("Invalid UTF-8 in TOML: {}", e)))?;
+    let toml_str =
+        std::str::from_utf8(data).map_err(|e| KreuzbergError::parsing(format!("Invalid UTF-8 in TOML: {}", e)))?;
 
     let value: toml::Value =
         toml::from_str(toml_str).map_err(|e| KreuzbergError::parsing(format!("Failed to parse TOML: {}", e)))?;

data/vendor/kreuzberg/src/extraction/table.rs

@@ -60,8 +60,7 @@ fn dataframe_to_markdown(df: &DataFrame) -> Result<String> {
         return Ok(String::new());
     }
 
-    let estimated_capacity = df.height().saturating_mul(df.width()).saturating_mul(12).max(64);
-    let mut markdown = String::with_capacity(estimated_capacity);
+    let mut markdown = String::new();
 
     markdown.push_str("| ");
     for col_name in df.get_column_names() {

data/vendor/kreuzberg/src/extraction/text.rs

@@ -26,7 +26,6 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 
 use crate::error::Result;
-use crate::text::utf8_validation;
 use crate::types::TextExtractionResult;
 
 static MARKDOWN_HEADER: Lazy<Regex> =
@@ -39,25 +38,18 @@ static CODE_BLOCK_DELIMITER: Lazy<Regex> = Lazy::new(|| {
 });
 
 pub fn parse_text(text_bytes: &[u8], is_markdown: bool) -> Result<TextExtractionResult> {
-    let text: std::borrow::Cow<'_, str> = match utf8_validation::from_utf8(text_bytes) {
-        Ok(s) => std::borrow::Cow::Borrowed(s),
-        Err(_) => std::borrow::Cow::Owned(String::from_utf8_lossy(text_bytes).into_owned()),
-    };
+    let text = String::from_utf8_lossy(text_bytes).into_owned();
 
     let mut line_count = 0;
     let mut word_count = 0;
     let character_count = text.len();
 
-    let estimated_headers_capacity = text.len().saturating_div(20).max(16);
-    let estimated_links_capacity = text.lines().count().saturating_div(20).max(4);
-    let estimated_code_blocks_capacity = 8;
-
-    let mut headers = Vec::with_capacity(estimated_headers_capacity);
-    let mut links = Vec::with_capacity(estimated_links_capacity);
-    let mut code_blocks = Vec::with_capacity(estimated_code_blocks_capacity);
+    let mut headers = Vec::new();
+    let mut links = Vec::new();
+    let mut code_blocks = Vec::new();
     let mut in_code_block = false;
-    let mut current_code_lang = String::with_capacity(16);
-    let mut current_code = String::with_capacity(128);
+    let mut current_code_lang = String::new();
+    let mut current_code = String::new();
 
     for line in text.lines() {
         line_count += 1;
@@ -73,7 +65,7 @@ pub fn parse_text(text_bytes: &[u8], is_markdown: bool) -> Result<TextExtraction
                     if current_code_lang.is_empty() {
                         "plain".to_string()
                     } else {
-                        std::mem::take(&mut current_code_lang)
+                        current_code_lang.clone()
                     },
                     current_code.trim_end().to_string(),
                 ));
@@ -102,14 +94,14 @@ pub fn parse_text(text_bytes: &[u8], is_markdown: bool) -> Result<TextExtraction
         }
 
         for caps in MARKDOWN_LINK.captures_iter(line) {
-            if let (Some(text_match), Some(url)) = (caps.get(1), caps.get(2)) {
-                links.push((text_match.as_str().to_string(), url.as_str().to_string()));
+            if let (Some(text), Some(url)) = (caps.get(1), caps.get(2)) {
+                links.push((text.as_str().to_string(), url.as_str().to_string()));
             }
         }
     }
 
     Ok(TextExtractionResult {
-        content: text.into_owned(),
+        content: text,
         line_count,
         word_count,
         character_count,

data/vendor/kreuzberg/src/extractors/archive.rs

@@ -77,7 +77,6 @@ fn build_archive_result(
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     }
 }
 
@@ -127,13 +126,6 @@ impl Plugin for ZipExtractor {
 
 #[async_trait]
 impl DocumentExtractor for ZipExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -205,13 +197,6 @@ impl Plugin for TarExtractor {
 
 #[async_trait]
 impl DocumentExtractor for TarExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -288,13 +273,6 @@ impl Plugin for SevenZExtractor {
 
 #[async_trait]
 impl DocumentExtractor for SevenZExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],

data/vendor/kreuzberg/src/extractors/docx.rs

@@ -1,14 +1,12 @@
-#![cfg(all(feature = "tokio-runtime", feature = "office"))]
-
 //! DOCX extractor using docx-lite for high-performance text extraction.
 //!
 //! Supports: Microsoft Word (.docx)
 
 use crate::Result;
 use crate::core::config::ExtractionConfig;
-use crate::extraction::{cells_to_markdown, office_metadata};
+use crate::extraction::office_metadata;
 use crate::plugins::{DocumentExtractor, Plugin};
-use crate::types::{ExtractionResult, Metadata, PageBoundary, PageInfo, PageStructure, PageUnitType, Table};
+use crate::types::{ExtractionResult, Metadata, Table};
 use async_trait::async_trait;
 use std::io::Cursor;
 
@@ -17,6 +15,7 @@ use std::io::Cursor;
 /// This extractor provides:
 /// - Fast text extraction via streaming XML parsing (~160 MB/s average)
 /// - Comprehensive metadata extraction (core.xml, app.xml, custom.xml)
+/// - ~400x faster than Pandoc subprocess approach
 pub struct DocxExtractor;
 
 impl DocxExtractor {
@@ -67,6 +66,7 @@ impl Plugin for DocxExtractor {
 /// # Returns
 /// * `Table` - Converted table with cells and markdown representation
 fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize) -> Table {
+    // Extract cells as 2D vector
     let cells: Vec<Vec<String>> = docx_table
         .rows
         .iter()
@@ -74,6 +74,7 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
             row.cells
                 .iter()
                 .map(|cell| {
+                    // Extract text from all paragraphs in the cell
                     cell.paragraphs
                         .iter()
                         .map(|para| para.to_text())
@@ -86,12 +87,13 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
         })
         .collect();
 
+    // Generate markdown representation
     let markdown = cells_to_markdown(&cells);
 
     Table {
         cells,
         markdown,
-        page_number: table_index + 1,
+        page_number: table_index + 1, // 1-indexed
     }
 }
 
@@ -102,55 +104,103 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
 ///
 /// # Returns
 /// * `String` - Markdown formatted table
+fn cells_to_markdown(cells: &[Vec<String>]) -> String {
+    if cells.is_empty() {
+        return String::new();
+    }
+
+    let mut markdown = String::new();
+
+    // Determine number of columns from first row
+    let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
+    if num_cols == 0 {
+        return String::new();
+    }
+
+    // Header row (first row)
+    if let Some(header) = cells.first() {
+        markdown.push_str("| ");
+        for cell in header {
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+
+        // Separator row
+        markdown.push('|');
+        for _ in 0..num_cols {
+            markdown.push_str("------|");
+        }
+        markdown.push('\n');
+    }
+
+    // Data rows (skip first row as it's the header)
+    for row in cells.iter().skip(1) {
+        markdown.push_str("| ");
+        for (idx, cell) in row.iter().enumerate() {
+            if idx >= num_cols {
+                break; // Handle irregular tables
+            }
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        // Pad with empty cells if row is shorter than expected
+        for _ in row.len()..num_cols {
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+    }
+
+    markdown
+}
 
 #[async_trait]
 impl DocumentExtractor for DocxExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
         mime_type: &str,
         _config: &ExtractionConfig,
     ) -> Result<ExtractionResult> {
-        let (text, tables, page_boundaries) = if crate::core::batch_mode::is_batch_mode() {
+        // Parse the DOCX document to extract both text and tables
+        let (text, tables) = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
-            tokio::task::spawn_blocking(
-                move || -> crate::error::Result<(String, Vec<Table>, Option<Vec<PageBoundary>>)> {
-                    let _guard = span.entered();
-                    let cursor = Cursor::new(&content_owned);
-                    let doc = docx_lite::parse_document(cursor)
-                        .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX parsing failed: {}", e)))?;
-
-                    let text = doc.extract_text();
-
-                    let tables: Vec<Table> = doc
-                        .tables
-                        .iter()
-                        .enumerate()
-                        .map(|(idx, table)| convert_docx_table_to_table(table, idx))
-                        .collect();
-
-                    let page_boundaries = crate::extraction::docx::detect_page_breaks_from_docx(&content_owned)?;
-
-                    Ok((text, tables, page_boundaries))
-                },
-            )
+            tokio::task::spawn_blocking(move || -> crate::error::Result<(String, Vec<Table>)> {
+                // Parse document structure
+                let cursor = Cursor::new(&content_owned);
+                let doc = docx_lite::parse_document(cursor)
+                    .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX parsing failed: {}", e)))?;
+
+                // Extract text
+                let text = doc.extract_text();
+
+                // Extract tables
+                let tables: Vec<Table> = doc
+                    .tables
+                    .iter()
+                    .enumerate()
+                    .map(|(idx, table)| convert_docx_table_to_table(table, idx))
+                    .collect();
+
+                Ok((text, tables))
+            })
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX extraction task failed: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
             let cursor = Cursor::new(content);
             let doc = docx_lite::parse_document(cursor)
                 .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX parsing failed: {}", e)))?;
 
+            // Extract text
             let text = doc.extract_text();
 
+            // Extract tables
             let tables: Vec<Table> = doc
                 .tables
                 .iter()
@@ -158,16 +208,14 @@ impl DocumentExtractor for DocxExtractor {
                 .map(|(idx, table)| convert_docx_table_to_table(table, idx))
                 .collect();
 
-            let page_boundaries = crate::extraction::docx::detect_page_breaks_from_docx(content)?;
-
-            (text, tables, page_boundaries)
+            (text, tables)
         };
 
+        // Extract metadata using existing office_metadata module
         let mut archive = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || -> crate::error::Result<_> {
-                let _guard = span.entered();
                 let cursor = Cursor::new(content_owned);
                 zip::ZipArchive::new(cursor)
                     .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))
@@ -175,6 +223,8 @@ impl DocumentExtractor for DocxExtractor {
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("Task join error: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
+            // Note: We still need to clone for ZipArchive type consistency with batch mode
             let content_owned = content.to_vec();
             let cursor = Cursor::new(content_owned);
             zip::ZipArchive::new(cursor)
@@ -183,6 +233,7 @@ impl DocumentExtractor for DocxExtractor {
 
         let mut metadata_map = std::collections::HashMap::new();
 
+        // Extract core properties (title, creator, dates, keywords, etc.)
         if let Ok(core) = office_metadata::extract_core_properties(&mut archive) {
             if let Some(title) = core.title {
                 metadata_map.insert("title".to_string(), serde_json::Value::String(title));
@@ -226,6 +277,7 @@ impl DocumentExtractor for DocxExtractor {
             }
         }
 
+        // Extract app properties (page count, word count, etc.)
         if let Ok(app) = office_metadata::extract_docx_app_properties(&mut archive) {
             if let Some(pages) = app.pages {
                 metadata_map.insert("page_count".to_string(), serde_json::Value::Number(pages.into()));
@@ -262,48 +314,24 @@ impl DocumentExtractor for DocxExtractor {
             }
         }
 
+        // Extract custom properties
         if let Ok(custom) = office_metadata::extract_custom_properties(&mut archive) {
             for (key, value) in custom {
                 metadata_map.insert(format!("custom_{}", key), value);
             }
         }
 
-        let page_structure = if let Some(boundaries) = page_boundaries {
-            let total_count = boundaries.len();
-            Some(PageStructure {
-                total_count,
-                unit_type: PageUnitType::Page,
-                boundaries: Some(boundaries),
-                pages: Some(
-                    (1..=total_count)
-                        .map(|page_num| PageInfo {
-                            number: page_num,
-                            title: None,
-                            dimensions: None,
-                            image_count: None,
-                            table_count: None,
-                            hidden: None,
-                        })
-                        .collect(),
-                ),
-            })
-        } else {
-            None
-        };
-
         Ok(ExtractionResult {
             content: text,
             mime_type: mime_type.to_string(),
             metadata: Metadata {
-                pages: page_structure,
                 additional: metadata_map,
                 ..Default::default()
             },
-            pages: None,
             tables,
             detected_languages: None,
             chunks: None,
-            images: Some(vec![]),
+            images: None,
         })
     }
 
@@ -312,7 +340,7 @@ impl DocumentExtractor for DocxExtractor {
     }
 
     fn priority(&self) -> i32 {
-        50
+        50 // Higher priority than Pandoc (40) to take precedence
     }
 }
 
@@ -352,12 +380,61 @@ mod tests {
         assert!(extractor.shutdown().is_ok());
     }
 
+    #[test]
+    fn test_cells_to_markdown_basic_table() {
+        let cells = vec![
+            vec!["Header1".to_string(), "Header2".to_string()],
+            vec!["Row1Col1".to_string(), "Row1Col2".to_string()],
+            vec!["Row2Col1".to_string(), "Row2Col2".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        assert!(markdown.contains("| Header1 | Header2 |"));
+        assert!(markdown.contains("|------|------|"));
+        assert!(markdown.contains("| Row1Col1 | Row1Col2 |"));
+        assert!(markdown.contains("| Row2Col1 | Row2Col2 |"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_empty() {
+        let cells: Vec<Vec<String>> = vec![];
+        let markdown = cells_to_markdown(&cells);
+        assert_eq!(markdown, "");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_escape_pipes() {
+        let cells = vec![vec!["Header".to_string()], vec!["Cell with | pipe".to_string()]];
+
+        let markdown = cells_to_markdown(&cells);
+        assert!(markdown.contains("Cell with \\| pipe"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_irregular_rows() {
+        let cells = vec![
+            vec!["H1".to_string(), "H2".to_string(), "H3".to_string()],
+            vec!["R1C1".to_string(), "R1C2".to_string()], // Missing third column
+            vec!["R2C1".to_string(), "R2C2".to_string(), "R2C3".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        // Should have 3 columns in header
+        assert!(markdown.contains("| H1 | H2 | H3 |"));
+        // Should pad short rows
+        assert!(markdown.contains("| R1C1 | R1C2 |  |"));
+    }
+
     #[test]
     fn test_convert_docx_table_to_table() {
         use docx_lite::{Paragraph, Run, Table as DocxTable, TableCell, TableRow};
 
+        // Create a simple docx-lite table
         let mut table = DocxTable::new();
 
+        // Header row
         let mut header_row = TableRow::default();
         let mut cell1 = TableCell::default();
         let mut para1 = Paragraph::new();
@@ -373,6 +450,7 @@ mod tests {
 
         table.rows.push(header_row);
 
+        // Data row
         let mut data_row = TableRow::default();
         let mut cell3 = TableCell::default();
         let mut para3 = Paragraph::new();
@@ -388,10 +466,11 @@ mod tests {
 
         table.rows.push(data_row);
 
+        // Convert to Kreuzberg Table
         let result = convert_docx_table_to_table(&table, 0);
 
-        assert_eq!(result.page_number, 1);
-        assert_eq!(result.cells.len(), 2);
+        assert_eq!(result.page_number, 1); // 0 + 1 = 1 (1-indexed)
+        assert_eq!(result.cells.len(), 2); // 2 rows
         assert_eq!(result.cells[0], vec!["Name", "Age"]);
         assert_eq!(result.cells[1], vec!["Alice", "30"]);
         assert!(result.markdown.contains("| Name | Age |"));

data/vendor/kreuzberg/src/extractors/email.rs

@@ -2,11 +2,9 @@
 
 use crate::Result;
 use crate::core::config::ExtractionConfig;
-use crate::extractors::SyncExtractor;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::types::{EmailMetadata, ExtractionResult, Metadata};
 use async_trait::async_trait;
-#[cfg(feature = "tokio-runtime")]
 use std::path::Path;
 
 /// Email message extractor.
@@ -44,8 +42,14 @@ impl Plugin for EmailExtractor {
     }
 }
 
-impl SyncExtractor for EmailExtractor {
-    fn extract_sync(&self, content: &[u8], mime_type: &str, _config: &ExtractionConfig) -> Result<ExtractionResult> {
+#[async_trait]
+impl DocumentExtractor for EmailExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
         let email_result = crate::extraction::email::extract_email_content(content, mime_type)?;
 
         let text = crate::extraction::email::build_email_text_output(&email_result);
@@ -77,7 +81,7 @@ impl SyncExtractor for EmailExtractor {
             metadata: Metadata {
                 format: Some(crate::types::FormatMetadata::Email(email_metadata)),
                 subject: email_result.subject.clone(),
-                created_at: email_result.date.clone(),
+                date: email_result.date.clone(),
                 additional,
                 ..Default::default()
             },
@@ -85,37 +89,9 @@ impl SyncExtractor for EmailExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         })
     }
-}
 
-#[async_trait]
-impl DocumentExtractor for EmailExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
-    async fn extract_bytes(
-        &self,
-        content: &[u8],
-        mime_type: &str,
-        config: &ExtractionConfig,
-    ) -> Result<ExtractionResult> {
-        self.extract_sync(content, mime_type, config)
-    }
-
-    #[cfg(feature = "tokio-runtime")]
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
-    #[cfg(feature = "tokio-runtime")]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await
@@ -128,10 +104,6 @@ impl DocumentExtractor for EmailExtractor {
     fn priority(&self) -> i32 {
         50
     }
-
-    fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
-        Some(self)
-    }
 }
 
 #[cfg(test)]