kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/extraction/pandoc/version.rs

@@ -0,0 +1,162 @@
+use crate::error::{KreuzbergError, Result};
+use once_cell::sync::OnceCell;
+use regex::Regex;
+use tokio::process::Command;
+
+static PANDOC_VALIDATED: OnceCell<bool> = OnceCell::new();
+
+/// Validate that Pandoc version 2 or above is installed and available
+pub async fn validate_pandoc_version() -> Result<()> {
+    if PANDOC_VALIDATED.get().is_some() {
+        return Ok(());
+    }
+
+    let output = Command::new("pandoc").arg("--version").output().await.map_err(|e| {
+        KreuzbergError::MissingDependency(format!(
+            "Pandoc version 2 or above is required but not found in PATH: {}",
+            e
+        ))
+    })?;
+
+    if !output.status.success() {
+        return Err(KreuzbergError::MissingDependency(
+            "Pandoc version 2 or above is required but command failed".to_string(),
+        ));
+    }
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+
+    let version = extract_version(&stdout).ok_or_else(|| {
+        KreuzbergError::MissingDependency(format!("Could not parse Pandoc version from output: {}", stdout))
+    })?;
+
+    if version.major < 2 {
+        return Err(KreuzbergError::MissingDependency(format!(
+            "Pandoc version 2 or above is required, found version {}.{}.{}",
+            version.major, version.minor, version.patch
+        )));
+    }
+
+    let _ = PANDOC_VALIDATED.set(true);
+
+    Ok(())
+}
+
+#[derive(Debug, Clone)]
+struct Version {
+    major: u32,
+    minor: u32,
+    patch: u32,
+}
+
+fn extract_version(output: &str) -> Option<Version> {
+    let patterns = [
+        r"pandoc(?:\.exe)?(?:\s+|\s+v|\s+version\s+)(\d+)\.(\d+)(?:\.(\d+))?",
+        r"pandoc\s+\(version\s+(\d+)\.(\d+)(?:\.(\d+))?\)",
+        r"pandoc-(\d+)\.(\d+)(?:\.(\d+))?",
+        r"^(\d+)\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?",
+        r"(?:^|\s)(\d+)\.(\d+)(?:\.(\d+))?(?:\s|$)",
+    ];
+
+    for pattern in &patterns {
+        if let Ok(re) = Regex::new(pattern)
+            && let Some(caps) = re.captures(output)
+        {
+            let major = caps.get(1)?.as_str().parse().ok()?;
+            let minor = caps.get(2)?.as_str().parse().ok()?;
+            let patch = caps.get(3).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
+
+            return Some(Version { major, minor, patch });
+        }
+    }
+
+    for line in output.lines() {
+        for token in line.split_whitespace() {
+            if let Some(version) = parse_version_token(token) {
+                return Some(version);
+            }
+        }
+    }
+
+    None
+}
+
+fn parse_version_token(token: &str) -> Option<Version> {
+    let parts: Vec<&str> = token.split('.').collect();
+    if parts.len() >= 2
+        && let (Ok(major), Ok(minor)) = (parts[0].parse(), parts[1].parse())
+    {
+        let patch = parts.get(2).and_then(|p| p.parse().ok()).unwrap_or(0);
+        return Some(Version { major, minor, patch });
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_version_standard_format() {
+        let output = "pandoc 3.1.2";
+        let version = extract_version(output).unwrap();
+        assert_eq!(version.major, 3);
+        assert_eq!(version.minor, 1);
+        assert_eq!(version.patch, 2);
+    }
+
+    #[test]
+    fn test_extract_version_with_parens() {
+        let output = "pandoc (version 2.19.2)";
+        let version = extract_version(output).unwrap();
+        assert_eq!(version.major, 2);
+        assert_eq!(version.minor, 19);
+    }
+
+    #[test]
+    fn test_extract_version_with_exe() {
+        let output = "pandoc.exe 3.0";
+        let version = extract_version(output).unwrap();
+        assert_eq!(version.major, 3);
+        assert_eq!(version.minor, 0);
+    }
+
+    #[test]
+    fn test_extract_version_multiline() {
+        let output = "pandoc 3.1.2\nCopyright (C) 2006-2023 John MacFarlane";
+        let version = extract_version(output).unwrap();
+        assert_eq!(version.major, 3);
+        assert_eq!(version.minor, 1);
+    }
+
+    #[test]
+    fn test_extract_version_no_patch() {
+        let output = "pandoc 2.5";
+        let version = extract_version(output).unwrap();
+        assert_eq!(version.major, 2);
+        assert_eq!(version.minor, 5);
+        assert_eq!(version.patch, 0);
+    }
+
+    #[test]
+    fn test_parse_version_token() {
+        let version = parse_version_token("2.19.2").unwrap();
+        assert_eq!(version.major, 2);
+        assert_eq!(version.minor, 19);
+        assert_eq!(version.patch, 2);
+    }
+
+    #[test]
+    fn test_parse_version_token_no_patch() {
+        let version = parse_version_token("3.1").unwrap();
+        assert_eq!(version.major, 3);
+        assert_eq!(version.minor, 1);
+        assert_eq!(version.patch, 0);
+    }
+
+    #[test]
+    fn test_parse_version_token_invalid() {
+        let version = parse_version_token("abc");
+        assert!(version.is_none());
+    }
+}

data/vendor/kreuzberg/src/extractors/archive.rs

@@ -126,13 +126,6 @@ impl Plugin for ZipExtractor {
 
 #[async_trait]
 impl DocumentExtractor for ZipExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -204,13 +197,6 @@ impl Plugin for TarExtractor {
 
 #[async_trait]
 impl DocumentExtractor for TarExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -287,13 +273,6 @@ impl Plugin for SevenZExtractor {
 
 #[async_trait]
 impl DocumentExtractor for SevenZExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],

data/vendor/kreuzberg/src/extractors/docx.rs

@@ -4,7 +4,7 @@
 
 use crate::Result;
 use crate::core::config::ExtractionConfig;
-use crate::extraction::{cells_to_markdown, office_metadata};
+use crate::extraction::office_metadata;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::types::{ExtractionResult, Metadata, Table};
 use async_trait::async_trait;
@@ -15,6 +15,7 @@ use std::io::Cursor;
 /// This extractor provides:
 /// - Fast text extraction via streaming XML parsing (~160 MB/s average)
 /// - Comprehensive metadata extraction (core.xml, app.xml, custom.xml)
+/// - ~400x faster than Pandoc subprocess approach
 pub struct DocxExtractor;
 
 impl DocxExtractor {
@@ -65,6 +66,7 @@ impl Plugin for DocxExtractor {
 /// # Returns
 /// * `Table` - Converted table with cells and markdown representation
 fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize) -> Table {
+    // Extract cells as 2D vector
     let cells: Vec<Vec<String>> = docx_table
         .rows
         .iter()
@@ -72,6 +74,7 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
             row.cells
                 .iter()
                 .map(|cell| {
+                    // Extract text from all paragraphs in the cell
                     cell.paragraphs
                         .iter()
                         .map(|para| para.to_text())
@@ -84,12 +87,13 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
         })
         .collect();
 
+    // Generate markdown representation
     let markdown = cells_to_markdown(&cells);
 
     Table {
         cells,
         markdown,
-        page_number: table_index + 1,
+        page_number: table_index + 1, // 1-indexed
     }
 }
 
@@ -100,33 +104,82 @@ fn convert_docx_table_to_table(docx_table: &docx_lite::Table, table_index: usize
 ///
 /// # Returns
 /// * `String` - Markdown formatted table
+fn cells_to_markdown(cells: &[Vec<String>]) -> String {
+    if cells.is_empty() {
+        return String::new();
+    }
+
+    let mut markdown = String::new();
+
+    // Determine number of columns from first row
+    let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
+    if num_cols == 0 {
+        return String::new();
+    }
+
+    // Header row (first row)
+    if let Some(header) = cells.first() {
+        markdown.push_str("| ");
+        for cell in header {
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+
+        // Separator row
+        markdown.push('|');
+        for _ in 0..num_cols {
+            markdown.push_str("------|");
+        }
+        markdown.push('\n');
+    }
+
+    // Data rows (skip first row as it's the header)
+    for row in cells.iter().skip(1) {
+        markdown.push_str("| ");
+        for (idx, cell) in row.iter().enumerate() {
+            if idx >= num_cols {
+                break; // Handle irregular tables
+            }
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        // Pad with empty cells if row is shorter than expected
+        for _ in row.len()..num_cols {
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+    }
+
+    markdown
+}
 
 #[async_trait]
 impl DocumentExtractor for DocxExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
         mime_type: &str,
         _config: &ExtractionConfig,
     ) -> Result<ExtractionResult> {
+        // Parse the DOCX document to extract both text and tables
         let (text, tables) = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || -> crate::error::Result<(String, Vec<Table>)> {
-                let _guard = span.entered();
+                // Parse document structure
                 let cursor = Cursor::new(&content_owned);
                 let doc = docx_lite::parse_document(cursor)
                     .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX parsing failed: {}", e)))?;
 
+                // Extract text
                 let text = doc.extract_text();
 
+                // Extract tables
                 let tables: Vec<Table> = doc
                     .tables
                     .iter()
@@ -139,12 +192,15 @@ impl DocumentExtractor for DocxExtractor {
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX extraction task failed: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
             let cursor = Cursor::new(content);
             let doc = docx_lite::parse_document(cursor)
                 .map_err(|e| crate::error::KreuzbergError::parsing(format!("DOCX parsing failed: {}", e)))?;
 
+            // Extract text
             let text = doc.extract_text();
 
+            // Extract tables
             let tables: Vec<Table> = doc
                 .tables
                 .iter()
@@ -155,11 +211,11 @@ impl DocumentExtractor for DocxExtractor {
             (text, tables)
         };
 
+        // Extract metadata using existing office_metadata module
         let mut archive = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || -> crate::error::Result<_> {
-                let _guard = span.entered();
                 let cursor = Cursor::new(content_owned);
                 zip::ZipArchive::new(cursor)
                     .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))
@@ -167,6 +223,8 @@ impl DocumentExtractor for DocxExtractor {
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("Task join error: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
+            // Note: We still need to clone for ZipArchive type consistency with batch mode
             let content_owned = content.to_vec();
             let cursor = Cursor::new(content_owned);
             zip::ZipArchive::new(cursor)
@@ -175,6 +233,7 @@ impl DocumentExtractor for DocxExtractor {
 
         let mut metadata_map = std::collections::HashMap::new();
 
+        // Extract core properties (title, creator, dates, keywords, etc.)
         if let Ok(core) = office_metadata::extract_core_properties(&mut archive) {
             if let Some(title) = core.title {
                 metadata_map.insert("title".to_string(), serde_json::Value::String(title));
@@ -218,6 +277,7 @@ impl DocumentExtractor for DocxExtractor {
             }
         }
 
+        // Extract app properties (page count, word count, etc.)
         if let Ok(app) = office_metadata::extract_docx_app_properties(&mut archive) {
             if let Some(pages) = app.pages {
                 metadata_map.insert("page_count".to_string(), serde_json::Value::Number(pages.into()));
@@ -254,6 +314,7 @@ impl DocumentExtractor for DocxExtractor {
             }
         }
 
+        // Extract custom properties
         if let Ok(custom) = office_metadata::extract_custom_properties(&mut archive) {
             for (key, value) in custom {
                 metadata_map.insert(format!("custom_{}", key), value);
@@ -279,7 +340,7 @@ impl DocumentExtractor for DocxExtractor {
     }
 
     fn priority(&self) -> i32 {
-        50
+        50 // Higher priority than Pandoc (40) to take precedence
     }
 }
 
@@ -319,12 +380,61 @@ mod tests {
         assert!(extractor.shutdown().is_ok());
     }
 
+    #[test]
+    fn test_cells_to_markdown_basic_table() {
+        let cells = vec![
+            vec!["Header1".to_string(), "Header2".to_string()],
+            vec!["Row1Col1".to_string(), "Row1Col2".to_string()],
+            vec!["Row2Col1".to_string(), "Row2Col2".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        assert!(markdown.contains("| Header1 | Header2 |"));
+        assert!(markdown.contains("|------|------|"));
+        assert!(markdown.contains("| Row1Col1 | Row1Col2 |"));
+        assert!(markdown.contains("| Row2Col1 | Row2Col2 |"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_empty() {
+        let cells: Vec<Vec<String>> = vec![];
+        let markdown = cells_to_markdown(&cells);
+        assert_eq!(markdown, "");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_escape_pipes() {
+        let cells = vec![vec!["Header".to_string()], vec!["Cell with | pipe".to_string()]];
+
+        let markdown = cells_to_markdown(&cells);
+        assert!(markdown.contains("Cell with \\| pipe"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_irregular_rows() {
+        let cells = vec![
+            vec!["H1".to_string(), "H2".to_string(), "H3".to_string()],
+            vec!["R1C1".to_string(), "R1C2".to_string()], // Missing third column
+            vec!["R2C1".to_string(), "R2C2".to_string(), "R2C3".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        // Should have 3 columns in header
+        assert!(markdown.contains("| H1 | H2 | H3 |"));
+        // Should pad short rows
+        assert!(markdown.contains("| R1C1 | R1C2 |  |"));
+    }
+
     #[test]
     fn test_convert_docx_table_to_table() {
         use docx_lite::{Paragraph, Run, Table as DocxTable, TableCell, TableRow};
 
+        // Create a simple docx-lite table
         let mut table = DocxTable::new();
 
+        // Header row
         let mut header_row = TableRow::default();
         let mut cell1 = TableCell::default();
         let mut para1 = Paragraph::new();
@@ -340,6 +450,7 @@ mod tests {
 
         table.rows.push(header_row);
 
+        // Data row
         let mut data_row = TableRow::default();
         let mut cell3 = TableCell::default();
         let mut para3 = Paragraph::new();
@@ -355,10 +466,11 @@ mod tests {
 
         table.rows.push(data_row);
 
+        // Convert to Kreuzberg Table
         let result = convert_docx_table_to_table(&table, 0);
 
-        assert_eq!(result.page_number, 1);
-        assert_eq!(result.cells.len(), 2);
+        assert_eq!(result.page_number, 1); // 0 + 1 = 1 (1-indexed)
+        assert_eq!(result.cells.len(), 2); // 2 rows
         assert_eq!(result.cells[0], vec!["Name", "Age"]);
         assert_eq!(result.cells[1], vec!["Alice", "30"]);
         assert!(result.markdown.contains("| Name | Age |"));

data/vendor/kreuzberg/src/extractors/email.rs

@@ -44,13 +44,6 @@ impl Plugin for EmailExtractor {
 
 #[async_trait]
 impl DocumentExtractor for EmailExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -99,13 +92,6 @@ impl DocumentExtractor for EmailExtractor {
         })
     }
 
-    #[cfg(feature = "tokio-runtime")]
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await

data/vendor/kreuzberg/src/extractors/excel.rs

@@ -31,34 +31,46 @@ impl ExcelExtractor {
         let mut tables = Vec::with_capacity(workbook.sheets.len());
 
         for (sheet_index, sheet) in workbook.sheets.iter().enumerate() {
+            // Skip empty sheets
             if sheet.row_count == 0 || sheet.col_count == 0 {
                 continue;
             }
 
+            // We need to re-parse the sheet to get structured cell data
+            // The workbook.sheets only contains markdown, not raw cell data
+            // So we'll extract from the markdown table representation
+
+            // Parse cells from markdown
             let lines: Vec<&str> = sheet.markdown.lines().collect();
             let mut cells: Vec<Vec<String>> = Vec::new();
 
+            // Find the table content (skip header line "## Sheet Name" and blank line)
             let table_start = lines.iter().position(|line| line.starts_with("| "));
 
             if let Some(start_idx) = table_start {
                 for line in lines.iter().skip(start_idx) {
                     if line.starts_with("| ") && !line.contains("---") {
+                        // Parse table row
                         let row: Vec<String> = line
                             .trim_start_matches("| ")
                             .trim_end_matches(" |")
                             .split(" | ")
-                            .map(|cell| cell.replace("\\|", "|").replace("\\\\", "\\"))
+                            .map(|cell| {
+                                // Unescape markdown pipes and backslashes
+                                cell.replace("\\|", "|").replace("\\\\", "\\")
+                            })
                             .collect();
                         cells.push(row);
                     }
                 }
             }
 
+            // Only create table if we have data
             if !cells.is_empty() {
                 tables.push(Table {
                     cells,
                     markdown: sheet.markdown.clone(),
-                    page_number: sheet_index + 1,
+                    page_number: sheet_index + 1, // 1-indexed
                 });
             }
         }
@@ -87,13 +99,6 @@ impl Plugin for ExcelExtractor {
 
 #[async_trait]
 impl DocumentExtractor for ExcelExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -112,17 +117,18 @@ impl DocumentExtractor for ExcelExtractor {
             _ => ".xlsx",
         };
 
+        // Extract workbook
         let workbook = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
             let extension_owned = extension.to_string();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || {
-                let _guard = span.entered();
                 crate::extraction::excel::read_excel_bytes(&content_owned, &extension_owned)
             })
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("Excel extraction task failed: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
             crate::extraction::excel::read_excel_bytes(content, extension)?
         };
 
@@ -157,12 +163,6 @@ impl DocumentExtractor for ExcelExtractor {
         })
     }
 
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, _config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
     async fn extract_file(&self, path: &Path, mime_type: &str, _config: &ExtractionConfig) -> Result<ExtractionResult> {
         let path_str = path
             .to_str()
@@ -244,6 +244,7 @@ mod tests {
         use crate::types::ExcelSheet;
         use std::collections::HashMap;
 
+        // Create a mock workbook with a single sheet
         let sheet = ExcelSheet {
             name: "TestSheet".to_string(),
             markdown: r#"## TestSheet
@@ -268,7 +269,7 @@ mod tests {
 
         assert_eq!(tables.len(), 1);
         assert_eq!(tables[0].page_number, 1);
-        assert_eq!(tables[0].cells.len(), 3);
+        assert_eq!(tables[0].cells.len(), 3); // Header + 2 data rows
         assert_eq!(tables[0].cells[0], vec!["Name", "Age", "City"]);
         assert_eq!(tables[0].cells[1], vec!["Alice", "30", "NYC"]);
         assert_eq!(tables[0].cells[2], vec!["Bob", "25", "LA"]);
@@ -293,7 +294,7 @@ mod tests {
         };
 
         let tables = ExcelExtractor::sheets_to_tables(&workbook);
-        assert_eq!(tables.len(), 0);
+        assert_eq!(tables.len(), 0); // Empty sheets should not create tables
     }
 
     #[test]