kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/html.rs

@@ -2,16 +2,12 @@
 
 use crate::Result;
 use crate::core::config::ExtractionConfig;
-use crate::extractors::SyncExtractor;
 use crate::plugins::{DocumentExtractor, Plugin};
-use crate::text::utf8_validation;
 use crate::types::{ExtractionResult, Metadata, Table};
 use async_trait::async_trait;
-#[cfg(feature = "tokio-runtime")]
+use scraper::{Html, Selector};
 use std::path::Path;
 
-// NOTE: scraper dependency has been removed in favor of html-to-markdown-rs
-
 /// HTML document extractor using html-to-markdown.
 pub struct HtmlExtractor;
 
@@ -27,152 +23,134 @@ impl HtmlExtractor {
     }
 }
 
-/// Extract all tables from pre-converted markdown content.
-///
-/// Parses markdown pipe-delimited format to extract table structures.
-/// This function accepts already-converted markdown to enable reuse of
-/// a single HTML-to-markdown conversion pass, improving performance by
-/// eliminating duplicate conversions.
+/// Extract all tables from HTML content.
 ///
-/// # Optimization
-/// By accepting markdown instead of HTML, callers can convert HTML once
-/// and reuse the result for both table extraction and metadata parsing,
-/// reducing computational overhead by ~50% on table extraction flows.
-fn extract_html_tables(markdown: &str) -> Result<Vec<Table>> {
-    let tables = parse_markdown_tables(markdown);
-    Ok(tables)
-}
+/// Parses HTML to find `<table>` elements and extracts their structure
+/// into `Table` objects with cells and markdown representation.
+fn extract_html_tables(html: &str) -> Result<Vec<Table>> {
+    let document = Html::parse_document(html);
+    let table_selector = Selector::parse("table")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse table selector: {}", e)))?;
+    let row_selector = Selector::parse("tr")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse row selector: {}", e)))?;
+    let header_selector = Selector::parse("th")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse header selector: {}", e)))?;
+    let cell_selector = Selector::parse("td")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse cell selector: {}", e)))?;
 
-/// Parse markdown tables from HTML-converted markdown.
-///
-/// Extracts table data from markdown pipe-delimited format.
-/// This maintains the existing Table structure API.
-fn parse_markdown_tables(markdown: &str) -> Vec<Table> {
     let mut tables = Vec::new();
-    let mut table_index = 0;
-    let lines: Vec<&str> = markdown.lines().collect();
-    let mut i = 0;
-
-    while i < lines.len() {
-        if lines[i].trim_start().starts_with('|')
-            && let Some((cells, end_idx)) = extract_markdown_table(&lines, i)
-            && !cells.is_empty()
-        {
-            let markdown_table = reconstruct_markdown_table(&cells);
-            tables.push(Table {
-                cells,
-                markdown: markdown_table,
-                page_number: table_index + 1,
-            });
-            table_index += 1;
-            i = end_idx;
-            continue;
-        }
-        i += 1;
-    }
 
-    tables
-}
-
-/// Extract a single markdown table from lines.
-///
-/// Returns the parsed table cells and the index after the table ends.
-fn extract_markdown_table(lines: &[&str], start_idx: usize) -> Option<(Vec<Vec<String>>, usize)> {
-    let header_line = lines.get(start_idx)?;
-
-    if !header_line.trim_start().starts_with('|') {
-        return None;
-    }
-
-    let mut cells = Vec::new();
-    let mut i = start_idx;
-
-    if let Some(header_cells) = parse_markdown_table_row(header_line) {
-        cells.push(header_cells);
-        i += 1;
-    } else {
-        return None;
-    }
+    for (table_index, table_elem) in document.select(&table_selector).enumerate() {
+        let mut cells: Vec<Vec<String>> = Vec::new();
+
+        for row in table_elem.select(&row_selector) {
+            let mut row_cells = Vec::new();
+
+            // Try headers first (th elements)
+            let headers: Vec<_> = row.select(&header_selector).collect();
+            if !headers.is_empty() {
+                for header in headers {
+                    let text = header
+                        .text()
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                        .split_whitespace()
+                        .collect::<Vec<_>>()
+                        .join(" ");
+                    row_cells.push(text);
+                }
+            } else {
+                // Use data cells (td elements)
+                for cell in row.select(&cell_selector) {
+                    let text = cell
+                        .text()
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                        .split_whitespace()
+                        .collect::<Vec<_>>()
+                        .join(" ");
+                    row_cells.push(text);
+                }
+            }
 
-    if i < lines.len() {
-        let sep_line = lines[i];
-        if is_markdown_table_separator(sep_line) {
-            i += 1;
+            if !row_cells.is_empty() {
+                cells.push(row_cells);
+            }
         }
-    }
 
-    while i < lines.len() {
-        let line = lines[i];
-        if let Some(row_cells) = parse_markdown_table_row(line) {
-            cells.push(row_cells);
-            i += 1;
-        } else if !line.trim_start().starts_with('|') {
-            break;
-        } else {
-            i += 1;
+        // Only create a table if it has content
+        if !cells.is_empty() {
+            let markdown = cells_to_markdown(&cells);
+            tables.push(Table {
+                cells,
+                markdown,
+                page_number: table_index + 1, // 1-indexed
+            });
         }
     }
 
-    if cells.len() > 1 { Some((cells, i)) } else { None }
-}
-
-/// Parse a single markdown table row into cell contents.
-fn parse_markdown_table_row(line: &str) -> Option<Vec<String>> {
-    let trimmed = line.trim_start();
-
-    if !trimmed.starts_with('|') || !trimmed.contains('|') {
-        return None;
-    }
-
-    let cells: Vec<String> = trimmed
-        .split('|')
-        .skip(1)
-        .map(|cell| cell.trim().to_string())
-        .filter(|cell| !cell.is_empty())
-        .collect();
-
-    if cells.is_empty() { None } else { Some(cells) }
-}
-
-/// Check if a line is a markdown table separator.
-fn is_markdown_table_separator(line: &str) -> bool {
-    let trimmed = line.trim_start();
-    if !trimmed.starts_with('|') {
-        return false;
-    }
-
-    trimmed
-        .split('|')
-        .all(|cell| cell.trim().chars().all(|c| c == '-' || c == ':' || c.is_whitespace()))
+    Ok(tables)
 }
 
-/// Reconstruct markdown table from cells.
+/// Convert table cells to markdown format.
+///
+/// Reuses the same logic as DOCX extractor for consistency.
+/// First row is treated as header, remaining rows as data.
 ///
-/// Takes parsed table cells and creates a properly formatted markdown table string.
-fn reconstruct_markdown_table(cells: &[Vec<String>]) -> String {
+/// # Arguments
+/// * `cells` - 2D vector of cell strings (rows × columns)
+///
+/// # Returns
+/// * `String` - Markdown formatted table
+fn cells_to_markdown(cells: &[Vec<String>]) -> String {
     if cells.is_empty() {
         return String::new();
     }
 
     let mut markdown = String::new();
 
-    for (row_idx, row) in cells.iter().enumerate() {
+    // Determine number of columns from first row
+    let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
+    if num_cols == 0 {
+        return String::new();
+    }
+
+    // Header row (first row)
+    if let Some(header) = cells.first() {
+        markdown.push_str("| ");
+        for cell in header {
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+
+        // Separator row
         markdown.push('|');
-        for cell in row {
-            markdown.push(' ');
-            markdown.push_str(cell);
-            markdown.push(' ');
-            markdown.push('|');
+        for _ in 0..num_cols {
+            markdown.push_str("------|");
         }
         markdown.push('\n');
+    }
 
-        if row_idx == 0 {
-            markdown.push('|');
-            for _ in row {
-                markdown.push_str("------|");
+    // Data rows (skip first row as it's the header)
+    for row in cells.iter().skip(1) {
+        markdown.push_str("| ");
+        for (idx, cell) in row.iter().enumerate() {
+            if idx >= num_cols {
+                break; // Handle irregular tables
             }
-            markdown.push('\n');
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
         }
+        // Pad with empty cells if row is shorter than expected
+        for _ in row.len()..num_cols {
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
     }
 
     markdown
@@ -196,18 +174,24 @@ impl Plugin for HtmlExtractor {
     }
 }
 
-impl SyncExtractor for HtmlExtractor {
-    fn extract_sync(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
-        let html = utf8_validation::from_utf8(content)
+#[async_trait]
+impl DocumentExtractor for HtmlExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let html = std::str::from_utf8(content)
             .map(|s| s.to_string())
             .unwrap_or_else(|_| String::from_utf8_lossy(content).to_string());
 
-        let (markdown, html_metadata) =
-            crate::extraction::html::convert_html_to_markdown_with_metadata(&html, config.html_options.clone())?;
+        // Extract tables from HTML
+        let tables = extract_html_tables(&html)?;
 
-        let tables = extract_html_tables(&markdown)?;
+        let markdown = crate::extraction::html::convert_html_to_markdown(&html, config.html_options.clone())?;
 
-        let content_without_frontmatter = markdown;
+        let (html_metadata, content_without_frontmatter) = crate::extraction::html::parse_html_metadata(&markdown)?;
 
         Ok(ExtractionResult {
             content: content_without_frontmatter,
@@ -216,41 +200,13 @@ impl SyncExtractor for HtmlExtractor {
                 format: html_metadata.map(|m| crate::types::FormatMetadata::Html(Box::new(m))),
                 ..Default::default()
             },
-            pages: None,
             tables,
             detected_languages: None,
             chunks: None,
             images: None,
         })
     }
-}
 
-#[async_trait]
-impl DocumentExtractor for HtmlExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
-    async fn extract_bytes(
-        &self,
-        content: &[u8],
-        mime_type: &str,
-        config: &ExtractionConfig,
-    ) -> Result<ExtractionResult> {
-        self.extract_sync(content, mime_type, config)
-    }
-
-    #[cfg(feature = "tokio-runtime")]
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
-    #[cfg(feature = "tokio-runtime")]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await
@@ -263,21 +219,12 @@ impl DocumentExtractor for HtmlExtractor {
     fn priority(&self) -> i32 {
         50
     }
-
-    fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
-        Some(self)
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    /// Helper function to convert HTML to markdown for testing
-    fn html_to_markdown_for_test(html: &str) -> String {
-        crate::extraction::html::convert_html_to_markdown(html, None).unwrap()
-    }
-
     #[test]
     fn test_html_extractor_plugin_interface() {
         let extractor = HtmlExtractor::new();
@@ -305,8 +252,7 @@ mod tests {
             </table>
         "#;
 
-        let markdown = html_to_markdown_for_test(html);
-        let tables = extract_html_tables(&markdown).unwrap();
+        let tables = extract_html_tables(html).unwrap();
         assert_eq!(tables.len(), 1);
 
         let table = &tables[0];
@@ -316,6 +262,7 @@ mod tests {
         assert_eq!(table.cells[2], vec!["Row2Col1", "Row2Col2"]);
         assert_eq!(table.page_number, 1);
 
+        // Check markdown format
         assert!(table.markdown.contains("| Header1 | Header2 |"));
         assert!(table.markdown.contains("|------|------|"));
         assert!(table.markdown.contains("| Row1Col1 | Row1Col2 |"));
@@ -335,8 +282,7 @@ mod tests {
             </table>
         "#;
 
-        let markdown = html_to_markdown_for_test(html);
-        let tables = extract_html_tables(&markdown).unwrap();
+        let tables = extract_html_tables(html).unwrap();
         assert_eq!(tables.len(), 2);
         assert_eq!(tables[0].page_number, 1);
         assert_eq!(tables[1].page_number, 2);
@@ -351,8 +297,7 @@ mod tests {
             </table>
         "#;
 
-        let markdown = html_to_markdown_for_test(html);
-        let tables = extract_html_tables(&markdown).unwrap();
+        let tables = extract_html_tables(html).unwrap();
         assert_eq!(tables.len(), 1);
 
         let table = &tables[0];
@@ -364,8 +309,7 @@ mod tests {
     #[test]
     fn test_extract_html_tables_empty() {
         let html = "<p>No tables here</p>";
-        let markdown = html_to_markdown_for_test(html);
-        let tables = extract_html_tables(&markdown).unwrap();
+        let tables = extract_html_tables(html).unwrap();
         assert_eq!(tables.len(), 0);
     }
 
@@ -378,13 +322,60 @@ mod tests {
             </table>
         "#;
 
-        let markdown = html_to_markdown_for_test(html);
-        let tables = extract_html_tables(&markdown).unwrap();
+        let tables = extract_html_tables(html).unwrap();
         assert_eq!(tables.len(), 1);
 
         let table = &tables[0];
-        assert_eq!(table.cells[0][0], "Header **Bold**");
-        assert_eq!(table.cells[1][0], "Data with *emphasis*");
+        // Whitespace is normalized during text extraction
+        assert_eq!(table.cells[0][0], "Header Bold");
+        assert_eq!(table.cells[1][0], "Data with emphasis");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_basic() {
+        let cells = vec![
+            vec!["Header1".to_string(), "Header2".to_string()],
+            vec!["Row1Col1".to_string(), "Row1Col2".to_string()],
+            vec!["Row2Col1".to_string(), "Row2Col2".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        assert!(markdown.contains("| Header1 | Header2 |"));
+        assert!(markdown.contains("|------|------|"));
+        assert!(markdown.contains("| Row1Col1 | Row1Col2 |"));
+        assert!(markdown.contains("| Row2Col1 | Row2Col2 |"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_empty() {
+        let cells: Vec<Vec<String>> = vec![];
+        let markdown = cells_to_markdown(&cells);
+        assert_eq!(markdown, "");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_escape_pipes() {
+        let cells = vec![vec!["Header".to_string()], vec!["Cell with | pipe".to_string()]];
+
+        let markdown = cells_to_markdown(&cells);
+        assert!(markdown.contains("Cell with \\| pipe"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_irregular_rows() {
+        let cells = vec![
+            vec!["H1".to_string(), "H2".to_string(), "H3".to_string()],
+            vec!["R1C1".to_string(), "R1C2".to_string()], // Missing third column
+            vec!["R2C1".to_string(), "R2C2".to_string(), "R2C3".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        // Should have 3 columns in header
+        assert!(markdown.contains("| H1 | H2 | H3 |"));
+        // Should pad short rows
+        assert!(markdown.contains("| R1C1 | R1C2 |  |"));
     }
 
     #[tokio::test]

data/vendor/kreuzberg/src/extractors/image.rs

@@ -20,14 +20,9 @@ impl ImageExtractor {
         Self
     }
 
-    /// Extract text from image using OCR with optional page tracking for multi-frame TIFFs.
+    /// Extract text from image using OCR.
     #[cfg(feature = "ocr")]
-    async fn extract_with_ocr(
-        &self,
-        content: &[u8],
-        mime_type: &str,
-        config: &ExtractionConfig,
-    ) -> Result<ExtractionResult> {
+    async fn extract_with_ocr(&self, content: &[u8], config: &ExtractionConfig) -> Result<ExtractionResult> {
         use crate::plugins::registry::get_ocr_backend_registry;
 
         let ocr_config = config.ocr.as_ref().ok_or_else(|| crate::KreuzbergError::Parsing {
@@ -44,21 +39,8 @@ impl ImageExtractor {
             registry.get(&ocr_config.backend)?
         };
 
-        let ocr_result = backend.process_image(content, ocr_config).await?;
-
-        let ocr_text = ocr_result.content.clone();
-        let ocr_extraction_result = crate::extraction::image::extract_text_from_image_with_ocr(
-            content,
-            mime_type,
-            ocr_text,
-            config.pages.as_ref(),
-        )?;
-
-        let mut result = ocr_result;
-        result.content = ocr_extraction_result.content;
-        result.pages = ocr_extraction_result.page_contents;
-
-        Ok(result)
+        // Process image using the backend - returns full ExtractionResult with tables/metadata
+        backend.process_image(content, ocr_config).await
     }
 }
 
@@ -96,13 +78,6 @@ impl Plugin for ImageExtractor {
 
 #[async_trait]
 impl DocumentExtractor for ImageExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -118,11 +93,13 @@ impl DocumentExtractor for ImageExtractor {
             exif: extraction_metadata.exif_data,
         };
 
+        // If OCR is enabled, use OCR result (which includes tables and OCR-specific metadata)
         if config.ocr.is_some() {
             #[cfg(feature = "ocr")]
             {
-                let mut ocr_result = self.extract_with_ocr(content, mime_type, config).await?;
+                let mut ocr_result = self.extract_with_ocr(content, config).await?;
 
+                // Add image metadata to the OCR result
                 ocr_result.metadata.format = Some(crate::types::FormatMetadata::Image(image_metadata));
                 ocr_result.mime_type = mime_type.to_string();
 
@@ -142,7 +119,6 @@ impl DocumentExtractor for ImageExtractor {
                         format: Some(crate::types::FormatMetadata::Image(image_metadata)),
                         ..Default::default()
                     },
-                    pages: None,
                     tables: vec![],
                     detected_languages: None,
                     chunks: None,
@@ -151,6 +127,7 @@ impl DocumentExtractor for ImageExtractor {
             }
         }
 
+        // No OCR - just return image dimensions
         Ok(ExtractionResult {
             content: format!(
                 "Image: {} {}x{}",
@@ -161,7 +138,6 @@ impl DocumentExtractor for ImageExtractor {
                 format: Some(crate::types::FormatMetadata::Image(image_metadata)),
                 ..Default::default()
             },
-            pages: None,
             tables: vec![],
             detected_languages: None,
             chunks: None,