kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/html.rs

@@ -5,10 +5,9 @@ use crate::core::config::ExtractionConfig;
 use crate::plugins::{DocumentExtractor, Plugin};
 use crate::types::{ExtractionResult, Metadata, Table};
 use async_trait::async_trait;
+use scraper::{Html, Selector};
 use std::path::Path;
 
-// NOTE: scraper dependency has been removed in favor of html-to-markdown-rs
-
 /// HTML document extractor using html-to-markdown.
 pub struct HtmlExtractor;
 
@@ -24,152 +23,134 @@ impl HtmlExtractor {
     }
 }
 
-/// Extract all tables from HTML content using html-to-markdown-rs.
-///
-/// Uses html-to-markdown-rs to convert HTML to Markdown, which preserves
-/// table structure in markdown format. Tables are then parsed from the
-/// resulting markdown to maintain compatibility with existing Table API.
+/// Extract all tables from HTML content.
 ///
-/// This approach eliminates the need for the `scraper` dependency as
-/// html-to-markdown-rs already handles all table parsing.
+/// Parses HTML to find `<table>` elements and extracts their structure
+/// into `Table` objects with cells and markdown representation.
 fn extract_html_tables(html: &str) -> Result<Vec<Table>> {
-    let markdown = crate::extraction::html::convert_html_to_markdown(html, None)?;
+    let document = Html::parse_document(html);
+    let table_selector = Selector::parse("table")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse table selector: {}", e)))?;
+    let row_selector = Selector::parse("tr")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse row selector: {}", e)))?;
+    let header_selector = Selector::parse("th")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse header selector: {}", e)))?;
+    let cell_selector = Selector::parse("td")
+        .map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse cell selector: {}", e)))?;
 
-    let tables = parse_markdown_tables(&markdown);
+    let mut tables = Vec::new();
 
-    Ok(tables)
-}
+    for (table_index, table_elem) in document.select(&table_selector).enumerate() {
+        let mut cells: Vec<Vec<String>> = Vec::new();
+
+        for row in table_elem.select(&row_selector) {
+            let mut row_cells = Vec::new();
+
+            // Try headers first (th elements)
+            let headers: Vec<_> = row.select(&header_selector).collect();
+            if !headers.is_empty() {
+                for header in headers {
+                    let text = header
+                        .text()
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                        .split_whitespace()
+                        .collect::<Vec<_>>()
+                        .join(" ");
+                    row_cells.push(text);
+                }
+            } else {
+                // Use data cells (td elements)
+                for cell in row.select(&cell_selector) {
+                    let text = cell
+                        .text()
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                        .split_whitespace()
+                        .collect::<Vec<_>>()
+                        .join(" ");
+                    row_cells.push(text);
+                }
+            }
 
-/// Parse markdown tables from HTML-converted markdown.
-///
-/// Extracts table data from markdown pipe-delimited format.
-/// This maintains the existing Table structure API.
-fn parse_markdown_tables(markdown: &str) -> Vec<Table> {
-    let mut tables = Vec::new();
-    let mut table_index = 0;
-    let lines: Vec<&str> = markdown.lines().collect();
-    let mut i = 0;
-
-    while i < lines.len() {
-        if lines[i].trim_start().starts_with('|')
-            && let Some((cells, end_idx)) = extract_markdown_table(&lines, i)
-            && !cells.is_empty()
-        {
-            let markdown_table = reconstruct_markdown_table(&cells);
+            if !row_cells.is_empty() {
+                cells.push(row_cells);
+            }
+        }
+
+        // Only create a table if it has content
+        if !cells.is_empty() {
+            let markdown = cells_to_markdown(&cells);
             tables.push(Table {
                 cells,
-                markdown: markdown_table,
-                page_number: table_index + 1,
+                markdown,
+                page_number: table_index + 1, // 1-indexed
             });
-            table_index += 1;
-            i = end_idx;
-            continue;
         }
-        i += 1;
     }
 
-    tables
+    Ok(tables)
 }
 
-/// Extract a single markdown table from lines.
+/// Convert table cells to markdown format.
 ///
-/// Returns the parsed table cells and the index after the table ends.
-fn extract_markdown_table(lines: &[&str], start_idx: usize) -> Option<(Vec<Vec<String>>, usize)> {
-    let header_line = lines.get(start_idx)?;
-
-    if !header_line.trim_start().starts_with('|') {
-        return None;
-    }
-
-    let mut cells = Vec::new();
-    let mut i = start_idx;
-
-    if let Some(header_cells) = parse_markdown_table_row(header_line) {
-        cells.push(header_cells);
-        i += 1;
-    } else {
-        return None;
-    }
-
-    if i < lines.len() {
-        let sep_line = lines[i];
-        if is_markdown_table_separator(sep_line) {
-            i += 1;
-        }
-    }
-
-    while i < lines.len() {
-        let line = lines[i];
-        if let Some(row_cells) = parse_markdown_table_row(line) {
-            cells.push(row_cells);
-            i += 1;
-        } else if !line.trim_start().starts_with('|') {
-            break;
-        } else {
-            i += 1;
-        }
-    }
-
-    if cells.len() > 1 { Some((cells, i)) } else { None }
-}
-
-/// Parse a single markdown table row into cell contents.
-fn parse_markdown_table_row(line: &str) -> Option<Vec<String>> {
-    let trimmed = line.trim_start();
-
-    if !trimmed.starts_with('|') || !trimmed.contains('|') {
-        return None;
-    }
-
-    let cells: Vec<String> = trimmed
-        .split('|')
-        .skip(1)
-        .map(|cell| cell.trim().to_string())
-        .filter(|cell| !cell.is_empty())
-        .collect();
-
-    if cells.is_empty() { None } else { Some(cells) }
-}
-
-/// Check if a line is a markdown table separator.
-fn is_markdown_table_separator(line: &str) -> bool {
-    let trimmed = line.trim_start();
-    if !trimmed.starts_with('|') {
-        return false;
-    }
-
-    trimmed
-        .split('|')
-        .all(|cell| cell.trim().chars().all(|c| c == '-' || c == ':' || c.is_whitespace()))
-}
-
-/// Reconstruct markdown table from cells.
+/// Reuses the same logic as DOCX extractor for consistency.
+/// First row is treated as header, remaining rows as data.
 ///
-/// Takes parsed table cells and creates a properly formatted markdown table string.
-fn reconstruct_markdown_table(cells: &[Vec<String>]) -> String {
+/// # Arguments
+/// * `cells` - 2D vector of cell strings (rows × columns)
+///
+/// # Returns
+/// * `String` - Markdown formatted table
+fn cells_to_markdown(cells: &[Vec<String>]) -> String {
     if cells.is_empty() {
         return String::new();
     }
 
     let mut markdown = String::new();
 
-    for (row_idx, row) in cells.iter().enumerate() {
+    // Determine number of columns from first row
+    let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
+    if num_cols == 0 {
+        return String::new();
+    }
+
+    // Header row (first row)
+    if let Some(header) = cells.first() {
+        markdown.push_str("| ");
+        for cell in header {
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        markdown.push('\n');
+
+        // Separator row
         markdown.push('|');
-        for cell in row {
-            markdown.push(' ');
-            markdown.push_str(cell);
-            markdown.push(' ');
-            markdown.push('|');
+        for _ in 0..num_cols {
+            markdown.push_str("------|");
         }
         markdown.push('\n');
+    }
 
-        if row_idx == 0 {
-            markdown.push('|');
-            for _ in row {
-                markdown.push_str("------|");
+    // Data rows (skip first row as it's the header)
+    for row in cells.iter().skip(1) {
+        markdown.push_str("| ");
+        for (idx, cell) in row.iter().enumerate() {
+            if idx >= num_cols {
+                break; // Handle irregular tables
             }
-            markdown.push('\n');
+            // Escape pipe characters in cell content
+            let escaped = cell.replace('|', "\\|");
+            markdown.push_str(&escaped);
+            markdown.push_str(" | ");
+        }
+        // Pad with empty cells if row is shorter than expected
+        for _ in row.len()..num_cols {
+            markdown.push_str(" | ");
         }
+        markdown.push('\n');
     }
 
     markdown
@@ -195,13 +176,6 @@ impl Plugin for HtmlExtractor {
 
 #[async_trait]
 impl DocumentExtractor for HtmlExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -212,6 +186,7 @@ impl DocumentExtractor for HtmlExtractor {
             .map(|s| s.to_string())
             .unwrap_or_else(|_| String::from_utf8_lossy(content).to_string());
 
+        // Extract tables from HTML
         let tables = extract_html_tables(&html)?;
 
         let markdown = crate::extraction::html::convert_html_to_markdown(&html, config.html_options.clone())?;
@@ -232,13 +207,6 @@ impl DocumentExtractor for HtmlExtractor {
         })
     }
 
-    #[cfg(feature = "tokio-runtime")]
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await
@@ -294,6 +262,7 @@ mod tests {
         assert_eq!(table.cells[2], vec!["Row2Col1", "Row2Col2"]);
         assert_eq!(table.page_number, 1);
 
+        // Check markdown format
         assert!(table.markdown.contains("| Header1 | Header2 |"));
         assert!(table.markdown.contains("|------|------|"));
         assert!(table.markdown.contains("| Row1Col1 | Row1Col2 |"));
@@ -357,8 +326,56 @@ mod tests {
         assert_eq!(tables.len(), 1);
 
         let table = &tables[0];
-        assert_eq!(table.cells[0][0], "Header **Bold**");
-        assert_eq!(table.cells[1][0], "Data with *emphasis*");
+        // Whitespace is normalized during text extraction
+        assert_eq!(table.cells[0][0], "Header Bold");
+        assert_eq!(table.cells[1][0], "Data with emphasis");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_basic() {
+        let cells = vec![
+            vec!["Header1".to_string(), "Header2".to_string()],
+            vec!["Row1Col1".to_string(), "Row1Col2".to_string()],
+            vec!["Row2Col1".to_string(), "Row2Col2".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        assert!(markdown.contains("| Header1 | Header2 |"));
+        assert!(markdown.contains("|------|------|"));
+        assert!(markdown.contains("| Row1Col1 | Row1Col2 |"));
+        assert!(markdown.contains("| Row2Col1 | Row2Col2 |"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_empty() {
+        let cells: Vec<Vec<String>> = vec![];
+        let markdown = cells_to_markdown(&cells);
+        assert_eq!(markdown, "");
+    }
+
+    #[test]
+    fn test_cells_to_markdown_escape_pipes() {
+        let cells = vec![vec!["Header".to_string()], vec!["Cell with | pipe".to_string()]];
+
+        let markdown = cells_to_markdown(&cells);
+        assert!(markdown.contains("Cell with \\| pipe"));
+    }
+
+    #[test]
+    fn test_cells_to_markdown_irregular_rows() {
+        let cells = vec![
+            vec!["H1".to_string(), "H2".to_string(), "H3".to_string()],
+            vec!["R1C1".to_string(), "R1C2".to_string()], // Missing third column
+            vec!["R2C1".to_string(), "R2C2".to_string(), "R2C3".to_string()],
+        ];
+
+        let markdown = cells_to_markdown(&cells);
+
+        // Should have 3 columns in header
+        assert!(markdown.contains("| H1 | H2 | H3 |"));
+        // Should pad short rows
+        assert!(markdown.contains("| R1C1 | R1C2 |  |"));
     }
 
     #[tokio::test]

data/vendor/kreuzberg/src/extractors/image.rs

@@ -39,6 +39,7 @@ impl ImageExtractor {
             registry.get(&ocr_config.backend)?
         };
 
+        // Process image using the backend - returns full ExtractionResult with tables/metadata
         backend.process_image(content, ocr_config).await
     }
 }
@@ -77,13 +78,6 @@ impl Plugin for ImageExtractor {
 
 #[async_trait]
 impl DocumentExtractor for ImageExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -99,11 +93,13 @@ impl DocumentExtractor for ImageExtractor {
             exif: extraction_metadata.exif_data,
         };
 
+        // If OCR is enabled, use OCR result (which includes tables and OCR-specific metadata)
         if config.ocr.is_some() {
             #[cfg(feature = "ocr")]
             {
                 let mut ocr_result = self.extract_with_ocr(content, config).await?;
 
+                // Add image metadata to the OCR result
                 ocr_result.metadata.format = Some(crate::types::FormatMetadata::Image(image_metadata));
                 ocr_result.mime_type = mime_type.to_string();
 
@@ -131,6 +127,7 @@ impl DocumentExtractor for ImageExtractor {
             }
         }
 
+        // No OCR - just return image dimensions
         Ok(ExtractionResult {
             content: format!(
                 "Image: {} {}x{}",

data/vendor/kreuzberg/src/extractors/mod.rs

@@ -8,7 +8,6 @@ use crate::plugins::registry::get_document_extractor_registry;
 use once_cell::sync::Lazy;
 use std::sync::Arc;
 
-pub mod security;
 pub mod structured;
 pub mod text;
 
@@ -27,44 +26,11 @@ pub mod excel;
 #[cfg(feature = "html")]
 pub mod html;
 
-#[cfg(feature = "office")]
-pub mod bibtex;
-
 #[cfg(feature = "office")]
 pub mod docx;
 
 #[cfg(feature = "office")]
-pub mod epub;
-
-#[cfg(feature = "office")]
-pub mod fictionbook;
-
-#[cfg(feature = "office")]
-pub mod markdown;
-
-#[cfg(feature = "office")]
-pub mod rst;
-
-#[cfg(feature = "office")]
-pub mod latex;
-
-#[cfg(feature = "office")]
-pub mod jupyter;
-
-#[cfg(feature = "office")]
-pub mod orgmode;
-
-#[cfg(feature = "office")]
-pub mod odt;
-
-#[cfg(feature = "office")]
-pub mod opml;
-
-#[cfg(feature = "office")]
-pub mod typst;
-
-#[cfg(feature = "xml")]
-pub mod jats;
+pub mod pandoc;
 
 #[cfg(feature = "pdf")]
 pub mod pdf;
@@ -72,15 +38,9 @@ pub mod pdf;
 #[cfg(feature = "office")]
 pub mod pptx;
 
-#[cfg(feature = "office")]
-pub mod rtf;
-
 #[cfg(feature = "xml")]
 pub mod xml;
 
-#[cfg(feature = "xml")]
-pub mod docbook;
-
 pub use structured::StructuredExtractor;
 pub use text::{MarkdownExtractor, PlainTextExtractor};
 
@@ -99,44 +59,11 @@ pub use excel::ExcelExtractor;
 #[cfg(feature = "html")]
 pub use html::HtmlExtractor;
 
-#[cfg(feature = "office")]
-pub use bibtex::BibtexExtractor;
-
 #[cfg(feature = "office")]
 pub use docx::DocxExtractor;
 
 #[cfg(feature = "office")]
-pub use epub::EpubExtractor;
-
-#[cfg(feature = "office")]
-pub use fictionbook::FictionBookExtractor;
-
-#[cfg(feature = "office")]
-pub use markdown::MarkdownExtractor as EnhancedMarkdownExtractor;
-
-#[cfg(feature = "office")]
-pub use rst::RstExtractor;
-
-#[cfg(feature = "office")]
-pub use latex::LatexExtractor;
-
-#[cfg(feature = "office")]
-pub use jupyter::JupyterExtractor;
-
-#[cfg(feature = "office")]
-pub use orgmode::OrgModeExtractor;
-
-#[cfg(feature = "office")]
-pub use odt::OdtExtractor;
-
-#[cfg(feature = "xml")]
-pub use jats::JatsExtractor;
-
-#[cfg(feature = "office")]
-pub use opml::OpmlExtractor;
-
-#[cfg(feature = "office")]
-pub use typst::TypstExtractor;
+pub use pandoc::PandocExtractor;
 
 #[cfg(feature = "pdf")]
 pub use pdf::PdfExtractor;
@@ -144,15 +71,9 @@ pub use pdf::PdfExtractor;
 #[cfg(feature = "office")]
 pub use pptx::PptxExtractor;
 
-#[cfg(feature = "office")]
-pub use rtf::RtfExtractor;
-
 #[cfg(feature = "xml")]
 pub use xml::XmlExtractor;
 
-#[cfg(feature = "xml")]
-pub use docbook::DocbookExtractor;
-
 /// Lazy-initialized flag that ensures extractors are registered exactly once.
 ///
 /// This static is accessed on first extraction operation to automatically
@@ -165,6 +86,7 @@ static EXTRACTORS_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_default_ext
 /// It's safe to call multiple times - registration only happens once,
 /// unless the registry was cleared, in which case extractors are re-registered.
 pub fn ensure_initialized() -> Result<()> {
+    // First, try the lazy initialization
     EXTRACTORS_INITIALIZED
         .as_ref()
         .map(|_| ())
@@ -173,12 +95,15 @@ pub fn ensure_initialized() -> Result<()> {
             plugin_name: "built-in-extractors".to_string(),
         })?;
 
+    // Check if registry is empty (e.g., after clear_document_extractors)
+    // If so, re-register the default extractors
     let registry = get_document_extractor_registry();
     let registry_guard = registry
         .read()
         .map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
 
     if registry_guard.list().is_empty() {
+        // Drop read lock before acquiring write lock
         drop(registry_guard);
         register_default_extractors()?;
     }
@@ -228,20 +153,9 @@ pub fn register_default_extractors() -> Result<()> {
 
     #[cfg(feature = "office")]
     {
-        registry.register(Arc::new(EnhancedMarkdownExtractor::new()))?;
-        registry.register(Arc::new(BibtexExtractor::new()))?;
         registry.register(Arc::new(DocxExtractor::new()))?;
-        registry.register(Arc::new(EpubExtractor::new()))?;
-        registry.register(Arc::new(FictionBookExtractor::new()))?;
         registry.register(Arc::new(PptxExtractor::new()))?;
-        registry.register(Arc::new(OdtExtractor::new()))?;
-        registry.register(Arc::new(RtfExtractor::new()))?;
-        registry.register(Arc::new(RstExtractor::new()))?;
-        registry.register(Arc::new(LatexExtractor::new()))?;
-        registry.register(Arc::new(JupyterExtractor::new()))?;
-        registry.register(Arc::new(OrgModeExtractor::new()))?;
-        registry.register(Arc::new(OpmlExtractor::new()))?;
-        registry.register(Arc::new(TypstExtractor::new()))?;
+        registry.register(Arc::new(PandocExtractor::new()))?;
     }
 
     #[cfg(feature = "email")]
@@ -313,21 +227,10 @@ mod tests {
 
         #[cfg(feature = "office")]
         {
-            expected_count += 13;
-            assert!(extractor_names.contains(&"markdown-extractor".to_string()));
-            assert!(extractor_names.contains(&"bibtex-extractor".to_string()));
+            expected_count += 3;
             assert!(extractor_names.contains(&"docx-extractor".to_string()));
-            assert!(extractor_names.contains(&"epub-extractor".to_string()));
-            assert!(extractor_names.contains(&"fictionbook-extractor".to_string()));
             assert!(extractor_names.contains(&"pptx-extractor".to_string()));
-            assert!(extractor_names.contains(&"odt-extractor".to_string()));
-            assert!(extractor_names.contains(&"rtf-extractor".to_string()));
-            assert!(extractor_names.contains(&"rst-extractor".to_string()));
-            assert!(extractor_names.contains(&"latex-extractor".to_string()));
-            assert!(extractor_names.contains(&"jupyter-extractor".to_string()));
-            assert!(extractor_names.contains(&"orgmode-extractor".to_string()));
-            assert!(extractor_names.contains(&"opml-extractor".to_string()));
-            assert!(extractor_names.contains(&"typst-extractor".to_string()));
+            assert!(extractor_names.contains(&"pandoc-extractor".to_string()));
         }
 
         #[cfg(feature = "email")]