kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/pandoc.rs

@@ -0,0 +1,201 @@
+//! Pandoc-based extractors for various document formats.
+//!
+//! Supports: DOCX, ODT, EPUB, LaTeX, RST, RTF, and many more formats via Pandoc.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::extraction::pandoc::extract_bytes_from_mime;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+
+/// Generic Pandoc extractor for all Pandoc-supported formats.
+///
+/// This extractor handles all document formats supported by Pandoc, including:
+/// - Microsoft Word (DOCX)
+/// - OpenDocument Text (ODT)
+/// - EPUB
+/// - LaTeX
+/// - reStructuredText (RST)
+/// - RTF
+/// - And many more
+pub struct PandocExtractor;
+
+impl PandocExtractor {
+    /// Create a new Pandoc extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for PandocExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for PandocExtractor {
+    fn name(&self) -> &str {
+        "pandoc-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from Pandoc-supported formats (DOCX, ODT, EPUB, LaTeX, RST, RTF, etc.)"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for PandocExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let pandoc_result = extract_bytes_from_mime(content, mime_type).await?;
+
+        let mut additional = std::collections::HashMap::new();
+        for (key, value) in pandoc_result.metadata {
+            additional.insert(key, value);
+        }
+
+        Ok(ExtractionResult {
+            content: pandoc_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "application/vnd.oasis.opendocument.text",
+            "application/epub+zip",
+            "application/x-latex",
+            "text/x-tex",
+            "text/x-rst",
+            "text/prs.fallenstein.rst",
+            "application/rtf",
+            "text/rtf",
+            "application/x-typst",
+            "application/x-ipynb+json",
+            "application/x-fictionbook+xml",
+            "text/x-org",
+            "text/x-commonmark",
+            "text/x-gfm",
+            "text/x-multimarkdown",
+            "text/x-markdown-extra",
+            "application/docbook+xml",
+            "application/x-jats+xml",
+            "application/x-opml+xml",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        40
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::extraction::pandoc::validate_pandoc_version;
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_plugin_interface() {
+        let extractor = PandocExtractor::new();
+        assert_eq!(extractor.name(), "pandoc-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.priority(), 40);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_docx() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_odt() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.oasis.opendocument.text")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_epub() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/epub+zip"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_latex() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/x-latex"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_rst() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"text/x-rst"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_markdown() {
+        if validate_pandoc_version().await.is_err() {
+            return;
+        }
+
+        let extractor = PandocExtractor::new();
+        let markdown = b"# Hello World\n\nThis is a test.";
+        let config = ExtractionConfig::default();
+
+        let result = extractor.extract_bytes(markdown, "text/x-rst", &config).await;
+
+        let _ = result;
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_default() {
+        let extractor = PandocExtractor;
+        assert_eq!(extractor.name(), "pandoc-extractor");
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_initialize_shutdown() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+}

data/vendor/kreuzberg/src/extractors/pdf.rs

@@ -147,25 +147,31 @@ fn extract_tables_from_document(
 
     let mut all_tables = Vec::new();
 
+    // Process each page
     for (page_index, page) in document.pages().iter().enumerate() {
-        let words = extract_words_from_page(&page, 0.0)?;
+        // Extract words with positions from the page
+        let words = extract_words_from_page(&page, 0.0)?; // Use 0.0 confidence for PDF (always high quality)
 
         if words.is_empty() {
             continue;
         }
 
+        // Use existing table reconstruction logic
+        // These thresholds match the defaults from TesseractConfig
         let column_threshold = 50;
         let row_threshold_ratio = 0.5;
 
+        // Reconstruct table from positioned words
         let table_cells = reconstruct_table(&words, column_threshold, row_threshold_ratio, true);
 
         if !table_cells.is_empty() {
+            // Generate markdown representation
             let markdown = table_to_markdown(&table_cells);
 
             all_tables.push(Table {
                 cells: table_cells,
                 markdown,
-                page_number: page_index + 1,
+                page_number: page_index + 1, // 1-indexed
             });
         }
     }
@@ -281,13 +287,6 @@ impl Plugin for PdfExtractor {
 
 #[async_trait]
 impl DocumentExtractor for PdfExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -296,10 +295,9 @@ impl DocumentExtractor for PdfExtractor {
     ) -> Result<ExtractionResult> {
         #[cfg(feature = "pdf")]
         let (pdf_metadata, native_text, tables) = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Move PDF extraction to blocking thread pool to enable parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || {
-                let _guard = span.entered();
                 let bindings = Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path("./"))
                     .or_else(|_| Pdfium::bind_to_system_library())
                     .map_err(|e| PdfError::MetadataExtractionFailed(format!("Failed to initialize Pdfium: {}", e)))?;
@@ -318,6 +316,7 @@ impl DocumentExtractor for PdfExtractor {
                 let metadata = crate::pdf::metadata::extract_metadata_from_document(&document)?;
                 let native_text = crate::pdf::text::extract_text_from_pdf_document(&document)?;
 
+                // Extract tables from native PDF text (when not using OCR)
                 let tables = extract_tables_from_document(&document, &metadata)?;
 
                 Ok::<_, crate::error::KreuzbergError>((metadata, native_text, tables))
@@ -325,6 +324,7 @@ impl DocumentExtractor for PdfExtractor {
             .await
             .map_err(|e| crate::error::KreuzbergError::Other(format!("PDF extraction task failed: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
             let bindings = Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path("./"))
                 .or_else(|_| Pdfium::bind_to_system_library())
                 .map_err(|e| PdfError::MetadataExtractionFailed(format!("Failed to initialize Pdfium: {}", e)))?;
@@ -343,6 +343,7 @@ impl DocumentExtractor for PdfExtractor {
             let metadata = crate::pdf::metadata::extract_metadata_from_document(&document)?;
             let native_text = crate::pdf::text::extract_text_from_pdf_document(&document)?;
 
+            // Extract tables from native PDF text (when not using OCR)
             let tables = extract_tables_from_document(&document, &metadata)?;
 
             (metadata, native_text, tables)
@@ -415,6 +416,9 @@ impl DocumentExtractor for PdfExtractor {
             None
         };
 
+        // Tables were extracted during metadata/text extraction phase
+        // (see extract_tables_from_document function below)
+
         Ok(ExtractionResult {
             content: text,
             mime_type: mime_type.to_string(),
@@ -430,7 +434,6 @@ impl DocumentExtractor for PdfExtractor {
         })
     }
 
-    #[cfg(feature = "tokio-runtime")]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await

data/vendor/kreuzberg/src/extractors/pptx.rs

@@ -43,10 +43,8 @@ impl PptxExtractor {
         for image in &mut images {
             let image_data = image.data.clone();
             let tess_config_clone = tess_config.clone();
-            let span = tracing::Span::current();
 
             let ocr_result = tokio::task::spawn_blocking(move || {
-                let _guard = span.entered();
                 let cache_dir = std::env::var("KREUZBERG_CACHE_DIR").ok().map(std::path::PathBuf::from);
 
                 let proc = OcrProcessor::new(cache_dir)?;
@@ -102,13 +100,6 @@ impl Plugin for PptxExtractor {
 
 #[async_trait]
 impl DocumentExtractor for PptxExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -117,16 +108,17 @@ impl DocumentExtractor for PptxExtractor {
     ) -> Result<ExtractionResult> {
         let extract_images = config.images.as_ref().is_some_and(|img| img.extract_images);
 
+        // Extract PPTX content
         let pptx_result = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
             let content_owned = content.to_vec();
-            let span = tracing::Span::current();
             tokio::task::spawn_blocking(move || {
-                let _guard = span.entered();
                 crate::extraction::pptx::extract_pptx_from_bytes(&content_owned, extract_images)
             })
             .await
             .map_err(|e| crate::error::KreuzbergError::parsing(format!("PPTX extraction task failed: {}", e)))??
         } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
             crate::extraction::pptx::extract_pptx_from_bytes(content, extract_images)?
         };
 
@@ -164,12 +156,6 @@ impl DocumentExtractor for PptxExtractor {
         })
     }
 
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let path_str = path
             .to_str()

data/vendor/kreuzberg/src/extractors/structured.rs

@@ -42,13 +42,6 @@ impl Plugin for StructuredExtractor {
 
 #[async_trait]
 impl DocumentExtractor for StructuredExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
@@ -87,13 +80,6 @@ impl DocumentExtractor for StructuredExtractor {
         })
     }
 
-    #[cfg(feature = "tokio-runtime")]
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, path, config),
-        fields(
-            extractor.name = self.name(),
-        )
-    ))]
     async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
         let bytes = tokio::fs::read(path).await?;
         self.extract_bytes(&bytes, mime_type, config).await

data/vendor/kreuzberg/src/extractors/text.rs

@@ -53,33 +53,22 @@ impl Plugin for PlainTextExtractor {
 
 #[async_trait]
 impl DocumentExtractor for PlainTextExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],
         mime_type: &str,
         _config: &ExtractionConfig,
     ) -> Result<ExtractionResult> {
-        let text = String::from_utf8_lossy(content).into_owned();
-        let text = text.trim_end_matches('\n').trim_end_matches('\r').to_string();
-        let line_count = text.lines().count();
-        let word_count = text.split_whitespace().count();
-        let character_count = text.len();
+        let text_result = parse_text(content, false)?;
 
         Ok(ExtractionResult {
-            content: text,
+            content: text_result.content,
             mime_type: mime_type.to_string(),
             metadata: crate::types::Metadata {
                 format: Some(crate::types::FormatMetadata::Text(crate::types::TextMetadata {
-                    line_count,
-                    word_count,
-                    character_count,
+                    line_count: text_result.line_count,
+                    word_count: text_result.word_count,
+                    character_count: text_result.character_count,
                     headers: None,
                     links: None,
                     code_blocks: None,
@@ -149,13 +138,6 @@ impl Plugin for MarkdownExtractor {
 
 #[async_trait]
 impl DocumentExtractor for MarkdownExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],

data/vendor/kreuzberg/src/extractors/xml.rs

@@ -53,13 +53,6 @@ impl Plugin for XmlExtractor {
 
 #[async_trait]
 impl DocumentExtractor for XmlExtractor {
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, content, _config),
-        fields(
-            extractor.name = self.name(),
-            content.size_bytes = content.len(),
-        )
-    ))]
     async fn extract_bytes(
         &self,
         content: &[u8],

data/vendor/kreuzberg/src/keywords/rake.rs

@@ -248,6 +248,7 @@ mod tests {
         let english_text = "Natural language processing is a subfield of artificial intelligence.";
         let config = KeywordConfig::rake().with_language("fr");
         let keywords = extract_keywords_rake(english_text, &config).unwrap();
+        dbg!(&keywords);
         assert!(
             !keywords.is_empty(),
             "Should fall back to English stopwords and extract keywords"

data/vendor/kreuzberg/src/lib.rs

@@ -39,7 +39,6 @@ pub mod core;
 pub mod error;
 pub mod extraction;
 pub mod extractors;
-pub mod panic_context;
 pub mod plugins;
 pub mod text;
 pub mod types;
@@ -80,9 +79,7 @@ pub mod pdf;
 pub use error::{KreuzbergError, Result};
 pub use types::*;
 
-#[cfg(feature = "tokio-runtime")]
-pub use core::extractor::{batch_extract_bytes, batch_extract_file};
-pub use core::extractor::{extract_bytes, extract_file};
+pub use core::extractor::{batch_extract_bytes, batch_extract_file, extract_bytes, extract_file};
 
 pub use core::extractor::{batch_extract_bytes_sync, batch_extract_file_sync, extract_bytes_sync, extract_file_sync};
 

data/vendor/kreuzberg/src/mcp/mod.rs

@@ -16,7 +16,7 @@
 //! use kreuzberg::mcp::start_mcp_server;
 //!
 //! #[tokio::main]
-//! async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+//! async fn main() -> anyhow::Result<()> {
 //!     start_mcp_server().await?;
 //!     Ok(())
 //! }

data/vendor/kreuzberg/src/mcp/server.rs

@@ -428,12 +428,12 @@ impl Default for KreuzbergMcp {
 /// use kreuzberg::mcp::start_mcp_server;
 ///
 /// #[tokio::main]
-/// async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+/// async fn main() -> anyhow::Result<()> {
 ///     start_mcp_server().await?;
 ///     Ok(())
 /// }
 /// ```
-pub async fn start_mcp_server() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+pub async fn start_mcp_server() -> Result<(), Box<dyn std::error::Error>> {
     let service = KreuzbergMcp::new()?.serve(stdio()).await?;
 
     service.waiting().await?;
@@ -444,9 +444,7 @@ pub async fn start_mcp_server() -> Result<(), Box<dyn std::error::Error + Send +
 ///
 /// This variant allows specifying a custom extraction configuration
 /// (e.g., loaded from a file) instead of using defaults.
-pub async fn start_mcp_server_with_config(
-    config: ExtractionConfig,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+pub async fn start_mcp_server_with_config(config: ExtractionConfig) -> Result<(), Box<dyn std::error::Error>> {
     let service = KreuzbergMcp::with_config(config).serve(stdio()).await?;
 
     service.waiting().await?;

data/vendor/kreuzberg/src/ocr/processor.rs

@@ -51,14 +51,6 @@ impl OcrProcessor {
         Ok(Self { cache })
     }
 
-    #[cfg_attr(feature = "otel", tracing::instrument(
-        skip(self, image_bytes),
-        fields(
-            ocr.backend = "tesseract",
-            ocr.language = %config.language,
-            image.size_bytes = image_bytes.len(),
-        )
-    ))]
     pub fn process_image(&self, image_bytes: &[u8], config: &TesseractConfig) -> Result<OcrExtractionResult, OcrError> {
         config.validate().map_err(OcrError::InvalidConfiguration)?;
 
@@ -72,14 +64,9 @@ impl OcrProcessor {
         if config.use_cache
             && let Some(cached_result) = self.cache.get_cached_result(&image_hash, "tesseract", &config_str)?
         {
-            #[cfg(feature = "otel")]
-            tracing::Span::current().record("cache.hit", true);
             return Ok(cached_result);
         }
 
-        #[cfg(feature = "otel")]
-        tracing::Span::current().record("cache.hit", false);
-
         let result = self.perform_ocr(image_bytes, config)?;
 
         if config.use_cache {
@@ -241,6 +228,7 @@ impl OcrProcessor {
         });
 
         // Validate language before initializing to prevent segfault ~keep
+        // tesseract-rs can crash on empty language or missing language files
         if config.language.trim().is_empty() {
             return Err(OcrError::TesseractInitializationFailed(
                 "Language cannot be empty. Please specify a valid language code (e.g., 'eng')".to_string(),
@@ -248,6 +236,7 @@ impl OcrProcessor {
         }
 
         // Validate language file exists before initializing to prevent segfault ~keep
+        // tesseract-rs can crash if language file is missing instead of returning error
         if !tessdata_path.is_empty() {
             let languages: Vec<&str> = config.language.split('+').collect();
             for lang in languages {
@@ -373,11 +362,6 @@ impl OcrProcessor {
             )
         });
 
-        api.recognize()
-            .map_err(|e| OcrError::ProcessingFailed(format!("Failed to recognize text: {}", e)))?;
-
-        log_ci_debug(ci_debug_enabled, "recognize", || "completed".to_string());
-
         let tsv_data_for_tables = if config.enable_table_detection || config.output_format == "tsv" {
             Some(
                 api.get_tsv_text(0)

data/vendor/kreuzberg/src/pdf/error.rs

@@ -40,7 +40,7 @@ impl std::error::Error for PdfError {}
 impl From<lopdf::Error> for PdfError {
     fn from(err: lopdf::Error) -> Self {
         match err {
-            lopdf::Error::IO(io_err) => PdfError::IOError(io_err.to_string()),
+            lopdf::Error::IO(_) => panic!("lopdf IO errors should not be converted to PdfError - let them bubble up"),
             _ => PdfError::InvalidPdf(err.to_string()),
         }
     }