kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/mod.rs

@@ -4,69 +4,13 @@
 //! All extractors implement the `DocumentExtractor` plugin trait.
 
 use crate::Result;
-use crate::core::config::ExtractionConfig;
 use crate::plugins::registry::get_document_extractor_registry;
-use crate::types::ExtractionResult;
 use once_cell::sync::Lazy;
 use std::sync::Arc;
 
-/// Trait for extractors that can work synchronously (WASM-compatible).
-///
-/// This trait defines the synchronous extraction interface for WASM targets and other
-/// environments where async/tokio runtimes are not available or desirable.
-///
-/// # Implementation
-///
-/// Extractors that need to support WASM should implement this trait in addition to
-/// the async `DocumentExtractor` trait. This allows the same extractor to work in both
-/// environments by delegating to the sync implementation.
-///
-/// # MIME Type Validation
-///
-/// The `mime_type` parameter is guaranteed to be already validated.
-///
-/// # Example
-///
-/// ```rust,ignore
-/// impl SyncExtractor for PlainTextExtractor {
-///     fn extract_sync(&self, content: &[u8], config: &ExtractionConfig) -> Result<ExtractionResult> {
-///         let text = String::from_utf8_lossy(content).to_string();
-///         Ok(ExtractionResult {
-///             content: text,
-///             mime_type: "text/plain".to_string(),
-///             metadata: Metadata::default(),
-///             tables: vec![],
-///             detected_languages: None,
-///             chunks: None,
-///             images: None,
-///         })
-///     }
-/// }
-/// ```
-pub trait SyncExtractor {
-    /// Extract content from a byte array synchronously.
-    ///
-    /// This method performs extraction without requiring an async runtime.
-    /// It is called by `extract_bytes_sync()` when the `tokio-runtime` feature is disabled.
-    ///
-    /// # Arguments
-    ///
-    /// * `content` - Raw document bytes
-    /// * `mime_type` - MIME type of the document (already validated)
-    /// * `config` - Extraction configuration
-    ///
-    /// # Returns
-    ///
-    /// An `ExtractionResult` containing the extracted content and metadata.
-    fn extract_sync(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult>;
-}
-
 pub mod structured;
 pub mod text;
 
-#[cfg(feature = "archives")]
-pub mod security;
-
 #[cfg(feature = "ocr")]
 pub mod image;
 
@@ -83,59 +27,20 @@ pub mod excel;
 pub mod html;
 
 #[cfg(feature = "office")]
-pub mod bibtex;
-
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
 pub mod docx;
 
 #[cfg(feature = "office")]
-pub mod epub;
-
-#[cfg(feature = "office")]
-pub mod fictionbook;
-
-#[cfg(feature = "office")]
-pub mod markdown;
-
-#[cfg(feature = "office")]
-pub mod rst;
-
-#[cfg(feature = "office")]
-pub mod latex;
-
-#[cfg(feature = "office")]
-pub mod jupyter;
-
-#[cfg(feature = "office")]
-pub mod orgmode;
-
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
-pub mod odt;
-
-#[cfg(feature = "office")]
-pub mod opml;
-
-#[cfg(feature = "office")]
-pub mod typst;
-
-#[cfg(feature = "xml")]
-pub mod jats;
+pub mod pandoc;
 
 #[cfg(feature = "pdf")]
 pub mod pdf;
 
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
-pub mod pptx;
-
 #[cfg(feature = "office")]
-pub mod rtf;
+pub mod pptx;
 
 #[cfg(feature = "xml")]
 pub mod xml;
 
-#[cfg(feature = "xml")]
-pub mod docbook;
-
 pub use structured::StructuredExtractor;
 pub use text::{MarkdownExtractor, PlainTextExtractor};
 
@@ -155,59 +60,20 @@ pub use excel::ExcelExtractor;
 pub use html::HtmlExtractor;
 
 #[cfg(feature = "office")]
-pub use bibtex::BibtexExtractor;
-
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
 pub use docx::DocxExtractor;
 
 #[cfg(feature = "office")]
-pub use epub::EpubExtractor;
-
-#[cfg(feature = "office")]
-pub use fictionbook::FictionBookExtractor;
-
-#[cfg(feature = "office")]
-pub use markdown::MarkdownExtractor as EnhancedMarkdownExtractor;
-
-#[cfg(feature = "office")]
-pub use rst::RstExtractor;
-
-#[cfg(feature = "office")]
-pub use latex::LatexExtractor;
-
-#[cfg(feature = "office")]
-pub use jupyter::JupyterExtractor;
-
-#[cfg(feature = "office")]
-pub use orgmode::OrgModeExtractor;
-
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
-pub use odt::OdtExtractor;
-
-#[cfg(feature = "xml")]
-pub use jats::JatsExtractor;
-
-#[cfg(feature = "office")]
-pub use opml::OpmlExtractor;
-
-#[cfg(feature = "office")]
-pub use typst::TypstExtractor;
+pub use pandoc::PandocExtractor;
 
 #[cfg(feature = "pdf")]
 pub use pdf::PdfExtractor;
 
-#[cfg(all(feature = "tokio-runtime", feature = "office"))]
-pub use pptx::PptxExtractor;
-
 #[cfg(feature = "office")]
-pub use rtf::RtfExtractor;
+pub use pptx::PptxExtractor;
 
 #[cfg(feature = "xml")]
 pub use xml::XmlExtractor;
 
-#[cfg(feature = "xml")]
-pub use docbook::DocbookExtractor;
-
 /// Lazy-initialized flag that ensures extractors are registered exactly once.
 ///
 /// This static is accessed on first extraction operation to automatically
@@ -220,6 +86,7 @@ static EXTRACTORS_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_default_ext
 /// It's safe to call multiple times - registration only happens once,
 /// unless the registry was cleared, in which case extractors are re-registered.
 pub fn ensure_initialized() -> Result<()> {
+    // First, try the lazy initialization
     EXTRACTORS_INITIALIZED
         .as_ref()
         .map(|_| ())
@@ -228,12 +95,15 @@ pub fn ensure_initialized() -> Result<()> {
             plugin_name: "built-in-extractors".to_string(),
         })?;
 
+    // Check if registry is empty (e.g., after clear_document_extractors)
+    // If so, re-register the default extractors
     let registry = get_document_extractor_registry();
     let registry_guard = registry
         .read()
         .map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
 
     if registry_guard.list().is_empty() {
+        // Drop read lock before acquiring write lock
         drop(registry_guard);
         register_default_extractors()?;
     }
@@ -282,25 +152,10 @@ pub fn register_default_extractors() -> Result<()> {
     registry.register(Arc::new(ExcelExtractor::new()))?;
 
     #[cfg(feature = "office")]
-    {
-        registry.register(Arc::new(EnhancedMarkdownExtractor::new()))?;
-        registry.register(Arc::new(BibtexExtractor::new()))?;
-        registry.register(Arc::new(EpubExtractor::new()))?;
-        registry.register(Arc::new(FictionBookExtractor::new()))?;
-        registry.register(Arc::new(RtfExtractor::new()))?;
-        registry.register(Arc::new(RstExtractor::new()))?;
-        registry.register(Arc::new(LatexExtractor::new()))?;
-        registry.register(Arc::new(JupyterExtractor::new()))?;
-        registry.register(Arc::new(OrgModeExtractor::new()))?;
-        registry.register(Arc::new(OpmlExtractor::new()))?;
-        registry.register(Arc::new(TypstExtractor::new()))?;
-    }
-
-    #[cfg(all(feature = "tokio-runtime", feature = "office"))]
     {
         registry.register(Arc::new(DocxExtractor::new()))?;
         registry.register(Arc::new(PptxExtractor::new()))?;
-        registry.register(Arc::new(OdtExtractor::new()))?;
+        registry.register(Arc::new(PandocExtractor::new()))?;
     }
 
     #[cfg(feature = "email")]
@@ -371,27 +226,11 @@ mod tests {
         }
 
         #[cfg(feature = "office")]
-        {
-            expected_count += 10;
-            assert!(extractor_names.contains(&"markdown-extractor".to_string()));
-            assert!(extractor_names.contains(&"bibtex-extractor".to_string()));
-            assert!(extractor_names.contains(&"epub-extractor".to_string()));
-            assert!(extractor_names.contains(&"fictionbook-extractor".to_string()));
-            assert!(extractor_names.contains(&"rtf-extractor".to_string()));
-            assert!(extractor_names.contains(&"rst-extractor".to_string()));
-            assert!(extractor_names.contains(&"latex-extractor".to_string()));
-            assert!(extractor_names.contains(&"jupyter-extractor".to_string()));
-            assert!(extractor_names.contains(&"orgmode-extractor".to_string()));
-            assert!(extractor_names.contains(&"opml-extractor".to_string()));
-            assert!(extractor_names.contains(&"typst-extractor".to_string()));
-        }
-
-        #[cfg(all(feature = "tokio-runtime", feature = "office"))]
         {
             expected_count += 3;
             assert!(extractor_names.contains(&"docx-extractor".to_string()));
             assert!(extractor_names.contains(&"pptx-extractor".to_string()));
-            assert!(extractor_names.contains(&"odt-extractor".to_string()));
+            assert!(extractor_names.contains(&"pandoc-extractor".to_string()));
         }
 
         #[cfg(feature = "email")]

data/vendor/kreuzberg/src/extractors/pandoc.rs

@@ -0,0 +1,201 @@
+//! Pandoc-based extractors for various document formats.
+//!
+//! Supports: DOCX, ODT, EPUB, LaTeX, RST, RTF, and many more formats via Pandoc.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::extraction::pandoc::extract_bytes_from_mime;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+
+/// Generic Pandoc extractor for all Pandoc-supported formats.
+///
+/// This extractor handles all document formats supported by Pandoc, including:
+/// - Microsoft Word (DOCX)
+/// - OpenDocument Text (ODT)
+/// - EPUB
+/// - LaTeX
+/// - reStructuredText (RST)
+/// - RTF
+/// - And many more
+pub struct PandocExtractor;
+
+impl PandocExtractor {
+    /// Create a new Pandoc extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for PandocExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for PandocExtractor {
+    fn name(&self) -> &str {
+        "pandoc-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from Pandoc-supported formats (DOCX, ODT, EPUB, LaTeX, RST, RTF, etc.)"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for PandocExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let pandoc_result = extract_bytes_from_mime(content, mime_type).await?;
+
+        let mut additional = std::collections::HashMap::new();
+        for (key, value) in pandoc_result.metadata {
+            additional.insert(key, value);
+        }
+
+        Ok(ExtractionResult {
+            content: pandoc_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "application/vnd.oasis.opendocument.text",
+            "application/epub+zip",
+            "application/x-latex",
+            "text/x-tex",
+            "text/x-rst",
+            "text/prs.fallenstein.rst",
+            "application/rtf",
+            "text/rtf",
+            "application/x-typst",
+            "application/x-ipynb+json",
+            "application/x-fictionbook+xml",
+            "text/x-org",
+            "text/x-commonmark",
+            "text/x-gfm",
+            "text/x-multimarkdown",
+            "text/x-markdown-extra",
+            "application/docbook+xml",
+            "application/x-jats+xml",
+            "application/x-opml+xml",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        40
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::extraction::pandoc::validate_pandoc_version;
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_plugin_interface() {
+        let extractor = PandocExtractor::new();
+        assert_eq!(extractor.name(), "pandoc-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.priority(), 40);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_docx() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_odt() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.oasis.opendocument.text")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_epub() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/epub+zip"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_latex() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/x-latex"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_rst() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"text/x-rst"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_markdown() {
+        if validate_pandoc_version().await.is_err() {
+            return;
+        }
+
+        let extractor = PandocExtractor::new();
+        let markdown = b"# Hello World\n\nThis is a test.";
+        let config = ExtractionConfig::default();
+
+        let result = extractor.extract_bytes(markdown, "text/x-rst", &config).await;
+
+        let _ = result;
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_default() {
+        let extractor = PandocExtractor;
+        assert_eq!(extractor.name(), "pandoc-extractor");
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_initialize_shutdown() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+}