kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/mod.rs

@@ -0,0 +1,268 @@
+//! Built-in document extractors.
+//!
+//! This module contains the default extractors that ship with Kreuzberg.
+//! All extractors implement the `DocumentExtractor` plugin trait.
+
+use crate::Result;
+use crate::plugins::registry::get_document_extractor_registry;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
+
+pub mod structured;
+pub mod text;
+
+#[cfg(feature = "ocr")]
+pub mod image;
+
+#[cfg(feature = "archives")]
+pub mod archive;
+
+#[cfg(feature = "email")]
+pub mod email;
+
+#[cfg(feature = "excel")]
+pub mod excel;
+
+#[cfg(feature = "html")]
+pub mod html;
+
+#[cfg(feature = "office")]
+pub mod docx;
+
+#[cfg(feature = "office")]
+pub mod pandoc;
+
+#[cfg(feature = "pdf")]
+pub mod pdf;
+
+#[cfg(feature = "office")]
+pub mod pptx;
+
+#[cfg(feature = "xml")]
+pub mod xml;
+
+pub use structured::StructuredExtractor;
+pub use text::{MarkdownExtractor, PlainTextExtractor};
+
+#[cfg(feature = "ocr")]
+pub use image::ImageExtractor;
+
+#[cfg(feature = "archives")]
+pub use archive::{SevenZExtractor, TarExtractor, ZipExtractor};
+
+#[cfg(feature = "email")]
+pub use email::EmailExtractor;
+
+#[cfg(feature = "excel")]
+pub use excel::ExcelExtractor;
+
+#[cfg(feature = "html")]
+pub use html::HtmlExtractor;
+
+#[cfg(feature = "office")]
+pub use docx::DocxExtractor;
+
+#[cfg(feature = "office")]
+pub use pandoc::PandocExtractor;
+
+#[cfg(feature = "pdf")]
+pub use pdf::PdfExtractor;
+
+#[cfg(feature = "office")]
+pub use pptx::PptxExtractor;
+
+#[cfg(feature = "xml")]
+pub use xml::XmlExtractor;
+
+/// Lazy-initialized flag that ensures extractors are registered exactly once.
+///
+/// This static is accessed on first extraction operation to automatically
+/// register all built-in extractors with the plugin registry.
+static EXTRACTORS_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_default_extractors);
+
+/// Ensure built-in extractors are registered.
+///
+/// This function is called automatically on first extraction operation.
+/// It's safe to call multiple times - registration only happens once,
+/// unless the registry was cleared, in which case extractors are re-registered.
+pub fn ensure_initialized() -> Result<()> {
+    // First, try the lazy initialization
+    EXTRACTORS_INITIALIZED
+        .as_ref()
+        .map(|_| ())
+        .map_err(|e| crate::KreuzbergError::Plugin {
+            message: format!("Failed to register default extractors: {}", e),
+            plugin_name: "built-in-extractors".to_string(),
+        })?;
+
+    // Check if registry is empty (e.g., after clear_document_extractors)
+    // If so, re-register the default extractors
+    let registry = get_document_extractor_registry();
+    let registry_guard = registry
+        .read()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
+
+    if registry_guard.list().is_empty() {
+        // Drop read lock before acquiring write lock
+        drop(registry_guard);
+        register_default_extractors()?;
+    }
+
+    Ok(())
+}
+
+/// Register all built-in extractors with the global registry.
+///
+/// This function should be called once at application startup to register
+/// the default extractors (PlainText, Markdown, XML, etc.).
+///
+/// **Note:** This is called automatically on first extraction operation.
+/// Explicit calling is optional.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::extractors::register_default_extractors;
+///
+/// # fn main() -> kreuzberg::Result<()> {
+/// register_default_extractors()?;
+/// # Ok(())
+/// # }
+/// ```
+pub fn register_default_extractors() -> Result<()> {
+    let registry = get_document_extractor_registry();
+    let mut registry = registry
+        .write()
+        .map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
+
+    registry.register(Arc::new(PlainTextExtractor::new()))?;
+    registry.register(Arc::new(MarkdownExtractor::new()))?;
+    registry.register(Arc::new(StructuredExtractor::new()))?;
+
+    #[cfg(feature = "ocr")]
+    registry.register(Arc::new(ImageExtractor::new()))?;
+
+    #[cfg(feature = "xml")]
+    registry.register(Arc::new(XmlExtractor::new()))?;
+
+    #[cfg(feature = "pdf")]
+    registry.register(Arc::new(PdfExtractor::new()))?;
+
+    #[cfg(feature = "excel")]
+    registry.register(Arc::new(ExcelExtractor::new()))?;
+
+    #[cfg(feature = "office")]
+    {
+        registry.register(Arc::new(DocxExtractor::new()))?;
+        registry.register(Arc::new(PptxExtractor::new()))?;
+        registry.register(Arc::new(PandocExtractor::new()))?;
+    }
+
+    #[cfg(feature = "email")]
+    registry.register(Arc::new(EmailExtractor::new()))?;
+
+    #[cfg(feature = "html")]
+    registry.register(Arc::new(HtmlExtractor::new()))?;
+
+    #[cfg(feature = "archives")]
+    {
+        registry.register(Arc::new(ZipExtractor::new()))?;
+        registry.register(Arc::new(TarExtractor::new()))?;
+        registry.register(Arc::new(SevenZExtractor::new()))?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_register_default_extractors() {
+        let registry = get_document_extractor_registry();
+        {
+            let mut reg = registry
+                .write()
+                .expect("Failed to acquire write lock on registry in test");
+            *reg = crate::plugins::registry::DocumentExtractorRegistry::new();
+        }
+
+        register_default_extractors().expect("Failed to register extractors");
+
+        let reg = registry
+            .read()
+            .expect("Failed to acquire read lock on registry in test");
+        let extractor_names = reg.list();
+
+        #[allow(unused_mut)]
+        let mut expected_count = 3;
+        assert!(extractor_names.contains(&"plain-text-extractor".to_string()));
+        assert!(extractor_names.contains(&"markdown-extractor".to_string()));
+        assert!(extractor_names.contains(&"structured-extractor".to_string()));
+
+        #[cfg(feature = "ocr")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"image-extractor".to_string()));
+        }
+
+        #[cfg(feature = "xml")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"xml-extractor".to_string()));
+        }
+
+        #[cfg(feature = "pdf")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"pdf-extractor".to_string()));
+        }
+
+        #[cfg(feature = "excel")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"excel-extractor".to_string()));
+        }
+
+        #[cfg(feature = "office")]
+        {
+            expected_count += 3;
+            assert!(extractor_names.contains(&"docx-extractor".to_string()));
+            assert!(extractor_names.contains(&"pptx-extractor".to_string()));
+            assert!(extractor_names.contains(&"pandoc-extractor".to_string()));
+        }
+
+        #[cfg(feature = "email")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"email-extractor".to_string()));
+        }
+
+        #[cfg(feature = "html")]
+        {
+            expected_count += 1;
+            assert!(extractor_names.contains(&"html-extractor".to_string()));
+        }
+
+        #[cfg(feature = "archives")]
+        {
+            expected_count += 3;
+            assert!(extractor_names.contains(&"zip-extractor".to_string()));
+            assert!(extractor_names.contains(&"tar-extractor".to_string()));
+            assert!(extractor_names.contains(&"7z-extractor".to_string()));
+        }
+
+        assert_eq!(
+            extractor_names.len(),
+            expected_count,
+            "Expected {} extractors based on enabled features",
+            expected_count
+        );
+    }
+
+    #[test]
+    fn test_ensure_initialized() {
+        ensure_initialized().expect("Failed to ensure extractors initialized");
+    }
+}

data/vendor/kreuzberg/src/extractors/pandoc.rs

@@ -0,0 +1,201 @@
+//! Pandoc-based extractors for various document formats.
+//!
+//! Supports: DOCX, ODT, EPUB, LaTeX, RST, RTF, and many more formats via Pandoc.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::extraction::pandoc::extract_bytes_from_mime;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+
+/// Generic Pandoc extractor for all Pandoc-supported formats.
+///
+/// This extractor handles all document formats supported by Pandoc, including:
+/// - Microsoft Word (DOCX)
+/// - OpenDocument Text (ODT)
+/// - EPUB
+/// - LaTeX
+/// - reStructuredText (RST)
+/// - RTF
+/// - And many more
+pub struct PandocExtractor;
+
+impl PandocExtractor {
+    /// Create a new Pandoc extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for PandocExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for PandocExtractor {
+    fn name(&self) -> &str {
+        "pandoc-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from Pandoc-supported formats (DOCX, ODT, EPUB, LaTeX, RST, RTF, etc.)"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for PandocExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let pandoc_result = extract_bytes_from_mime(content, mime_type).await?;
+
+        let mut additional = std::collections::HashMap::new();
+        for (key, value) in pandoc_result.metadata {
+            additional.insert(key, value);
+        }
+
+        Ok(ExtractionResult {
+            content: pandoc_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "application/vnd.oasis.opendocument.text",
+            "application/epub+zip",
+            "application/x-latex",
+            "text/x-tex",
+            "text/x-rst",
+            "text/prs.fallenstein.rst",
+            "application/rtf",
+            "text/rtf",
+            "application/x-typst",
+            "application/x-ipynb+json",
+            "application/x-fictionbook+xml",
+            "text/x-org",
+            "text/x-commonmark",
+            "text/x-gfm",
+            "text/x-multimarkdown",
+            "text/x-markdown-extra",
+            "application/docbook+xml",
+            "application/x-jats+xml",
+            "application/x-opml+xml",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        40
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::extraction::pandoc::validate_pandoc_version;
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_plugin_interface() {
+        let extractor = PandocExtractor::new();
+        assert_eq!(extractor.name(), "pandoc-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.priority(), 40);
+        assert!(!extractor.supported_mime_types().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_docx() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_odt() {
+        let extractor = PandocExtractor::new();
+        assert!(
+            extractor
+                .supported_mime_types()
+                .contains(&"application/vnd.oasis.opendocument.text")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_epub() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/epub+zip"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_latex() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"application/x-latex"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_supports_rst() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.supported_mime_types().contains(&"text/x-rst"));
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_markdown() {
+        if validate_pandoc_version().await.is_err() {
+            return;
+        }
+
+        let extractor = PandocExtractor::new();
+        let markdown = b"# Hello World\n\nThis is a test.";
+        let config = ExtractionConfig::default();
+
+        let result = extractor.extract_bytes(markdown, "text/x-rst", &config).await;
+
+        let _ = result;
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_default() {
+        let extractor = PandocExtractor;
+        assert_eq!(extractor.name(), "pandoc-extractor");
+    }
+
+    #[tokio::test]
+    async fn test_pandoc_extractor_initialize_shutdown() {
+        let extractor = PandocExtractor::new();
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+}