kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/extractors/pptx.rs

@@ -0,0 +1,234 @@
+//! PowerPoint presentation extractor.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+use std::path::Path;
+
+#[cfg(feature = "ocr")]
+use crate::ocr::OcrProcessor;
+
+/// PowerPoint presentation extractor.
+///
+/// Supports: .pptx, .pptm, .ppsx
+pub struct PptxExtractor;
+
+impl Default for PptxExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl PptxExtractor {
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Process extracted images with OCR if configured.
+    #[cfg(feature = "ocr")]
+    async fn process_images_with_ocr(
+        &self,
+        mut images: Vec<crate::types::ExtractedImage>,
+        config: &ExtractionConfig,
+    ) -> Result<Vec<crate::types::ExtractedImage>> {
+        if config.ocr.is_none() {
+            return Ok(images);
+        }
+
+        let ocr_config = config.ocr.as_ref().unwrap();
+        let tess_config = ocr_config.tesseract_config.as_ref().cloned().unwrap_or_default();
+
+        for image in &mut images {
+            let image_data = image.data.clone();
+            let tess_config_clone = tess_config.clone();
+
+            let ocr_result = tokio::task::spawn_blocking(move || {
+                let cache_dir = std::env::var("KREUZBERG_CACHE_DIR").ok().map(std::path::PathBuf::from);
+
+                let proc = OcrProcessor::new(cache_dir)?;
+                let ocr_tess_config: crate::ocr::types::TesseractConfig = (&tess_config_clone).into();
+                proc.process_image(&image_data, &ocr_tess_config)
+            })
+            .await
+            .map_err(|e| crate::KreuzbergError::Ocr {
+                message: format!("OCR task failed: {}", e),
+                source: None,
+            })?;
+
+            match ocr_result {
+                Ok(ocr_extraction) => {
+                    let extraction_result = ExtractionResult {
+                        content: ocr_extraction.content,
+                        mime_type: image.format.clone(),
+                        metadata: Metadata::default(),
+                        tables: vec![],
+                        detected_languages: None,
+                        chunks: None,
+                        images: None,
+                    };
+                    image.ocr_result = Some(Box::new(extraction_result));
+                }
+                Err(_) => {
+                    image.ocr_result = None;
+                }
+            }
+        }
+
+        Ok(images)
+    }
+}
+
+impl Plugin for PptxExtractor {
+    fn name(&self) -> &str {
+        "pptx-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for PptxExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let extract_images = config.images.as_ref().is_some_and(|img| img.extract_images);
+
+        // Extract PPTX content
+        let pptx_result = if crate::core::batch_mode::is_batch_mode() {
+            // Batch mode: Use spawn_blocking for parallelism
+            let content_owned = content.to_vec();
+            tokio::task::spawn_blocking(move || {
+                crate::extraction::pptx::extract_pptx_from_bytes(&content_owned, extract_images)
+            })
+            .await
+            .map_err(|e| crate::error::KreuzbergError::parsing(format!("PPTX extraction task failed: {}", e)))??
+        } else {
+            // Single-file mode: Direct extraction (no spawn overhead)
+            crate::extraction::pptx::extract_pptx_from_bytes(content, extract_images)?
+        };
+
+        let mut additional = std::collections::HashMap::new();
+        additional.insert("slide_count".to_string(), serde_json::json!(pptx_result.slide_count));
+        additional.insert("image_count".to_string(), serde_json::json!(pptx_result.image_count));
+        additional.insert("table_count".to_string(), serde_json::json!(pptx_result.table_count));
+
+        let images = if !pptx_result.images.is_empty() {
+            #[cfg(feature = "ocr")]
+            {
+                let processed_images = self.process_images_with_ocr(pptx_result.images, config).await?;
+                Some(processed_images)
+            }
+            #[cfg(not(feature = "ocr"))]
+            {
+                Some(pptx_result.images)
+            }
+        } else {
+            None
+        };
+
+        Ok(ExtractionResult {
+            content: pptx_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images,
+        })
+    }
+
+    async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+        let path_str = path
+            .to_str()
+            .ok_or_else(|| crate::KreuzbergError::validation("Invalid file path".to_string()))?;
+
+        let extract_images = config.images.as_ref().is_some_and(|img| img.extract_images);
+
+        let pptx_result = crate::extraction::pptx::extract_pptx_from_path(path_str, extract_images)?;
+
+        let mut additional = std::collections::HashMap::new();
+        additional.insert("slide_count".to_string(), serde_json::json!(pptx_result.slide_count));
+        additional.insert("image_count".to_string(), serde_json::json!(pptx_result.image_count));
+        additional.insert("table_count".to_string(), serde_json::json!(pptx_result.table_count));
+
+        let images = if !pptx_result.images.is_empty() {
+            #[cfg(feature = "ocr")]
+            {
+                let processed_images = self.process_images_with_ocr(pptx_result.images, config).await?;
+                Some(processed_images)
+            }
+            #[cfg(not(feature = "ocr"))]
+            {
+                Some(pptx_result.images)
+            }
+        } else {
+            None
+        };
+
+        Ok(ExtractionResult {
+            content: pptx_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                format: Some(crate::types::FormatMetadata::Pptx(pptx_result.metadata)),
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+            "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pptx_extractor_plugin_interface() {
+        let extractor = PptxExtractor::new();
+        assert_eq!(extractor.name(), "pptx-extractor");
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_pptx_extractor_supported_mime_types() {
+        let extractor = PptxExtractor::new();
+        let mime_types = extractor.supported_mime_types();
+        assert_eq!(mime_types.len(), 3);
+        assert!(mime_types.contains(&"application/vnd.openxmlformats-officedocument.presentationml.presentation"));
+    }
+}

data/vendor/kreuzberg/src/extractors/structured.rs

@@ -0,0 +1,126 @@
+//! Structured data extractor (JSON, YAML, TOML).
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::{ExtractionResult, Metadata};
+use async_trait::async_trait;
+use std::path::Path;
+
+/// Structured data extractor supporting JSON, YAML, and TOML.
+pub struct StructuredExtractor;
+
+impl Default for StructuredExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl StructuredExtractor {
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Plugin for StructuredExtractor {
+    fn name(&self) -> &str {
+        "structured-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for StructuredExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let structured_result = match mime_type {
+            "application/json" | "text/json" => crate::extraction::structured::parse_json(content, None)?,
+            "application/x-yaml" | "text/yaml" | "text/x-yaml" => crate::extraction::structured::parse_yaml(content)?,
+            "application/toml" | "text/toml" => crate::extraction::structured::parse_toml(content)?,
+            _ => return Err(crate::KreuzbergError::UnsupportedFormat(mime_type.to_string())),
+        };
+
+        let mut additional = std::collections::HashMap::new();
+        additional.insert(
+            "field_count".to_string(),
+            serde_json::json!(structured_result.text_fields.len()),
+        );
+        additional.insert("data_format".to_string(), serde_json::json!(structured_result.format));
+
+        for (key, value) in structured_result.metadata {
+            additional.insert(key, serde_json::json!(value));
+        }
+
+        Ok(ExtractionResult {
+            content: structured_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: Metadata {
+                additional,
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+        let bytes = tokio::fs::read(path).await?;
+        self.extract_bytes(&bytes, mime_type, config).await
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &[
+            "application/json",
+            "text/json",
+            "application/x-yaml",
+            "text/yaml",
+            "text/x-yaml",
+            "application/toml",
+            "text/toml",
+        ]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_structured_extractor_plugin_interface() {
+        let extractor = StructuredExtractor::new();
+        assert_eq!(extractor.name(), "structured-extractor");
+        assert!(extractor.initialize().is_ok());
+        assert!(extractor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_structured_extractor_supported_mime_types() {
+        let extractor = StructuredExtractor::new();
+        let mime_types = extractor.supported_mime_types();
+        assert_eq!(mime_types.len(), 7);
+        assert!(mime_types.contains(&"application/json"));
+        assert!(mime_types.contains(&"application/x-yaml"));
+        assert!(mime_types.contains(&"application/toml"));
+    }
+}

data/vendor/kreuzberg/src/extractors/text.rs

@@ -0,0 +1,242 @@
+//! Plain text and Markdown extractors.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::extraction::text::parse_text;
+use crate::plugins::{DocumentExtractor, Plugin};
+use crate::types::ExtractionResult;
+use async_trait::async_trait;
+
+/// Plain text extractor.
+///
+/// Extracts content from plain text files (.txt).
+pub struct PlainTextExtractor;
+
+impl PlainTextExtractor {
+    /// Create a new plain text extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for PlainTextExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for PlainTextExtractor {
+    fn name(&self) -> &str {
+        "plain-text-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from plain text files"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for PlainTextExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let text_result = parse_text(content, false)?;
+
+        Ok(ExtractionResult {
+            content: text_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: crate::types::Metadata {
+                format: Some(crate::types::FormatMetadata::Text(crate::types::TextMetadata {
+                    line_count: text_result.line_count,
+                    word_count: text_result.word_count,
+                    character_count: text_result.character_count,
+                    headers: None,
+                    links: None,
+                    code_blocks: None,
+                })),
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["text/plain"]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+/// Markdown extractor.
+///
+/// Extracts content from Markdown files (.md, .markdown).
+/// Preserves markdown syntax and extracts metadata like headers, links, and code blocks.
+pub struct MarkdownExtractor;
+
+impl MarkdownExtractor {
+    /// Create a new Markdown extractor.
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for MarkdownExtractor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Plugin for MarkdownExtractor {
+    fn name(&self) -> &str {
+        "markdown-extractor"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn description(&self) -> &str {
+        "Extracts content from Markdown files with metadata parsing"
+    }
+
+    fn author(&self) -> &str {
+        "Kreuzberg Team"
+    }
+}
+
+#[async_trait]
+impl DocumentExtractor for MarkdownExtractor {
+    async fn extract_bytes(
+        &self,
+        content: &[u8],
+        mime_type: &str,
+        _config: &ExtractionConfig,
+    ) -> Result<ExtractionResult> {
+        let text_result = parse_text(content, true)?;
+
+        Ok(ExtractionResult {
+            content: text_result.content,
+            mime_type: mime_type.to_string(),
+            metadata: crate::types::Metadata {
+                format: Some(crate::types::FormatMetadata::Text(crate::types::TextMetadata {
+                    line_count: text_result.line_count,
+                    word_count: text_result.word_count,
+                    character_count: text_result.character_count,
+                    headers: text_result.headers,
+                    links: text_result.links,
+                    code_blocks: text_result.code_blocks,
+                })),
+                ..Default::default()
+            },
+            tables: vec![],
+            detected_languages: None,
+            chunks: None,
+            images: None,
+        })
+    }
+
+    fn supported_mime_types(&self) -> &[&str] {
+        &["text/markdown", "text/x-markdown"]
+    }
+
+    fn priority(&self) -> i32 {
+        50
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_plain_text_extractor() {
+        let extractor = PlainTextExtractor::new();
+        let content = b"Hello, World!\nThis is a test.";
+        let config = ExtractionConfig::default();
+
+        let result = extractor.extract_bytes(content, "text/plain", &config).await.unwrap();
+
+        assert_eq!(result.mime_type, "text/plain");
+        assert!(result.content.contains("Hello, World!"));
+        assert!(result.metadata.format.is_some());
+        let text_meta = match result.metadata.format.as_ref().unwrap() {
+            crate::types::FormatMetadata::Text(meta) => meta,
+            _ => panic!("Expected Text metadata"),
+        };
+        assert_eq!(text_meta.line_count, 2);
+        assert_eq!(text_meta.word_count, 6);
+    }
+
+    #[tokio::test]
+    async fn test_markdown_extractor() {
+        let extractor = MarkdownExtractor::new();
+        let content = b"# Header\n\nThis is [a link](https://example.com).\n\n```python\nprint(\"hello\")\n```";
+        let config = ExtractionConfig::default();
+
+        let result = extractor
+            .extract_bytes(content, "text/markdown", &config)
+            .await
+            .unwrap();
+
+        assert_eq!(result.mime_type, "text/markdown");
+        assert!(result.content.contains("# Header"));
+        assert!(result.metadata.format.is_some());
+        let text_meta = match result.metadata.format.as_ref().unwrap() {
+            crate::types::FormatMetadata::Text(meta) => meta,
+            _ => panic!("Expected Text metadata"),
+        };
+        assert!(text_meta.headers.is_some());
+        assert!(text_meta.links.is_some());
+        assert!(text_meta.code_blocks.is_some());
+    }
+
+    #[test]
+    fn test_plain_text_plugin_interface() {
+        let extractor = PlainTextExtractor::new();
+        assert_eq!(extractor.name(), "plain-text-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.supported_mime_types(), &["text/plain"]);
+        assert_eq!(extractor.priority(), 50);
+    }
+
+    #[test]
+    fn test_markdown_plugin_interface() {
+        let extractor = MarkdownExtractor::new();
+        assert_eq!(extractor.name(), "markdown-extractor");
+        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
+        assert_eq!(extractor.supported_mime_types(), &["text/markdown", "text/x-markdown"]);
+        assert_eq!(extractor.priority(), 50);
+    }
+}