kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/extraction/pandoc/batch.rs

@@ -0,0 +1,275 @@
+//! Batch extraction with automatic pandoc-server mode optimization
+//!
+//! This module provides intelligent batch processing for Pandoc extractions with automatic
+//! server mode optimization. The `BatchExtractor` automatically detects if pandoc-server is
+//! available and uses it when beneficial for performance.
+//!
+//! # Server Mode Detection
+//!
+//! The extractor checks for pandoc-server availability in two ways:
+//! 1. Direct binary check: Looks for `pandoc-server` in PATH
+//! 2. Version detection: Checks if pandoc 3.8+ is installed (supports server mode)
+//!
+//! # Optimization Heuristic
+//!
+//! - **>3 files**: Uses server mode (amortizes ~100-200ms startup overhead)
+//! - **≤3 files**: Uses subprocess mode (avoids server startup cost)
+//! - **Server unavailable**: Always uses subprocess mode (graceful fallback)
+//!
+//! # Example Usage
+//!
+//! ```no_run
+//! use kreuzberg::extraction::pandoc::BatchExtractor;
+//! use std::path::Path;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Create extractor (auto-detects server availability)
+//!     let extractor = BatchExtractor::new().await;
+//!
+//!     // Extract multiple files
+//!     let paths = vec![
+//!         Path::new("doc1.docx"),
+//!         Path::new("doc2.docx"),
+//!         Path::new("doc3.docx"),
+//!         Path::new("doc4.docx"),
+//!     ];
+//!     let formats = vec!["docx", "docx", "docx", "docx"];
+//!
+//!     let results = extractor.extract_files(&paths, &formats).await?;
+//!
+//!     // Process results
+//!     for (i, result) in results.iter().enumerate() {
+//!         match result {
+//!             Ok(extraction) => println!("File {}: {} chars", i, extraction.content.len()),
+//!             Err(e) => eprintln!("File {}: Error: {}", i, e),
+//!         }
+//!     }
+//!
+//!     // Cleanup (optional, happens automatically on drop)
+//!     extractor.shutdown().await?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Logging
+//!
+//! Enable tracing to see server mode detection and usage:
+//!
+//! ```bash
+//! RUST_LOG=kreuzberg=debug cargo run
+//! ```
+//!
+//! Expected logs:
+//! - `DEBUG`: Server detection results, file counts
+//! - `INFO`: Server startup confirmation
+//! - `WARN`: Server failures with troubleshooting guidance
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PandocExtractionResult;
+use serde_json::Value;
+use std::path::Path;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+use super::server::PandocServer;
+use super::subprocess;
+
+/// Batch extractor with automatic server mode optimization
+///
+/// Automatically detects pandoc-server availability and uses it when beneficial for
+/// performance (>3 files). Falls back to subprocess mode gracefully.
+///
+/// # Performance
+///
+/// - **Server mode**: ~100-200ms savings per file (startup overhead eliminated)
+/// - **Subprocess mode**: Lightweight, suitable for small batches
+///
+/// # Thread Safety
+///
+/// This struct is safe to use across multiple async tasks. The server instance is
+/// protected by an Arc<Mutex> and can be safely shared.
+pub struct BatchExtractor {
+    server: Arc<Mutex<Option<PandocServer>>>,
+    use_server: bool,
+}
+
+impl BatchExtractor {
+    /// Create a new batch extractor
+    ///
+    /// Automatically detects if pandoc-server is available and uses it if beneficial
+    pub async fn new() -> Self {
+        let use_server = PandocServer::is_server_available().await;
+
+        if use_server {
+            tracing::info!("Pandoc server mode available - will use for batch processing");
+        } else {
+            tracing::debug!("Pandoc server mode not available - using subprocess mode");
+            tracing::debug!("To enable server mode:");
+            tracing::debug!("  1. Install pandoc 3.8+ (current version may be older)");
+            tracing::debug!("  2. OR create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+        }
+
+        Self {
+            server: Arc::new(Mutex::new(None)),
+            use_server,
+        }
+    }
+
+    /// Extract multiple files using optimal strategy
+    ///
+    /// Automatically starts server if available and beneficial (>3 files)
+    pub async fn extract_files(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        if paths.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if self.use_server && paths.len() > 3 {
+            self.extract_with_server(paths, formats).await
+        } else {
+            self.extract_with_subprocess(paths, formats).await
+        }
+    }
+
+    /// Extract using server mode (warm instance)
+    async fn extract_with_server(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        let mut server_lock = self.server.lock().await;
+
+        if server_lock.is_none() {
+            match PandocServer::new(None, None).await {
+                Ok(server) => {
+                    if let Err(e) = server.start().await {
+                        tracing::warn!("Failed to start pandoc-server: {}", e);
+                        tracing::warn!("Falling back to subprocess mode");
+                        tracing::warn!("To fix:");
+                        tracing::warn!("  1. Ensure pandoc 3.8+ is installed: pandoc --version");
+                        tracing::warn!("  2. Create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+                        drop(server_lock);
+                        return self.extract_with_subprocess(paths, formats).await;
+                    }
+
+                    tracing::info!("Started pandoc-server for batch processing ({} files)", paths.len());
+                    *server_lock = Some(server);
+                }
+                Err(e) => {
+                    tracing::warn!("Failed to create pandoc-server: {}", e);
+                    tracing::warn!("Falling back to subprocess mode");
+                    drop(server_lock);
+                    return self.extract_with_subprocess(paths, formats).await;
+                }
+            }
+        } else {
+            tracing::debug!("Reusing warm pandoc-server instance for {} files", paths.len());
+        }
+
+        let server = server_lock.as_ref().unwrap();
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (i, (path, format)) in paths.iter().zip(formats.iter()).enumerate() {
+            tracing::debug!("Extracting file {}/{} via server: {:?}", i + 1, paths.len(), path);
+
+            let content = match tokio::fs::read(path).await {
+                Ok(c) => c,
+                Err(e) => {
+                    results.push(Err(KreuzbergError::Io(e)));
+                    continue;
+                }
+            };
+
+            let result = match server.convert(&String::from_utf8_lossy(&content), format, "json").await {
+                Ok(json_output) => match serde_json::from_str::<Value>(&json_output) {
+                    Ok(json_data) => {
+                        let content = subprocess::extract_content_from_json(&json_data)?;
+                        let metadata = subprocess::extract_metadata_from_json(&json_data)?;
+
+                        Ok(PandocExtractionResult { content, metadata })
+                    }
+                    Err(e) => Err(KreuzbergError::parsing(format!(
+                        "Failed to parse JSON from server: {}",
+                        e
+                    ))),
+                },
+                Err(e) => Err(e),
+            };
+
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Extract using subprocess mode
+    async fn extract_with_subprocess(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        tracing::debug!("Extracting {} files via subprocess mode", paths.len());
+
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (path, format) in paths.iter().zip(formats.iter()) {
+            let result = subprocess::extract_with_pandoc(path, format)
+                .await
+                .map(|(content, metadata)| PandocExtractionResult { content, metadata });
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Stop the server if running
+    pub async fn shutdown(&self) -> Result<()> {
+        let mut server_lock = self.server.lock().await;
+        if let Some(server) = server_lock.take() {
+            tracing::info!("Shutting down pandoc-server");
+            server.stop().await?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for BatchExtractor {
+    fn drop(&mut self) {
+        if let Some(server) = self.server.try_lock().ok().and_then(|mut s| s.take()) {
+            tokio::spawn(async move {
+                let _ = server.stop().await;
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_batch_extractor_creation() {
+        let extractor = BatchExtractor::new().await;
+        assert!(extractor.server.lock().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_empty_batch() {
+        let extractor = BatchExtractor::new().await;
+        let results = extractor.extract_files(&[], &[]).await;
+        assert!(results.is_ok());
+        assert!(results.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_shutdown() {
+        let extractor = BatchExtractor::new().await;
+        let result = extractor.shutdown().await;
+        assert!(result.is_ok());
+    }
+}

data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs

@@ -0,0 +1,178 @@
+use crate::error::{KreuzbergError, Result};
+use std::collections::HashMap;
+
+/// MIME type to Pandoc format mapping
+pub fn get_pandoc_format_from_mime(mime_type: &str) -> Result<String> {
+    let mappings = get_mime_to_pandoc_mapping();
+
+    if let Some(format) = mappings.get(mime_type) {
+        return Ok(format.to_string());
+    }
+
+    if mime_type == "text/markdown" {
+        return Ok("markdown".to_string());
+    }
+
+    for (key, value) in &mappings {
+        if mime_type.starts_with(key) {
+            return Ok(value.to_string());
+        }
+    }
+
+    Err(KreuzbergError::validation(format!(
+        "Unsupported MIME type: {}",
+        mime_type
+    )))
+}
+
+/// MIME type to file extension mapping
+pub fn get_extension_from_mime(mime_type: &str) -> Result<String> {
+    let mappings = get_mime_to_extension_mapping();
+
+    mappings
+        .get(mime_type)
+        .map(|s| s.to_string())
+        .ok_or_else(|| KreuzbergError::validation(format!("No file extension mapping for MIME type: {}", mime_type)))
+}
+
+fn get_mime_to_pandoc_mapping() -> HashMap<&'static str, &'static str> {
+    HashMap::from([
+        ("application/csl+json", "csljson"),
+        ("application/docbook+xml", "docbook"),
+        ("application/epub+zip", "epub"),
+        ("application/rtf", "rtf"),
+        ("application/vnd.oasis.opendocument.text", "odt"),
+        (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "docx",
+        ),
+        ("application/x-biblatex", "biblatex"),
+        ("application/x-bibtex", "bibtex"),
+        ("application/x-endnote+xml", "endnotexml"),
+        ("application/x-fictionbook+xml", "fb2"),
+        ("application/x-ipynb+json", "ipynb"),
+        ("application/x-jats+xml", "jats"),
+        ("application/x-latex", "latex"),
+        ("application/x-opml+xml", "opml"),
+        ("application/x-research-info-systems", "ris"),
+        ("application/x-typst", "typst"),
+        ("text/csv", "csv"),
+        ("text/tab-separated-values", "tsv"),
+        ("text/troff", "man"),
+        ("text/x-commonmark", "commonmark"),
+        ("text/x-dokuwiki", "dokuwiki"),
+        ("text/x-gfm", "gfm"),
+        ("text/x-markdown", "markdown"),
+        ("text/x-markdown-extra", "markdown_phpextra"),
+        ("text/x-mdoc", "mdoc"),
+        ("text/x-multimarkdown", "markdown_mmd"),
+        ("text/x-org", "org"),
+        ("text/x-pod", "pod"),
+        ("text/x-rst", "rst"),
+    ])
+}
+
+fn get_mime_to_extension_mapping() -> HashMap<&'static str, &'static str> {
+    HashMap::from([
+        ("application/csl+json", "json"),
+        ("application/docbook+xml", "xml"),
+        ("application/epub+zip", "epub"),
+        ("application/rtf", "rtf"),
+        ("application/vnd.oasis.opendocument.text", "odt"),
+        (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "docx",
+        ),
+        ("application/x-biblatex", "bib"),
+        ("application/x-bibtex", "bib"),
+        ("application/x-endnote+xml", "xml"),
+        ("application/x-fictionbook+xml", "fb2"),
+        ("application/x-ipynb+json", "ipynb"),
+        ("application/x-jats+xml", "xml"),
+        ("application/x-latex", "tex"),
+        ("application/x-opml+xml", "opml"),
+        ("application/x-research-info-systems", "ris"),
+        ("application/x-typst", "typst"),
+        ("text/csv", "csv"),
+        ("text/tab-separated-values", "tsv"),
+        ("text/troff", "1"),
+        ("text/x-commonmark", "md"),
+        ("text/x-dokuwiki", "wiki"),
+        ("text/x-gfm", "md"),
+        ("text/x-markdown", "md"),
+        ("text/x-markdown-extra", "md"),
+        ("text/x-mdoc", "md"),
+        ("text/x-multimarkdown", "md"),
+        ("text/x-org", "org"),
+        ("text/x-pod", "pod"),
+        ("text/x-rst", "rst"),
+    ])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_docx() {
+        let format =
+            get_pandoc_format_from_mime("application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+                .unwrap();
+        assert_eq!(format, "docx");
+    }
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_markdown() {
+        let format = get_pandoc_format_from_mime("text/markdown").unwrap();
+        assert_eq!(format, "markdown");
+    }
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_gfm() {
+        let format = get_pandoc_format_from_mime("text/x-gfm").unwrap();
+        assert_eq!(format, "gfm");
+    }
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_rst() {
+        let format = get_pandoc_format_from_mime("text/x-rst").unwrap();
+        assert_eq!(format, "rst");
+    }
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_latex() {
+        let format = get_pandoc_format_from_mime("application/x-latex").unwrap();
+        assert_eq!(format, "latex");
+    }
+
+    #[test]
+    fn test_get_pandoc_format_from_mime_invalid() {
+        let result = get_pandoc_format_from_mime("application/invalid");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_get_extension_from_mime_docx() {
+        let ext =
+            get_extension_from_mime("application/vnd.openxmlformats-officedocument.wordprocessingml.document").unwrap();
+        assert_eq!(ext, "docx");
+    }
+
+    #[test]
+    fn test_get_extension_from_mime_md() {
+        let ext = get_extension_from_mime("text/x-markdown").unwrap();
+        assert_eq!(ext, "md");
+    }
+
+    #[test]
+    fn test_get_extension_from_mime_tex() {
+        let ext = get_extension_from_mime("application/x-latex").unwrap();
+        assert_eq!(ext, "tex");
+    }
+
+    #[test]
+    fn test_get_extension_from_mime_invalid() {
+        let result = get_extension_from_mime("application/invalid");
+        assert!(result.is_err());
+    }
+}