kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/extraction/pandoc/batch.rs

@@ -1,275 +0,0 @@
-//! Batch extraction with automatic pandoc-server mode optimization
-//!
-//! This module provides intelligent batch processing for Pandoc extractions with automatic
-//! server mode optimization. The `BatchExtractor` automatically detects if pandoc-server is
-//! available and uses it when beneficial for performance.
-//!
-//! # Server Mode Detection
-//!
-//! The extractor checks for pandoc-server availability in two ways:
-//! 1. Direct binary check: Looks for `pandoc-server` in PATH
-//! 2. Version detection: Checks if pandoc 3.8+ is installed (supports server mode)
-//!
-//! # Optimization Heuristic
-//!
-//! - **>3 files**: Uses server mode (amortizes ~100-200ms startup overhead)
-//! - **≤3 files**: Uses subprocess mode (avoids server startup cost)
-//! - **Server unavailable**: Always uses subprocess mode (graceful fallback)
-//!
-//! # Example Usage
-//!
-//! ```no_run
-//! use kreuzberg::extraction::pandoc::BatchExtractor;
-//! use std::path::Path;
-//!
-//! #[tokio::main]
-//! async fn main() -> kreuzberg::Result<()> {
-//!     // Create extractor (auto-detects server availability)
-//!     let extractor = BatchExtractor::new().await;
-//!
-//!     // Extract multiple files
-//!     let paths = vec![
-//!         Path::new("doc1.docx"),
-//!         Path::new("doc2.docx"),
-//!         Path::new("doc3.docx"),
-//!         Path::new("doc4.docx"),
-//!     ];
-//!     let formats = vec!["docx", "docx", "docx", "docx"];
-//!
-//!     let results = extractor.extract_files(&paths, &formats).await?;
-//!
-//!     // Process results
-//!     for (i, result) in results.iter().enumerate() {
-//!         match result {
-//!             Ok(extraction) => println!("File {}: {} chars", i, extraction.content.len()),
-//!             Err(e) => eprintln!("File {}: Error: {}", i, e),
-//!         }
-//!     }
-//!
-//!     // Cleanup (optional, happens automatically on drop)
-//!     extractor.shutdown().await?;
-//!
-//!     Ok(())
-//! }
-//! ```
-//!
-//! # Logging
-//!
-//! Enable tracing to see server mode detection and usage:
-//!
-//! ```bash
-//! RUST_LOG=kreuzberg=debug cargo run
-//! ```
-//!
-//! Expected logs:
-//! - `DEBUG`: Server detection results, file counts
-//! - `INFO`: Server startup confirmation
-//! - `WARN`: Server failures with troubleshooting guidance
-
-use crate::error::{KreuzbergError, Result};
-use crate::types::PandocExtractionResult;
-use serde_json::Value;
-use std::path::Path;
-use std::sync::Arc;
-use tokio::sync::Mutex;
-
-use super::server::PandocServer;
-use super::subprocess;
-
-/// Batch extractor with automatic server mode optimization
-///
-/// Automatically detects pandoc-server availability and uses it when beneficial for
-/// performance (>3 files). Falls back to subprocess mode gracefully.
-///
-/// # Performance
-///
-/// - **Server mode**: ~100-200ms savings per file (startup overhead eliminated)
-/// - **Subprocess mode**: Lightweight, suitable for small batches
-///
-/// # Thread Safety
-///
-/// This struct is safe to use across multiple async tasks. The server instance is
-/// protected by an Arc<Mutex> and can be safely shared.
-pub struct BatchExtractor {
-    server: Arc<Mutex<Option<PandocServer>>>,
-    use_server: bool,
-}
-
-impl BatchExtractor {
-    /// Create a new batch extractor
-    ///
-    /// Automatically detects if pandoc-server is available and uses it if beneficial
-    pub async fn new() -> Self {
-        let use_server = PandocServer::is_server_available().await;
-
-        if use_server {
-            tracing::info!("Pandoc server mode available - will use for batch processing");
-        } else {
-            tracing::debug!("Pandoc server mode not available - using subprocess mode");
-            tracing::debug!("To enable server mode:");
-            tracing::debug!("  1. Install pandoc 3.8+ (current version may be older)");
-            tracing::debug!("  2. OR create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
-        }
-
-        Self {
-            server: Arc::new(Mutex::new(None)),
-            use_server,
-        }
-    }
-
-    /// Extract multiple files using optimal strategy
-    ///
-    /// Automatically starts server if available and beneficial (>3 files)
-    pub async fn extract_files(
-        &self,
-        paths: &[&Path],
-        formats: &[&str],
-    ) -> Result<Vec<Result<PandocExtractionResult>>> {
-        if paths.is_empty() {
-            return Ok(vec![]);
-        }
-
-        if self.use_server && paths.len() > 3 {
-            self.extract_with_server(paths, formats).await
-        } else {
-            self.extract_with_subprocess(paths, formats).await
-        }
-    }
-
-    /// Extract using server mode (warm instance)
-    async fn extract_with_server(
-        &self,
-        paths: &[&Path],
-        formats: &[&str],
-    ) -> Result<Vec<Result<PandocExtractionResult>>> {
-        let mut server_lock = self.server.lock().await;
-
-        if server_lock.is_none() {
-            match PandocServer::new(None, None).await {
-                Ok(server) => {
-                    if let Err(e) = server.start().await {
-                        tracing::warn!("Failed to start pandoc-server: {}", e);
-                        tracing::warn!("Falling back to subprocess mode");
-                        tracing::warn!("To fix:");
-                        tracing::warn!("  1. Ensure pandoc 3.8+ is installed: pandoc --version");
-                        tracing::warn!("  2. Create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
-                        drop(server_lock);
-                        return self.extract_with_subprocess(paths, formats).await;
-                    }
-
-                    tracing::info!("Started pandoc-server for batch processing ({} files)", paths.len());
-                    *server_lock = Some(server);
-                }
-                Err(e) => {
-                    tracing::warn!("Failed to create pandoc-server: {}", e);
-                    tracing::warn!("Falling back to subprocess mode");
-                    drop(server_lock);
-                    return self.extract_with_subprocess(paths, formats).await;
-                }
-            }
-        } else {
-            tracing::debug!("Reusing warm pandoc-server instance for {} files", paths.len());
-        }
-
-        let server = server_lock.as_ref().unwrap();
-        let mut results = Vec::with_capacity(paths.len());
-
-        for (i, (path, format)) in paths.iter().zip(formats.iter()).enumerate() {
-            tracing::debug!("Extracting file {}/{} via server: {:?}", i + 1, paths.len(), path);
-
-            let content = match tokio::fs::read(path).await {
-                Ok(c) => c,
-                Err(e) => {
-                    results.push(Err(KreuzbergError::Io(e)));
-                    continue;
-                }
-            };
-
-            let result = match server.convert(&String::from_utf8_lossy(&content), format, "json").await {
-                Ok(json_output) => match serde_json::from_str::<Value>(&json_output) {
-                    Ok(json_data) => {
-                        let content = subprocess::extract_content_from_json(&json_data)?;
-                        let metadata = subprocess::extract_metadata_from_json(&json_data)?;
-
-                        Ok(PandocExtractionResult { content, metadata })
-                    }
-                    Err(e) => Err(KreuzbergError::parsing(format!(
-                        "Failed to parse JSON from server: {}",
-                        e
-                    ))),
-                },
-                Err(e) => Err(e),
-            };
-
-            results.push(result);
-        }
-
-        Ok(results)
-    }
-
-    /// Extract using subprocess mode
-    async fn extract_with_subprocess(
-        &self,
-        paths: &[&Path],
-        formats: &[&str],
-    ) -> Result<Vec<Result<PandocExtractionResult>>> {
-        tracing::debug!("Extracting {} files via subprocess mode", paths.len());
-
-        let mut results = Vec::with_capacity(paths.len());
-
-        for (path, format) in paths.iter().zip(formats.iter()) {
-            let result = subprocess::extract_with_pandoc(path, format)
-                .await
-                .map(|(content, metadata)| PandocExtractionResult { content, metadata });
-            results.push(result);
-        }
-
-        Ok(results)
-    }
-
-    /// Stop the server if running
-    pub async fn shutdown(&self) -> Result<()> {
-        let mut server_lock = self.server.lock().await;
-        if let Some(server) = server_lock.take() {
-            tracing::info!("Shutting down pandoc-server");
-            server.stop().await?;
-        }
-        Ok(())
-    }
-}
-
-impl Drop for BatchExtractor {
-    fn drop(&mut self) {
-        if let Some(server) = self.server.try_lock().ok().and_then(|mut s| s.take()) {
-            tokio::spawn(async move {
-                let _ = server.stop().await;
-            });
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_batch_extractor_creation() {
-        let extractor = BatchExtractor::new().await;
-        assert!(extractor.server.lock().await.is_none());
-    }
-
-    #[tokio::test]
-    async fn test_empty_batch() {
-        let extractor = BatchExtractor::new().await;
-        let results = extractor.extract_files(&[], &[]).await;
-        assert!(results.is_ok());
-        assert!(results.unwrap().is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_shutdown() {
-        let extractor = BatchExtractor::new().await;
-        let result = extractor.shutdown().await;
-        assert!(result.is_ok());
-    }
-}

data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs

@@ -1,178 +0,0 @@
-use crate::error::{KreuzbergError, Result};
-use std::collections::HashMap;
-
-/// MIME type to Pandoc format mapping
-pub fn get_pandoc_format_from_mime(mime_type: &str) -> Result<String> {
-    let mappings = get_mime_to_pandoc_mapping();
-
-    if let Some(format) = mappings.get(mime_type) {
-        return Ok(format.to_string());
-    }
-
-    if mime_type == "text/markdown" {
-        return Ok("markdown".to_string());
-    }
-
-    for (key, value) in &mappings {
-        if mime_type.starts_with(key) {
-            return Ok(value.to_string());
-        }
-    }
-
-    Err(KreuzbergError::validation(format!(
-        "Unsupported MIME type: {}",
-        mime_type
-    )))
-}
-
-/// MIME type to file extension mapping
-pub fn get_extension_from_mime(mime_type: &str) -> Result<String> {
-    let mappings = get_mime_to_extension_mapping();
-
-    mappings
-        .get(mime_type)
-        .map(|s| s.to_string())
-        .ok_or_else(|| KreuzbergError::validation(format!("No file extension mapping for MIME type: {}", mime_type)))
-}
-
-fn get_mime_to_pandoc_mapping() -> HashMap<&'static str, &'static str> {
-    HashMap::from([
-        ("application/csl+json", "csljson"),
-        ("application/docbook+xml", "docbook"),
-        ("application/epub+zip", "epub"),
-        ("application/rtf", "rtf"),
-        ("application/vnd.oasis.opendocument.text", "odt"),
-        (
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "docx",
-        ),
-        ("application/x-biblatex", "biblatex"),
-        ("application/x-bibtex", "bibtex"),
-        ("application/x-endnote+xml", "endnotexml"),
-        ("application/x-fictionbook+xml", "fb2"),
-        ("application/x-ipynb+json", "ipynb"),
-        ("application/x-jats+xml", "jats"),
-        ("application/x-latex", "latex"),
-        ("application/x-opml+xml", "opml"),
-        ("application/x-research-info-systems", "ris"),
-        ("application/x-typst", "typst"),
-        ("text/csv", "csv"),
-        ("text/tab-separated-values", "tsv"),
-        ("text/troff", "man"),
-        ("text/x-commonmark", "commonmark"),
-        ("text/x-dokuwiki", "dokuwiki"),
-        ("text/x-gfm", "gfm"),
-        ("text/x-markdown", "markdown"),
-        ("text/x-markdown-extra", "markdown_phpextra"),
-        ("text/x-mdoc", "mdoc"),
-        ("text/x-multimarkdown", "markdown_mmd"),
-        ("text/x-org", "org"),
-        ("text/x-pod", "pod"),
-        ("text/x-rst", "rst"),
-    ])
-}
-
-fn get_mime_to_extension_mapping() -> HashMap<&'static str, &'static str> {
-    HashMap::from([
-        ("application/csl+json", "json"),
-        ("application/docbook+xml", "xml"),
-        ("application/epub+zip", "epub"),
-        ("application/rtf", "rtf"),
-        ("application/vnd.oasis.opendocument.text", "odt"),
-        (
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "docx",
-        ),
-        ("application/x-biblatex", "bib"),
-        ("application/x-bibtex", "bib"),
-        ("application/x-endnote+xml", "xml"),
-        ("application/x-fictionbook+xml", "fb2"),
-        ("application/x-ipynb+json", "ipynb"),
-        ("application/x-jats+xml", "xml"),
-        ("application/x-latex", "tex"),
-        ("application/x-opml+xml", "opml"),
-        ("application/x-research-info-systems", "ris"),
-        ("application/x-typst", "typst"),
-        ("text/csv", "csv"),
-        ("text/tab-separated-values", "tsv"),
-        ("text/troff", "1"),
-        ("text/x-commonmark", "md"),
-        ("text/x-dokuwiki", "wiki"),
-        ("text/x-gfm", "md"),
-        ("text/x-markdown", "md"),
-        ("text/x-markdown-extra", "md"),
-        ("text/x-mdoc", "md"),
-        ("text/x-multimarkdown", "md"),
-        ("text/x-org", "org"),
-        ("text/x-pod", "pod"),
-        ("text/x-rst", "rst"),
-    ])
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_docx() {
-        let format =
-            get_pandoc_format_from_mime("application/vnd.openxmlformats-officedocument.wordprocessingml.document")
-                .unwrap();
-        assert_eq!(format, "docx");
-    }
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_markdown() {
-        let format = get_pandoc_format_from_mime("text/markdown").unwrap();
-        assert_eq!(format, "markdown");
-    }
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_gfm() {
-        let format = get_pandoc_format_from_mime("text/x-gfm").unwrap();
-        assert_eq!(format, "gfm");
-    }
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_rst() {
-        let format = get_pandoc_format_from_mime("text/x-rst").unwrap();
-        assert_eq!(format, "rst");
-    }
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_latex() {
-        let format = get_pandoc_format_from_mime("application/x-latex").unwrap();
-        assert_eq!(format, "latex");
-    }
-
-    #[test]
-    fn test_get_pandoc_format_from_mime_invalid() {
-        let result = get_pandoc_format_from_mime("application/invalid");
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_get_extension_from_mime_docx() {
-        let ext =
-            get_extension_from_mime("application/vnd.openxmlformats-officedocument.wordprocessingml.document").unwrap();
-        assert_eq!(ext, "docx");
-    }
-
-    #[test]
-    fn test_get_extension_from_mime_md() {
-        let ext = get_extension_from_mime("text/x-markdown").unwrap();
-        assert_eq!(ext, "md");
-    }
-
-    #[test]
-    fn test_get_extension_from_mime_tex() {
-        let ext = get_extension_from_mime("application/x-latex").unwrap();
-        assert_eq!(ext, "tex");
-    }
-
-    #[test]
-    fn test_get_extension_from_mime_invalid() {
-        let result = get_extension_from_mime("application/invalid");
-        assert!(result.is_err());
-    }
-}