kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/src/extraction/image.rs

@@ -1,7 +1,6 @@
 //! Image extraction functionality.
 //!
-//! This module provides functions for extracting metadata and EXIF data from images,
-//! including support for multi-frame TIFF files.
+//! This module provides functions for extracting metadata and EXIF data from images.
 
 use crate::error::{KreuzbergError, Result};
 use exif::{In, Reader, Tag};
@@ -40,7 +39,7 @@ pub fn extract_image_metadata(bytes: &[u8]) -> Result<ImageMetadata> {
 
     let width = image.width();
     let height = image.height();
-    let format_str = format!("{:?}", format).to_uppercase();
+    let format_str = format!("{:?}", format);
 
     let exif_data = extract_exif_data(bytes);
 
@@ -95,129 +94,6 @@ fn extract_exif_data(bytes: &[u8]) -> HashMap<String, String> {
     exif_map
 }
 
-/// Result of OCR extraction from an image with optional page tracking.
-#[derive(Debug, Clone)]
-pub struct ImageOcrResult {
-    /// Extracted text content
-    pub content: String,
-    /// Character byte boundaries per frame (for multi-frame TIFFs)
-    pub boundaries: Option<Vec<crate::types::PageBoundary>>,
-    /// Per-frame content information
-    pub page_contents: Option<Vec<crate::types::PageContent>>,
-}
-
-/// Detects the number of frames in a TIFF file.
-///
-/// Returns the count of image frames/pages in a TIFF. Single-frame TIFFs return 1.
-/// Invalid or non-TIFF data returns an error.
-///
-/// # Arguments
-/// * `bytes` - Raw TIFF file bytes
-///
-/// # Returns
-/// Frame count if valid TIFF, error otherwise.
-#[cfg(feature = "ocr")]
-fn detect_tiff_frame_count(bytes: &[u8]) -> Result<usize> {
-    use tiff::decoder::Decoder;
-    let mut decoder =
-        Decoder::new(Cursor::new(bytes)).map_err(|e| KreuzbergError::parsing(format!("TIFF decode: {}", e)))?;
-
-    let mut count = 1;
-    while decoder.next_image().is_ok() {
-        count += 1;
-    }
-    Ok(count)
-}
-
-/// Extract text from image bytes using OCR with optional page tracking for multi-frame TIFFs.
-///
-/// This function:
-/// - Detects if the image is a multi-frame TIFF
-/// - For multi-frame TIFFs with PageConfig enabled, iterates frames and tracks boundaries
-/// - For single-frame images or when page tracking is disabled, runs OCR on the whole image
-/// - Returns (content, boundaries, page_contents) tuple
-///
-/// # Arguments
-/// * `bytes` - Image file bytes
-/// * `mime_type` - MIME type (e.g., "image/tiff")
-/// * `ocr_result` - OCR backend result containing the text
-/// * `page_config` - Optional page configuration for boundary tracking
-///
-/// # Returns
-/// ImageOcrResult with content and optional boundaries for pagination
-#[cfg(feature = "ocr")]
-pub fn extract_text_from_image_with_ocr(
-    bytes: &[u8],
-    mime_type: &str,
-    ocr_result: String,
-    page_config: Option<&crate::core::config::PageConfig>,
-) -> Result<ImageOcrResult> {
-    let is_tiff = mime_type.to_lowercase().contains("tiff");
-    let should_track_pages = page_config.is_some() && is_tiff;
-
-    if !should_track_pages {
-        return Ok(ImageOcrResult {
-            content: ocr_result,
-            boundaries: None,
-            page_contents: None,
-        });
-    }
-
-    let frame_count = detect_tiff_frame_count(bytes)?;
-
-    if frame_count <= 1 {
-        return Ok(ImageOcrResult {
-            content: ocr_result,
-            boundaries: None,
-            page_contents: None,
-        });
-    }
-
-    let content_len = ocr_result.len();
-    let content_per_frame = if frame_count > 0 {
-        content_len / frame_count
-    } else {
-        content_len
-    };
-
-    let mut boundaries = Vec::new();
-    let mut page_contents = Vec::new();
-    let mut byte_offset = 0;
-
-    for frame_num in 1..=frame_count {
-        let frame_end = if frame_num == frame_count {
-            content_len
-        } else {
-            let raw_end = (frame_num * content_per_frame).min(content_len);
-            (raw_end..=content_len)
-                .find(|&i| ocr_result.is_char_boundary(i))
-                .unwrap_or(content_len)
-        };
-
-        boundaries.push(crate::types::PageBoundary {
-            byte_start: byte_offset,
-            byte_end: frame_end,
-            page_number: frame_num,
-        });
-
-        page_contents.push(crate::types::PageContent {
-            page_number: frame_num,
-            content: ocr_result[byte_offset..frame_end].to_string(),
-            tables: vec![],
-            images: vec![],
-            hierarchy: None,
-        });
-
-        byte_offset = frame_end;
-    }
-
-    Ok(ImageOcrResult {
-        content: ocr_result,
-        boundaries: Some(boundaries),
-        page_contents: Some(page_contents),
-    })
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -247,7 +123,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 100);
         assert_eq!(metadata.height, 80);
-        assert_eq!(metadata.format, "PNG");
+        assert_eq!(metadata.format, "Png");
     }
 
     #[test]
@@ -259,7 +135,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 200);
         assert_eq!(metadata.height, 150);
-        assert_eq!(metadata.format, "JPEG");
+        assert_eq!(metadata.format, "Jpeg");
     }
 
     #[test]
@@ -271,7 +147,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 120);
         assert_eq!(metadata.height, 90);
-        assert_eq!(metadata.format, "WEBP");
+        assert_eq!(metadata.format, "WebP");
     }
 
     #[test]
@@ -283,7 +159,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 50);
         assert_eq!(metadata.height, 50);
-        assert_eq!(metadata.format, "BMP");
+        assert_eq!(metadata.format, "Bmp");
     }
 
     #[test]
@@ -295,7 +171,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 180);
         assert_eq!(metadata.height, 120);
-        assert_eq!(metadata.format, "TIFF");
+        assert_eq!(metadata.format, "Tiff");
     }
 
     #[test]
@@ -307,7 +183,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 64);
         assert_eq!(metadata.height, 64);
-        assert_eq!(metadata.format, "GIF");
+        assert_eq!(metadata.format, "Gif");
     }
 
     #[test]
@@ -341,8 +217,8 @@ mod tests {
         let png_metadata = extract_image_metadata(&png_bytes).unwrap();
         let jpeg_metadata = extract_image_metadata(&jpeg_bytes).unwrap();
 
-        assert_eq!(png_metadata.format, "PNG");
-        assert_eq!(jpeg_metadata.format, "JPEG");
+        assert_eq!(png_metadata.format, "Png");
+        assert_eq!(jpeg_metadata.format, "Jpeg");
     }
 
     #[test]
@@ -408,7 +284,7 @@ mod tests {
         let metadata = result.unwrap();
         assert_eq!(metadata.width, 1);
         assert_eq!(metadata.height, 1);
-        assert_eq!(metadata.format, "PNG");
+        assert_eq!(metadata.format, "Png");
     }
 
     #[test]
@@ -485,8 +361,8 @@ mod tests {
         let jpeg_meta = extract_image_metadata(&jpeg_bytes).unwrap();
         let webp_meta = extract_image_metadata(&webp_bytes).unwrap();
 
-        assert_eq!(png_meta.format, "PNG");
-        assert_eq!(jpeg_meta.format, "JPEG");
-        assert_eq!(webp_meta.format, "WEBP");
+        assert_eq!(png_meta.format, "Png");
+        assert_eq!(jpeg_meta.format, "Jpeg");
+        assert_eq!(webp_meta.format, "WebP");
     }
 }

data/vendor/kreuzberg/src/extraction/libreoffice.rs

@@ -255,6 +255,7 @@ pub async fn convert_office_doc(
         let stderr = String::from_utf8_lossy(&output.stderr);
         let stdout = String::from_utf8_lossy(&output.stdout);
 
+        // Build detailed error message with both stdout and stderr
         let mut error_details = format!(
             "LibreOffice process failed with return code {}",
             output.status.code().unwrap_or(-1)
@@ -361,7 +362,6 @@ mod tests {
     use super::*;
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_check_libreoffice_available() {
         let result = check_libreoffice_available().await;
         if result.is_err() {
@@ -371,7 +371,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_office_doc_missing_file() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -393,7 +392,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_doc_to_docx_empty_bytes() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -406,7 +404,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_ppt_to_pptx_empty_bytes() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -419,7 +416,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_doc_to_docx_invalid_doc() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -432,7 +428,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_ppt_to_pptx_invalid_ppt() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -445,7 +440,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_office_doc_invalid_target_format() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -466,11 +460,11 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_check_libreoffice_missing_dependency_error() {
         let result = check_libreoffice_available().await;
 
-        if let Err(err) = result {
+        if result.is_err() {
+            let err = result.unwrap_err();
             match err {
                 KreuzbergError::MissingDependency(msg) => {
                     assert!(msg.contains("LibreOffice") || msg.contains("soffice"));
@@ -481,7 +475,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_office_doc_creates_output_dir() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -516,7 +509,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_doc_to_docx_temp_cleanup() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -527,7 +519,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_ppt_to_pptx_temp_cleanup() {
         if check_libreoffice_available().await.is_err() {
             return;
@@ -538,7 +529,6 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(not(target_os = "windows"))]
     async fn test_convert_office_doc_timeout_kills_process() {
         if check_libreoffice_available().await.is_err() {
             return;

data/vendor/kreuzberg/src/extraction/mod.rs

@@ -4,13 +4,6 @@ pub mod text;
 #[cfg(feature = "ocr")]
 pub mod image;
 
-/// Capacity estimation utilities for string pre-allocation.
-///
-/// This module provides functions to estimate the capacity needed for string buffers
-/// based on input file sizes and content types. This enables pre-allocation, reducing
-/// reallocation cycles during string building operations.
-pub mod capacity;
-
 #[cfg(feature = "archives")]
 pub mod archive;
 
@@ -32,6 +25,9 @@ pub mod libreoffice;
 #[cfg(feature = "office")]
 pub mod office_metadata;
 
+#[cfg(feature = "office")]
+pub mod pandoc;
+
 #[cfg(feature = "office")]
 pub mod pptx;
 
@@ -41,9 +37,6 @@ pub mod table;
 #[cfg(feature = "xml")]
 pub mod xml;
 
-#[cfg(any(feature = "office", feature = "html"))]
-pub mod markdown;
-
 pub use structured::{JsonExtractionConfig, StructuredDataResult, parse_json, parse_toml, parse_yaml};
 pub use text::parse_text;
 
@@ -70,9 +63,8 @@ pub use libreoffice::{check_libreoffice_available, convert_doc_to_docx, convert_
 
 #[cfg(feature = "office")]
 pub use office_metadata::{
-    CoreProperties, CustomProperties, DocxAppProperties, OdtProperties, PptxAppProperties, XlsxAppProperties,
-    extract_core_properties, extract_custom_properties, extract_docx_app_properties, extract_odt_properties,
-    extract_pptx_app_properties, extract_xlsx_app_properties,
+    CoreProperties, CustomProperties, DocxAppProperties, PptxAppProperties, XlsxAppProperties, extract_core_properties,
+    extract_custom_properties, extract_docx_app_properties, extract_pptx_app_properties, extract_xlsx_app_properties,
 };
 
 #[cfg(feature = "office")]
@@ -83,11 +75,3 @@ pub use table::table_from_arrow_to_markdown;
 
 #[cfg(feature = "xml")]
 pub use xml::parse_xml;
-
-#[cfg(any(feature = "office", feature = "html"))]
-pub use markdown::cells_to_markdown;
-
-pub use capacity::{
-    estimate_content_capacity, estimate_html_markdown_capacity, estimate_presentation_capacity,
-    estimate_spreadsheet_capacity, estimate_table_markdown_capacity,
-};

data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs

@@ -35,7 +35,6 @@
 pub mod app_properties;
 pub mod core_properties;
 pub mod custom_properties;
-pub mod odt_properties;
 
 pub use app_properties::{
     DocxAppProperties, PptxAppProperties, XlsxAppProperties, extract_docx_app_properties, extract_pptx_app_properties,
@@ -43,7 +42,6 @@ pub use app_properties::{
 };
 pub use core_properties::{CoreProperties, extract_core_properties};
 pub use custom_properties::{CustomProperties, extract_custom_properties};
-pub use odt_properties::{OdtProperties, extract_odt_properties};
 
 use roxmltree::Node;
 

data/vendor/kreuzberg/src/extraction/pandoc/batch.rs

@@ -0,0 +1,275 @@
+//! Batch extraction with automatic pandoc-server mode optimization
+//!
+//! This module provides intelligent batch processing for Pandoc extractions with automatic
+//! server mode optimization. The `BatchExtractor` automatically detects if pandoc-server is
+//! available and uses it when beneficial for performance.
+//!
+//! # Server Mode Detection
+//!
+//! The extractor checks for pandoc-server availability in two ways:
+//! 1. Direct binary check: Looks for `pandoc-server` in PATH
+//! 2. Version detection: Checks if pandoc 3.8+ is installed (supports server mode)
+//!
+//! # Optimization Heuristic
+//!
+//! - **>3 files**: Uses server mode (amortizes ~100-200ms startup overhead)
+//! - **≤3 files**: Uses subprocess mode (avoids server startup cost)
+//! - **Server unavailable**: Always uses subprocess mode (graceful fallback)
+//!
+//! # Example Usage
+//!
+//! ```no_run
+//! use kreuzberg::extraction::pandoc::BatchExtractor;
+//! use std::path::Path;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Create extractor (auto-detects server availability)
+//!     let extractor = BatchExtractor::new().await;
+//!
+//!     // Extract multiple files
+//!     let paths = vec![
+//!         Path::new("doc1.docx"),
+//!         Path::new("doc2.docx"),
+//!         Path::new("doc3.docx"),
+//!         Path::new("doc4.docx"),
+//!     ];
+//!     let formats = vec!["docx", "docx", "docx", "docx"];
+//!
+//!     let results = extractor.extract_files(&paths, &formats).await?;
+//!
+//!     // Process results
+//!     for (i, result) in results.iter().enumerate() {
+//!         match result {
+//!             Ok(extraction) => println!("File {}: {} chars", i, extraction.content.len()),
+//!             Err(e) => eprintln!("File {}: Error: {}", i, e),
+//!         }
+//!     }
+//!
+//!     // Cleanup (optional, happens automatically on drop)
+//!     extractor.shutdown().await?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Logging
+//!
+//! Enable tracing to see server mode detection and usage:
+//!
+//! ```bash
+//! RUST_LOG=kreuzberg=debug cargo run
+//! ```
+//!
+//! Expected logs:
+//! - `DEBUG`: Server detection results, file counts
+//! - `INFO`: Server startup confirmation
+//! - `WARN`: Server failures with troubleshooting guidance
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PandocExtractionResult;
+use serde_json::Value;
+use std::path::Path;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+use super::server::PandocServer;
+use super::subprocess;
+
+/// Batch extractor with automatic server mode optimization
+///
+/// Automatically detects pandoc-server availability and uses it when beneficial for
+/// performance (>3 files). Falls back to subprocess mode gracefully.
+///
+/// # Performance
+///
+/// - **Server mode**: ~100-200ms savings per file (startup overhead eliminated)
+/// - **Subprocess mode**: Lightweight, suitable for small batches
+///
+/// # Thread Safety
+///
+/// This struct is safe to use across multiple async tasks. The server instance is
+/// protected by an Arc<Mutex> and can be safely shared.
+pub struct BatchExtractor {
+    server: Arc<Mutex<Option<PandocServer>>>,
+    use_server: bool,
+}
+
+impl BatchExtractor {
+    /// Create a new batch extractor
+    ///
+    /// Automatically detects if pandoc-server is available and uses it if beneficial
+    pub async fn new() -> Self {
+        let use_server = PandocServer::is_server_available().await;
+
+        if use_server {
+            tracing::info!("Pandoc server mode available - will use for batch processing");
+        } else {
+            tracing::debug!("Pandoc server mode not available - using subprocess mode");
+            tracing::debug!("To enable server mode:");
+            tracing::debug!("  1. Install pandoc 3.8+ (current version may be older)");
+            tracing::debug!("  2. OR create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+        }
+
+        Self {
+            server: Arc::new(Mutex::new(None)),
+            use_server,
+        }
+    }
+
+    /// Extract multiple files using optimal strategy
+    ///
+    /// Automatically starts server if available and beneficial (>3 files)
+    pub async fn extract_files(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        if paths.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if self.use_server && paths.len() > 3 {
+            self.extract_with_server(paths, formats).await
+        } else {
+            self.extract_with_subprocess(paths, formats).await
+        }
+    }
+
+    /// Extract using server mode (warm instance)
+    async fn extract_with_server(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        let mut server_lock = self.server.lock().await;
+
+        if server_lock.is_none() {
+            match PandocServer::new(None, None).await {
+                Ok(server) => {
+                    if let Err(e) = server.start().await {
+                        tracing::warn!("Failed to start pandoc-server: {}", e);
+                        tracing::warn!("Falling back to subprocess mode");
+                        tracing::warn!("To fix:");
+                        tracing::warn!("  1. Ensure pandoc 3.8+ is installed: pandoc --version");
+                        tracing::warn!("  2. Create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+                        drop(server_lock);
+                        return self.extract_with_subprocess(paths, formats).await;
+                    }
+
+                    tracing::info!("Started pandoc-server for batch processing ({} files)", paths.len());
+                    *server_lock = Some(server);
+                }
+                Err(e) => {
+                    tracing::warn!("Failed to create pandoc-server: {}", e);
+                    tracing::warn!("Falling back to subprocess mode");
+                    drop(server_lock);
+                    return self.extract_with_subprocess(paths, formats).await;
+                }
+            }
+        } else {
+            tracing::debug!("Reusing warm pandoc-server instance for {} files", paths.len());
+        }
+
+        let server = server_lock.as_ref().unwrap();
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (i, (path, format)) in paths.iter().zip(formats.iter()).enumerate() {
+            tracing::debug!("Extracting file {}/{} via server: {:?}", i + 1, paths.len(), path);
+
+            let content = match tokio::fs::read(path).await {
+                Ok(c) => c,
+                Err(e) => {
+                    results.push(Err(KreuzbergError::Io(e)));
+                    continue;
+                }
+            };
+
+            let result = match server.convert(&String::from_utf8_lossy(&content), format, "json").await {
+                Ok(json_output) => match serde_json::from_str::<Value>(&json_output) {
+                    Ok(json_data) => {
+                        let content = subprocess::extract_content_from_json(&json_data)?;
+                        let metadata = subprocess::extract_metadata_from_json(&json_data)?;
+
+                        Ok(PandocExtractionResult { content, metadata })
+                    }
+                    Err(e) => Err(KreuzbergError::parsing(format!(
+                        "Failed to parse JSON from server: {}",
+                        e
+                    ))),
+                },
+                Err(e) => Err(e),
+            };
+
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Extract using subprocess mode
+    async fn extract_with_subprocess(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        tracing::debug!("Extracting {} files via subprocess mode", paths.len());
+
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (path, format) in paths.iter().zip(formats.iter()) {
+            let result = subprocess::extract_with_pandoc(path, format)
+                .await
+                .map(|(content, metadata)| PandocExtractionResult { content, metadata });
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Stop the server if running
+    pub async fn shutdown(&self) -> Result<()> {
+        let mut server_lock = self.server.lock().await;
+        if let Some(server) = server_lock.take() {
+            tracing::info!("Shutting down pandoc-server");
+            server.stop().await?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for BatchExtractor {
+    fn drop(&mut self) {
+        if let Some(server) = self.server.try_lock().ok().and_then(|mut s| s.take()) {
+            tokio::spawn(async move {
+                let _ = server.stop().await;
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_batch_extractor_creation() {
+        let extractor = BatchExtractor::new().await;
+        assert!(extractor.server.lock().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_empty_batch() {
+        let extractor = BatchExtractor::new().await;
+        let results = extractor.extract_files(&[], &[]).await;
+        assert!(results.is_ok());
+        assert!(results.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_shutdown() {
+        let extractor = BatchExtractor::new().await;
+        let result = extractor.shutdown().await;
+        assert!(result.is_ok());
+    }
+}