kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs

@@ -0,0 +1,240 @@
+//! Custom properties extraction from docProps/custom.xml
+//!
+//! Extracts user-defined custom metadata from Office Open XML documents.
+
+use crate::error::{KreuzbergError, Result};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::io::Read;
+use zip::ZipArchive;
+
+/// Custom properties from docProps/custom.xml
+///
+/// Maps property names to their values. Values are converted to JSON types
+/// based on the VT (Variant Type) specified in the XML.
+pub type CustomProperties = HashMap<String, Value>;
+
+/// Extract custom properties from an Office Open XML document
+///
+/// Parses `docProps/custom.xml` and extracts user-defined or application-defined
+/// custom metadata.
+///
+/// # Arguments
+///
+/// * `archive` - ZIP archive containing the Office document
+///
+/// # Returns
+///
+/// Returns a `HashMap` of property names to JSON values. Supported VT types:
+/// - `vt:lpwstr` / `vt:lpstr` → String
+/// - `vt:i4` → Number (integer)
+/// - `vt:r8` → Number (float)
+/// - `vt:bool` → Boolean
+/// - `vt:filetime` → String (ISO 8601 timestamp)
+///
+/// # Example
+///
+/// ```no_run
+/// use kreuzberg::extraction::office_metadata::extract_custom_properties;
+/// use std::fs::File;
+/// use zip::ZipArchive;
+///
+/// let file = File::open("document.docx")?;
+/// let mut archive = ZipArchive::new(file)?;
+/// let custom = extract_custom_properties(&mut archive)?;
+///
+/// for (name, value) in custom {
+///     println!("{}: {:?}", name, value);
+/// }
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+pub fn extract_custom_properties<R: Read + std::io::Seek>(archive: &mut ZipArchive<R>) -> Result<CustomProperties> {
+    let mut xml_content = String::new();
+
+    match archive.by_name("docProps/custom.xml") {
+        Ok(mut file) => {
+            file.read_to_string(&mut xml_content)
+                .map_err(|e| KreuzbergError::parsing(format!("Failed to read custom.xml: {}", e)))?;
+        }
+        Err(_) => {
+            return Ok(HashMap::new());
+        }
+    }
+
+    let doc = roxmltree::Document::parse(&xml_content)
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to parse custom.xml: {}", e)))?;
+
+    let root = doc.root_element();
+    let mut properties = HashMap::new();
+
+    for property_node in root.descendants().filter(|n| n.has_tag_name("property")) {
+        if let Some(name) = property_node.attribute("name")
+            && let Some(value) = extract_vt_value(property_node)
+        {
+            properties.insert(name.to_string(), value);
+        }
+    }
+
+    Ok(properties)
+}
+
+/// Extract value from a VT (Variant Type) element
+///
+/// Handles various VT types and converts them to appropriate JSON values.
+fn extract_vt_value(node: roxmltree::Node) -> Option<Value> {
+    for child in node.children().filter(|n| n.is_element()) {
+        let tag = child.tag_name().name();
+
+        match tag {
+            "lpwstr" | "lpstr" => {
+                return child.text().map(|s| Value::String(s.to_string()));
+            }
+            "i4" => {
+                return child
+                    .text()
+                    .and_then(|s| s.trim().parse::<i64>().ok().map(|n| Value::Number(n.into())));
+            }
+            "r8" => {
+                return child.text().and_then(|s| {
+                    s.trim()
+                        .parse::<f64>()
+                        .ok()
+                        .and_then(|f| serde_json::Number::from_f64(f).map(Value::Number))
+                });
+            }
+            "bool" => {
+                return child.text().and_then(|s| match s.trim().to_lowercase().as_str() {
+                    "true" | "1" => Some(Value::Bool(true)),
+                    "false" | "0" => Some(Value::Bool(false)),
+                    _ => None,
+                });
+            }
+            "filetime" => {
+                return child.text().map(|s| Value::String(s.to_string()));
+            }
+            _ => {
+                continue;
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::{Cursor, Write};
+
+    fn create_test_zip_with_custom_xml(custom_xml: &str) -> ZipArchive<Cursor<Vec<u8>>> {
+        let buffer = Vec::new();
+        let cursor = Cursor::new(buffer);
+        let mut zip = zip::ZipWriter::new(cursor);
+
+        let options = zip::write::FileOptions::<()>::default().compression_method(zip::CompressionMethod::Stored);
+
+        zip.start_file("docProps/custom.xml", options).unwrap();
+        zip.write_all(custom_xml.as_bytes()).unwrap();
+
+        let cursor = zip.finish().unwrap();
+        ZipArchive::new(cursor).unwrap()
+    }
+
+    #[test]
+    fn test_extract_custom_properties_string() {
+        let custom_xml = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/custom-properties"
+            xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+    <property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="2" name="ProjectName">
+        <vt:lpwstr>Alpha Project</vt:lpwstr>
+    </property>
+</Properties>"#;
+
+        let mut archive = create_test_zip_with_custom_xml(custom_xml);
+        let props = extract_custom_properties(&mut archive).unwrap();
+
+        assert_eq!(
+            props.get("ProjectName"),
+            Some(&Value::String("Alpha Project".to_string()))
+        );
+    }
+
+    #[test]
+    fn test_extract_custom_properties_multiple_types() {
+        let custom_xml = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/custom-properties"
+            xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+    <property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="2" name="StringProp">
+        <vt:lpwstr>Test String</vt:lpwstr>
+    </property>
+    <property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="3" name="IntProp">
+        <vt:i4>42</vt:i4>
+    </property>
+    <property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="4" name="BoolProp">
+        <vt:bool>true</vt:bool>
+    </property>
+    <property fmtid="{D5CDD505-2E9C-101B-9397-08002B2CF9AE}" pid="5" name="FloatProp">
+        <vt:r8>3.14159</vt:r8>
+    </property>
+</Properties>"#;
+
+        let mut archive = create_test_zip_with_custom_xml(custom_xml);
+        let props = extract_custom_properties(&mut archive).unwrap();
+
+        assert_eq!(props.get("StringProp"), Some(&Value::String("Test String".to_string())));
+        assert_eq!(props.get("IntProp"), Some(&Value::Number(42.into())));
+        assert_eq!(props.get("BoolProp"), Some(&Value::Bool(true)));
+        assert!(matches!(props.get("FloatProp"), Some(Value::Number(_))));
+    }
+
+    #[test]
+    fn test_extract_custom_properties_missing_file() {
+        let buffer = Vec::new();
+        let cursor = Cursor::new(buffer);
+        let zip = zip::ZipWriter::new(cursor);
+        let cursor = zip.finish().unwrap();
+        let mut archive = ZipArchive::new(cursor).unwrap();
+
+        let props = extract_custom_properties(&mut archive).unwrap();
+        assert_eq!(props, HashMap::new());
+    }
+
+    #[test]
+    fn test_extract_vt_value_lpstr() {
+        let xml = r#"<property xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+            <vt:lpstr>Test</vt:lpstr>
+        </property>"#;
+
+        let doc = roxmltree::Document::parse(xml).unwrap();
+        let value = extract_vt_value(doc.root_element()).unwrap();
+        assert_eq!(value, Value::String("Test".to_string()));
+    }
+
+    #[test]
+    fn test_extract_vt_value_bool_variants() {
+        let xml_true1 = r#"<property xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+            <vt:bool>true</vt:bool>
+        </property>"#;
+        let xml_true2 = r#"<property xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+            <vt:bool>1</vt:bool>
+        </property>"#;
+        let xml_false1 = r#"<property xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+            <vt:bool>false</vt:bool>
+        </property>"#;
+        let xml_false2 = r#"<property xmlns:vt="http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes">
+            <vt:bool>0</vt:bool>
+        </property>"#;
+
+        let doc1 = roxmltree::Document::parse(xml_true1).unwrap();
+        assert_eq!(extract_vt_value(doc1.root_element()), Some(Value::Bool(true)));
+
+        let doc2 = roxmltree::Document::parse(xml_true2).unwrap();
+        assert_eq!(extract_vt_value(doc2.root_element()), Some(Value::Bool(true)));
+
+        let doc3 = roxmltree::Document::parse(xml_false1).unwrap();
+        assert_eq!(extract_vt_value(doc3.root_element()), Some(Value::Bool(false)));
+
+        let doc4 = roxmltree::Document::parse(xml_false2).unwrap();
+        assert_eq!(extract_vt_value(doc4.root_element()), Some(Value::Bool(false)));
+    }
+}

data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs

@@ -0,0 +1,128 @@
+//! Office Open XML metadata extraction
+//!
+//! This module provides functionality to extract comprehensive metadata from Office Open XML
+//! documents (DOCX, XLSX, PPTX) by parsing the `docProps/core.xml`, `docProps/app.xml`,
+//! and `docProps/custom.xml` files within the ZIP container.
+//!
+//! # Overview
+//!
+//! Office documents store metadata in three XML files:
+//! - `docProps/core.xml` - Dublin Core metadata (title, creator, dates, keywords, etc.)
+//! - `docProps/app.xml` - Application-specific properties (page count, word count, etc.)
+//! - `docProps/custom.xml` - Custom properties defined by users or applications
+//!
+//! # Example
+//!
+//! ```no_run
+//! use kreuzberg::extraction::office_metadata::{extract_core_properties, extract_docx_app_properties};
+//! use std::fs::File;
+//! use zip::ZipArchive;
+//!
+//! let file = File::open("document.docx")?;
+//! let mut archive = ZipArchive::new(file)?;
+//!
+//! // Extract core properties
+//! let core = extract_core_properties(&mut archive)?;
+//! println!("Title: {:?}", core.title);
+//! println!("Created: {:?}", core.created);
+//!
+//! // Extract DOCX app properties
+//! let app = extract_docx_app_properties(&mut archive)?;
+//! println!("Word count: {:?}", app.words);
+//! # Ok::<(), Box<dyn std::error::Error>>(())
+//! ```
+
+pub mod app_properties;
+pub mod core_properties;
+pub mod custom_properties;
+
+pub use app_properties::{
+    DocxAppProperties, PptxAppProperties, XlsxAppProperties, extract_docx_app_properties, extract_pptx_app_properties,
+    extract_xlsx_app_properties,
+};
+pub use core_properties::{CoreProperties, extract_core_properties};
+pub use custom_properties::{CustomProperties, extract_custom_properties};
+
+use roxmltree::Node;
+
+/// Parse text content from an XML element by tag name
+///
+/// Returns the text content if the element exists and has non-empty text.
+pub(crate) fn parse_xml_text(node: Node, name: &str) -> Option<String> {
+    node.descendants()
+        .find(|n| n.has_tag_name(name))
+        .and_then(|n| n.text())
+        .map(|s| s.trim())
+        .filter(|s| !s.is_empty())
+        .map(String::from)
+}
+
+/// Parse integer content from an XML element by tag name
+///
+/// Returns the parsed integer if the element exists and contains valid integer text.
+pub(crate) fn parse_xml_int(node: Node, name: &str) -> Option<i32> {
+    node.descendants()
+        .find(|n| n.has_tag_name(name))
+        .and_then(|n| n.text())
+        .and_then(|s| s.trim().parse::<i32>().ok())
+}
+
+/// Parse boolean content from an XML element by tag name
+///
+/// Handles "true"/"false" string values and converts to boolean.
+pub(crate) fn parse_xml_bool(node: Node, name: &str) -> Option<bool> {
+    node.descendants()
+        .find(|n| n.has_tag_name(name))
+        .and_then(|n| n.text())
+        .map(|s| s.trim())
+        .and_then(|s| match s.to_lowercase().as_str() {
+            "true" => Some(true),
+            "false" => Some(false),
+            _ => None,
+        })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_xml_text() {
+        let xml = r#"<root><title>Test Document</title></root>"#;
+        let doc = roxmltree::Document::parse(xml).unwrap();
+        let root = doc.root_element();
+
+        assert_eq!(parse_xml_text(root, "title"), Some("Test Document".to_string()));
+        assert_eq!(parse_xml_text(root, "missing"), None);
+    }
+
+    #[test]
+    fn test_parse_xml_text_empty() {
+        let xml = r#"<root><title></title></root>"#;
+        let doc = roxmltree::Document::parse(xml).unwrap();
+        let root = doc.root_element();
+
+        assert_eq!(parse_xml_text(root, "title"), None);
+    }
+
+    #[test]
+    fn test_parse_xml_int() {
+        let xml = r#"<root><count>42</count></root>"#;
+        let doc = roxmltree::Document::parse(xml).unwrap();
+        let root = doc.root_element();
+
+        assert_eq!(parse_xml_int(root, "count"), Some(42));
+        assert_eq!(parse_xml_int(root, "missing"), None);
+    }
+
+    #[test]
+    fn test_parse_xml_bool() {
+        let xml = r#"<root><flag>true</flag><other>false</other></root>"#;
+        let doc = roxmltree::Document::parse(xml).unwrap();
+        let root = doc.root_element();
+
+        assert_eq!(parse_xml_bool(root, "flag"), Some(true));
+        assert_eq!(parse_xml_bool(root, "other"), Some(false));
+        assert_eq!(parse_xml_bool(root, "missing"), None);
+    }
+}

data/vendor/kreuzberg/src/extraction/pandoc/batch.rs

@@ -0,0 +1,275 @@
+//! Batch extraction with automatic pandoc-server mode optimization
+//!
+//! This module provides intelligent batch processing for Pandoc extractions with automatic
+//! server mode optimization. The `BatchExtractor` automatically detects if pandoc-server is
+//! available and uses it when beneficial for performance.
+//!
+//! # Server Mode Detection
+//!
+//! The extractor checks for pandoc-server availability in two ways:
+//! 1. Direct binary check: Looks for `pandoc-server` in PATH
+//! 2. Version detection: Checks if pandoc 3.8+ is installed (supports server mode)
+//!
+//! # Optimization Heuristic
+//!
+//! - **>3 files**: Uses server mode (amortizes ~100-200ms startup overhead)
+//! - **≤3 files**: Uses subprocess mode (avoids server startup cost)
+//! - **Server unavailable**: Always uses subprocess mode (graceful fallback)
+//!
+//! # Example Usage
+//!
+//! ```no_run
+//! use kreuzberg::extraction::pandoc::BatchExtractor;
+//! use std::path::Path;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Create extractor (auto-detects server availability)
+//!     let extractor = BatchExtractor::new().await;
+//!
+//!     // Extract multiple files
+//!     let paths = vec![
+//!         Path::new("doc1.docx"),
+//!         Path::new("doc2.docx"),
+//!         Path::new("doc3.docx"),
+//!         Path::new("doc4.docx"),
+//!     ];
+//!     let formats = vec!["docx", "docx", "docx", "docx"];
+//!
+//!     let results = extractor.extract_files(&paths, &formats).await?;
+//!
+//!     // Process results
+//!     for (i, result) in results.iter().enumerate() {
+//!         match result {
+//!             Ok(extraction) => println!("File {}: {} chars", i, extraction.content.len()),
+//!             Err(e) => eprintln!("File {}: Error: {}", i, e),
+//!         }
+//!     }
+//!
+//!     // Cleanup (optional, happens automatically on drop)
+//!     extractor.shutdown().await?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Logging
+//!
+//! Enable tracing to see server mode detection and usage:
+//!
+//! ```bash
+//! RUST_LOG=kreuzberg=debug cargo run
+//! ```
+//!
+//! Expected logs:
+//! - `DEBUG`: Server detection results, file counts
+//! - `INFO`: Server startup confirmation
+//! - `WARN`: Server failures with troubleshooting guidance
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PandocExtractionResult;
+use serde_json::Value;
+use std::path::Path;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+use super::server::PandocServer;
+use super::subprocess;
+
+/// Batch extractor with automatic server mode optimization
+///
+/// Automatically detects pandoc-server availability and uses it when beneficial for
+/// performance (>3 files). Falls back to subprocess mode gracefully.
+///
+/// # Performance
+///
+/// - **Server mode**: ~100-200ms savings per file (startup overhead eliminated)
+/// - **Subprocess mode**: Lightweight, suitable for small batches
+///
+/// # Thread Safety
+///
+/// This struct is safe to use across multiple async tasks. The server instance is
+/// protected by an Arc<Mutex> and can be safely shared.
+pub struct BatchExtractor {
+    server: Arc<Mutex<Option<PandocServer>>>,
+    use_server: bool,
+}
+
+impl BatchExtractor {
+    /// Create a new batch extractor
+    ///
+    /// Automatically detects if pandoc-server is available and uses it if beneficial
+    pub async fn new() -> Self {
+        let use_server = PandocServer::is_server_available().await;
+
+        if use_server {
+            tracing::info!("Pandoc server mode available - will use for batch processing");
+        } else {
+            tracing::debug!("Pandoc server mode not available - using subprocess mode");
+            tracing::debug!("To enable server mode:");
+            tracing::debug!("  1. Install pandoc 3.8+ (current version may be older)");
+            tracing::debug!("  2. OR create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+        }
+
+        Self {
+            server: Arc::new(Mutex::new(None)),
+            use_server,
+        }
+    }
+
+    /// Extract multiple files using optimal strategy
+    ///
+    /// Automatically starts server if available and beneficial (>3 files)
+    pub async fn extract_files(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        if paths.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if self.use_server && paths.len() > 3 {
+            self.extract_with_server(paths, formats).await
+        } else {
+            self.extract_with_subprocess(paths, formats).await
+        }
+    }
+
+    /// Extract using server mode (warm instance)
+    async fn extract_with_server(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        let mut server_lock = self.server.lock().await;
+
+        if server_lock.is_none() {
+            match PandocServer::new(None, None).await {
+                Ok(server) => {
+                    if let Err(e) = server.start().await {
+                        tracing::warn!("Failed to start pandoc-server: {}", e);
+                        tracing::warn!("Falling back to subprocess mode");
+                        tracing::warn!("To fix:");
+                        tracing::warn!("  1. Ensure pandoc 3.8+ is installed: pandoc --version");
+                        tracing::warn!("  2. Create symlink: ln -s $(which pandoc) /usr/local/bin/pandoc-server");
+                        drop(server_lock);
+                        return self.extract_with_subprocess(paths, formats).await;
+                    }
+
+                    tracing::info!("Started pandoc-server for batch processing ({} files)", paths.len());
+                    *server_lock = Some(server);
+                }
+                Err(e) => {
+                    tracing::warn!("Failed to create pandoc-server: {}", e);
+                    tracing::warn!("Falling back to subprocess mode");
+                    drop(server_lock);
+                    return self.extract_with_subprocess(paths, formats).await;
+                }
+            }
+        } else {
+            tracing::debug!("Reusing warm pandoc-server instance for {} files", paths.len());
+        }
+
+        let server = server_lock.as_ref().unwrap();
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (i, (path, format)) in paths.iter().zip(formats.iter()).enumerate() {
+            tracing::debug!("Extracting file {}/{} via server: {:?}", i + 1, paths.len(), path);
+
+            let content = match tokio::fs::read(path).await {
+                Ok(c) => c,
+                Err(e) => {
+                    results.push(Err(KreuzbergError::Io(e)));
+                    continue;
+                }
+            };
+
+            let result = match server.convert(&String::from_utf8_lossy(&content), format, "json").await {
+                Ok(json_output) => match serde_json::from_str::<Value>(&json_output) {
+                    Ok(json_data) => {
+                        let content = subprocess::extract_content_from_json(&json_data)?;
+                        let metadata = subprocess::extract_metadata_from_json(&json_data)?;
+
+                        Ok(PandocExtractionResult { content, metadata })
+                    }
+                    Err(e) => Err(KreuzbergError::parsing(format!(
+                        "Failed to parse JSON from server: {}",
+                        e
+                    ))),
+                },
+                Err(e) => Err(e),
+            };
+
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Extract using subprocess mode
+    async fn extract_with_subprocess(
+        &self,
+        paths: &[&Path],
+        formats: &[&str],
+    ) -> Result<Vec<Result<PandocExtractionResult>>> {
+        tracing::debug!("Extracting {} files via subprocess mode", paths.len());
+
+        let mut results = Vec::with_capacity(paths.len());
+
+        for (path, format) in paths.iter().zip(formats.iter()) {
+            let result = subprocess::extract_with_pandoc(path, format)
+                .await
+                .map(|(content, metadata)| PandocExtractionResult { content, metadata });
+            results.push(result);
+        }
+
+        Ok(results)
+    }
+
+    /// Stop the server if running
+    pub async fn shutdown(&self) -> Result<()> {
+        let mut server_lock = self.server.lock().await;
+        if let Some(server) = server_lock.take() {
+            tracing::info!("Shutting down pandoc-server");
+            server.stop().await?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for BatchExtractor {
+    fn drop(&mut self) {
+        if let Some(server) = self.server.try_lock().ok().and_then(|mut s| s.take()) {
+            tokio::spawn(async move {
+                let _ = server.stop().await;
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_batch_extractor_creation() {
+        let extractor = BatchExtractor::new().await;
+        assert!(extractor.server.lock().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_empty_batch() {
+        let extractor = BatchExtractor::new().await;
+        let results = extractor.extract_files(&[], &[]).await;
+        assert!(results.is_ok());
+        assert!(results.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_shutdown() {
+        let extractor = BatchExtractor::new().await;
+        let result = extractor.shutdown().await;
+        assert!(result.is_ok());
+    }
+}