kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/core/extractor/bytes.rs

@@ -0,0 +1,116 @@
+//! Byte array extraction operations.
+//!
+//! This module handles extraction from in-memory byte arrays, including:
+//! - MIME type validation
+//! - Legacy format conversion (DOC, PPT)
+//! - Extraction pipeline orchestration
+
+#[cfg(not(feature = "office"))]
+use crate::KreuzbergError;
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::core::mime::{LEGACY_POWERPOINT_MIME_TYPE, LEGACY_WORD_MIME_TYPE};
+#[cfg(feature = "office")]
+use crate::extraction::libreoffice::{convert_doc_to_docx, convert_ppt_to_pptx};
+use crate::types::ExtractionResult;
+
+#[cfg(feature = "office")]
+use super::file::apply_libreoffice_metadata;
+use super::file::extract_bytes_with_extractor;
+#[cfg(feature = "otel")]
+use super::file::record_error;
+
+/// Extract content from a byte array.
+///
+/// This is the main entry point for in-memory extraction. It performs the following steps:
+/// 1. Validate MIME type
+/// 2. Handle legacy format conversion if needed
+/// 3. Select appropriate extractor from registry
+/// 4. Extract content
+/// 5. Run post-processing pipeline
+///
+/// # Arguments
+///
+/// * `content` - The byte array to extract
+/// * `mime_type` - MIME type of the content
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` containing the extracted content and metadata.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if MIME type is invalid.
+/// Returns `KreuzbergError::UnsupportedFormat` if MIME type is not supported.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_bytes;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let bytes = b"Hello, world!";
+/// let result = extract_bytes(bytes, "text/plain", &config).await?;
+/// println!("Content: {}", result.content);
+/// # Ok(())
+/// # }
+/// ```
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, content),
+    fields(
+        extraction.mime_type = mime_type,
+        extraction.size_bytes = content.len(),
+    )
+))]
+pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    use crate::core::mime;
+
+    let result = async {
+        let validated_mime = mime::validate_mime_type(mime_type)?;
+
+        match validated_mime.as_str() {
+            #[cfg(feature = "office")]
+            LEGACY_WORD_MIME_TYPE => {
+                let conversion = convert_doc_to_docx(content).await?;
+                let mut result =
+                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+                apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
+                return Ok(result);
+            }
+            #[cfg(not(feature = "office"))]
+            LEGACY_WORD_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
+                ));
+            }
+            #[cfg(feature = "office")]
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                let conversion = convert_ppt_to_pptx(content).await?;
+                let mut result =
+                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+                apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
+                return Ok(result);
+            }
+            #[cfg(not(feature = "office"))]
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
+                ));
+            }
+            _ => {}
+        }
+
+        extract_bytes_with_extractor(content, &validated_mime, config).await
+    }
+    .await;
+
+    #[cfg(feature = "otel")]
+    if let Err(ref e) = result {
+        record_error(e);
+    }
+
+    result
+}

data/vendor/kreuzberg/src/core/extractor/file.rs

@@ -0,0 +1,240 @@
+//! File-based extraction operations.
+//!
+//! This module handles extraction from filesystem paths, including:
+//! - MIME type detection and validation
+//! - Legacy format conversion (DOC, PPT)
+//! - File validation and reading
+//! - Extraction pipeline orchestration
+
+#[cfg(any(feature = "otel", not(feature = "office")))]
+use crate::KreuzbergError;
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::core::mime::{LEGACY_POWERPOINT_MIME_TYPE, LEGACY_WORD_MIME_TYPE};
+#[cfg(feature = "office")]
+use crate::extraction::libreoffice::{convert_doc_to_docx, convert_ppt_to_pptx};
+use crate::types::ExtractionResult;
+#[cfg(feature = "office")]
+use crate::types::LibreOfficeConversionResult;
+#[cfg(feature = "office")]
+use serde_json::json;
+use std::path::Path;
+
+#[cfg(feature = "office")]
+use super::helpers::pool_mime_type;
+
+use super::helpers::get_extractor;
+
+/// Sanitize a file path to return only the filename.
+///
+/// This function extracts the filename from a path to avoid recording
+/// potentially sensitive full file paths in telemetry data.
+///
+/// # Arguments
+///
+/// * `path` - The path to sanitize
+///
+/// # Returns
+///
+/// The filename as a string, or "unknown" if extraction fails
+///
+/// # Security
+///
+/// This prevents PII (personally identifiable information) from appearing in
+/// traces by only recording filenames instead of full paths.
+///
+/// # Example
+///
+/// ```rust,ignore
+/// let path = Path::new("/home/user/documents/secret.pdf");
+/// assert_eq!(sanitize_path(path), "secret.pdf");
+/// ```
+#[cfg(feature = "otel")]
+pub(super) fn sanitize_path(path: &Path) -> String {
+    path.file_name()
+        .and_then(|n| n.to_str())
+        .unwrap_or("unknown")
+        .to_string()
+}
+
+/// Record error information in the current OpenTelemetry span.
+///
+/// This function records error details in the current span when the `otel` feature is enabled.
+/// It marks the span with `otel.status_code=ERROR` and adds error type and message fields.
+///
+/// # Arguments
+///
+/// * `error` - The error to record in the span
+///
+/// # Example
+///
+/// ```rust,ignore
+/// let result = extract_file("doc.pdf", None, &config).await;
+/// #[cfg(feature = "otel")]
+/// if let Err(ref e) = result {
+///     record_error(e);
+/// }
+/// result
+/// ```
+#[cfg(feature = "otel")]
+pub(in crate::core::extractor) fn record_error(error: &KreuzbergError) {
+    let span = tracing::Span::current();
+    span.record("otel.status_code", "ERROR");
+    span.record("error.type", format!("{:?}", error));
+    span.record("error.message", error.to_string());
+}
+
+/// Extract content from a file.
+///
+/// This is the main entry point for file-based extraction. It performs the following steps:
+/// 1. Check cache for existing result (if caching enabled)
+/// 2. Detect or validate MIME type
+/// 3. Select appropriate extractor from registry
+/// 4. Extract content
+/// 5. Run post-processing pipeline
+/// 6. Store result in cache (if caching enabled)
+///
+/// # Arguments
+///
+/// * `path` - Path to the file to extract
+/// * `mime_type` - Optional MIME type override. If None, will be auto-detected
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` containing the extracted content and metadata.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or path is invalid.
+/// Returns `KreuzbergError::UnsupportedFormat` if MIME type is not supported.
+/// Returns `KreuzbergError::Io` for file I/O errors (these always bubble up).
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_file;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let result = extract_file("document.pdf", None, &config).await?;
+/// println!("Content: {}", result.content);
+/// # Ok(())
+/// # }
+/// ```
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, path),
+    fields(
+        extraction.filename = tracing::field::Empty,
+    )
+))]
+pub async fn extract_file(
+    path: impl AsRef<Path>,
+    mime_type: Option<&str>,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    use crate::core::{io, mime};
+
+    let path = path.as_ref();
+
+    #[cfg(feature = "otel")]
+    {
+        let span = tracing::Span::current();
+        span.record("extraction.filename", sanitize_path(path));
+    }
+
+    let result = async {
+        io::validate_file_exists(path)?;
+
+        let detected_mime = mime::detect_or_validate(Some(path), mime_type)?;
+
+        match detected_mime.as_str() {
+            #[cfg(feature = "office")]
+            LEGACY_WORD_MIME_TYPE => {
+                let original_bytes = tokio::fs::read(path).await?;
+                let conversion = convert_doc_to_docx(&original_bytes).await?;
+                let mut result =
+                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+                apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
+                return Ok(result);
+            }
+            #[cfg(not(feature = "office"))]
+            LEGACY_WORD_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
+                ));
+            }
+            #[cfg(feature = "office")]
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                let original_bytes = tokio::fs::read(path).await?;
+                let conversion = convert_ppt_to_pptx(&original_bytes).await?;
+                let mut result =
+                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+                apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
+                return Ok(result);
+            }
+            #[cfg(not(feature = "office"))]
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
+                ));
+            }
+            _ => {}
+        }
+
+        extract_file_with_extractor(path, &detected_mime, config).await
+    }
+    .await;
+
+    #[cfg(feature = "otel")]
+    if let Err(ref e) = result {
+        record_error(e);
+    }
+
+    result
+}
+
+pub(in crate::core::extractor) async fn extract_file_with_extractor(
+    path: &Path,
+    mime_type: &str,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(mime_type)?;
+    let mut result = extractor.extract_file(path, mime_type, config).await?;
+    result = crate::core::pipeline::run_pipeline(result, config).await?;
+    Ok(result)
+}
+
+pub(in crate::core::extractor) async fn extract_bytes_with_extractor(
+    content: &[u8],
+    mime_type: &str,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(mime_type)?;
+    let mut result = extractor.extract_bytes(content, mime_type, config).await?;
+    result = crate::core::pipeline::run_pipeline(result, config).await?;
+    Ok(result)
+}
+
+#[cfg(feature = "office")]
+pub(in crate::core::extractor) fn apply_libreoffice_metadata(
+    result: &mut ExtractionResult,
+    legacy_mime: &str,
+    conversion: &LibreOfficeConversionResult,
+) {
+    result.mime_type = pool_mime_type(legacy_mime);
+    result.metadata.additional.insert(
+        "libreoffice_conversion".to_string(),
+        json!({
+            "converter": "libreoffice",
+            "original_format": conversion.original_format,
+            "target_format": conversion.target_format,
+            "target_mime": conversion.target_mime,
+        }),
+    );
+}

data/vendor/kreuzberg/src/core/extractor/helpers.rs

@@ -0,0 +1,71 @@
+//! Helper functions and utilities for extraction operations.
+//!
+//! This module provides shared utilities used across extraction modules.
+
+use crate::plugins::DocumentExtractor;
+#[cfg(feature = "office")]
+use crate::utils::intern_mime_type;
+use crate::utils::{PoolSizeHint, estimate_pool_size};
+use crate::{KreuzbergError, Result};
+use std::sync::Arc;
+
+/// Get an extractor from the registry.
+///
+/// This function acquires the registry read lock and retrieves the appropriate
+/// extractor for the given MIME type.
+///
+/// # Performance
+///
+/// RwLock read + HashMap lookup is ~100ns, fast enough without caching.
+/// Removed thread-local cache to avoid Tokio work-stealing scheduler issues.
+pub(in crate::core::extractor) fn get_extractor(mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
+    let registry = crate::plugins::registry::get_document_extractor_registry();
+    let registry_read = registry
+        .read()
+        .map_err(|e| KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
+    registry_read.get(mime_type)
+}
+
+/// Get optimal pool sizing hint for a document.
+///
+/// This function calculates recommended pool sizes based on the document's
+/// file size and MIME type. The hint can be used to create appropriately
+/// sized thread pools for extraction, reducing memory waste from over-allocation.
+///
+/// # Arguments
+///
+/// * `file_size` - The size of the file in bytes
+/// * `mime_type` - The MIME type of the document
+///
+/// # Returns
+///
+/// A `PoolSizeHint` with recommended pool configurations
+///
+/// # Example
+///
+/// ```rust,ignore
+/// use kreuzberg::core::extractor::get_pool_sizing_hint;
+///
+/// let hint = get_pool_sizing_hint(5_000_000, "application/pdf");
+/// println!("Recommended string buffers: {}", hint.string_buffer_count);
+/// ```
+#[inline]
+pub fn get_pool_sizing_hint(file_size: u64, mime_type: &str) -> PoolSizeHint {
+    estimate_pool_size(file_size, mime_type)
+}
+
+/// Convert a MIME type string to a pooled String for efficient deduplication.
+///
+/// This function uses the string interning pool to reduce memory allocations
+/// for repeatedly used MIME types (e.g., "application/pdf" appears thousands of times
+/// in batch processing). The interned string is converted to an owned String to satisfy
+/// the ExtractionResult::mime_type field type.
+///
+/// # Performance
+///
+/// For pre-interned MIME types (all common types), this is O(1) pointer dereference.
+/// For unknown MIME types, this allocates once per unique type and caches the result.
+#[cfg(feature = "office")]
+pub(in crate::core::extractor) fn pool_mime_type(mime_type: &str) -> String {
+    intern_mime_type(mime_type).to_string()
+}

data/vendor/kreuzberg/src/core/extractor/legacy.rs

@@ -0,0 +1,62 @@
+//! Legacy synchronous extraction for WASM compatibility.
+//!
+//! This module provides truly synchronous extraction implementations
+//! for environments where Tokio runtime is not available (e.g., WASM).
+
+/// Synchronous extraction implementation for WASM compatibility.
+///
+/// This function performs extraction without requiring a tokio runtime.
+/// It calls the sync extractor methods directly.
+///
+/// # Arguments
+///
+/// * `content` - The byte content to extract
+/// * `mime_type` - Optional MIME type to validate/use
+/// * `config` - Optional extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` or a `KreuzbergError`
+///
+/// # Implementation Notes
+///
+/// This is called when the `tokio-runtime` feature is disabled.
+/// It replicates the logic of `extract_bytes` but uses synchronous extractor methods.
+#[cfg(not(feature = "tokio-runtime"))]
+pub(super) fn extract_bytes_sync_impl(
+    content: Vec<u8>,
+    mime_type: Option<String>,
+    config: Option<crate::core::config::ExtractionConfig>,
+) -> crate::Result<crate::types::ExtractionResult> {
+    use crate::KreuzbergError;
+    use crate::core::extractor::helpers::get_extractor;
+    use crate::core::mime;
+
+    let config = config.unwrap_or_default();
+
+    let validated_mime = if let Some(mime) = mime_type {
+        mime::validate_mime_type(&mime)?
+    } else {
+        return Err(KreuzbergError::Validation {
+            message: "MIME type is required for synchronous extraction".to_string(),
+            source: None,
+        });
+    };
+
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(&validated_mime)?;
+
+    let sync_extractor = extractor.as_sync_extractor().ok_or_else(|| {
+        KreuzbergError::UnsupportedFormat(format!(
+            "Extractor for '{}' does not support synchronous extraction",
+            validated_mime
+        ))
+    })?;
+
+    let mut result = sync_extractor.extract_sync(&content, &validated_mime, &config)?;
+
+    result = crate::core::pipeline::run_pipeline_sync(result, &config)?;
+
+    Ok(result)
+}