kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/core/extractor.rs

@@ -26,65 +26,6 @@ use serde_json::json;
 use std::path::Path;
 use std::sync::Arc;
 
-/// Record error information in the current OpenTelemetry span.
-///
-/// This function records error details in the current span when the `otel` feature is enabled.
-/// It marks the span with `otel.status_code=ERROR` and adds error type and message fields.
-///
-/// # Arguments
-///
-/// * `error` - The error to record in the span
-///
-/// # Example
-///
-/// ```rust,ignore
-/// let result = extract_file("doc.pdf", None, &config).await;
-/// #[cfg(feature = "otel")]
-/// if let Err(ref e) = result {
-///     record_error(e);
-/// }
-/// result
-/// ```
-#[cfg(feature = "otel")]
-fn record_error(error: &KreuzbergError) {
-    let span = tracing::Span::current();
-    span.record("otel.status_code", "ERROR");
-    span.record("error.type", format!("{:?}", error));
-    span.record("error.message", error.to_string());
-}
-
-/// Sanitize a file path to return only the filename.
-///
-/// This function extracts the filename from a path to avoid recording
-/// potentially sensitive full file paths in telemetry data.
-///
-/// # Arguments
-///
-/// * `path` - The path to sanitize
-///
-/// # Returns
-///
-/// The filename as a string, or "unknown" if extraction fails
-///
-/// # Security
-///
-/// This prevents PII (personally identifiable information) from appearing in
-/// traces by only recording filenames instead of full paths.
-///
-/// # Example
-///
-/// ```rust,ignore
-/// let path = Path::new("/home/user/documents/secret.pdf");
-/// assert_eq!(sanitize_path(path), "secret.pdf");
-/// ```
-#[cfg(feature = "otel")]
-fn sanitize_path(path: &Path) -> String {
-    path.file_name()
-        .and_then(|n| n.to_str())
-        .unwrap_or("unknown")
-        .to_string()
-}
-
 /// Global Tokio runtime for synchronous operations.
 ///
 /// This runtime is lazily initialized on first use and shared across all sync wrappers.
@@ -160,12 +101,6 @@ fn get_extractor(mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
 /// # Ok(())
 /// # }
 /// ```
-#[cfg_attr(feature = "otel", tracing::instrument(
-    skip(config, path),
-    fields(
-        extraction.filename = tracing::field::Empty,
-    )
-))]
 pub async fn extract_file(
     path: impl AsRef<Path>,
     mime_type: Option<&str>,
@@ -175,119 +110,86 @@ pub async fn extract_file(
 
     let path = path.as_ref();
 
-    #[cfg(feature = "otel")]
-    {
-        let span = tracing::Span::current();
-        span.record("extraction.filename", sanitize_path(path));
-    }
-
-    let result = async {
-        io::validate_file_exists(path)?;
+    io::validate_file_exists(path)?;
 
-        let detected_mime = mime::detect_or_validate(Some(path), mime_type)?;
+    let detected_mime = mime::detect_or_validate(Some(path), mime_type)?;
 
-        match detected_mime.as_str() {
-            #[cfg(feature = "office")]
-            LEGACY_WORD_MIME_TYPE => {
-                let original_bytes = tokio::fs::read(path).await?;
-                let conversion = convert_doc_to_docx(&original_bytes).await?;
-                let mut result =
-                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
-                apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
-                return Ok(result);
-            }
-            #[cfg(not(feature = "office"))]
-            LEGACY_WORD_MIME_TYPE => {
-                return Err(KreuzbergError::UnsupportedFormat(
-                    "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
-                ));
-            }
-            #[cfg(feature = "office")]
-            LEGACY_POWERPOINT_MIME_TYPE => {
-                let original_bytes = tokio::fs::read(path).await?;
-                let conversion = convert_ppt_to_pptx(&original_bytes).await?;
-                let mut result =
-                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
-                apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
-                return Ok(result);
-            }
-            #[cfg(not(feature = "office"))]
-            LEGACY_POWERPOINT_MIME_TYPE => {
-                return Err(KreuzbergError::UnsupportedFormat(
-                    "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
-                ));
-            }
-            _ => {}
+    match detected_mime.as_str() {
+        #[cfg(feature = "office")]
+        LEGACY_WORD_MIME_TYPE => {
+            let original_bytes = tokio::fs::read(path).await?;
+            let conversion = convert_doc_to_docx(&original_bytes).await?;
+            let mut result =
+                extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+            apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
+            return Ok(result);
         }
-
-        extract_file_with_extractor(path, &detected_mime, config).await
-    }
-    .await;
-
-    #[cfg(feature = "otel")]
-    if let Err(ref e) = result {
-        record_error(e);
+        #[cfg(not(feature = "office"))]
+        LEGACY_WORD_MIME_TYPE => {
+            return Err(KreuzbergError::UnsupportedFormat(
+                "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
+            ));
+        }
+        #[cfg(feature = "office")]
+        LEGACY_POWERPOINT_MIME_TYPE => {
+            let original_bytes = tokio::fs::read(path).await?;
+            let conversion = convert_ppt_to_pptx(&original_bytes).await?;
+            let mut result =
+                extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+            apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
+            return Ok(result);
+        }
+        #[cfg(not(feature = "office"))]
+        LEGACY_POWERPOINT_MIME_TYPE => {
+            return Err(KreuzbergError::UnsupportedFormat(
+                "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
+            ));
+        }
+        _ => {}
     }
 
-    result
+    extract_file_with_extractor(path, &detected_mime, config).await
 }
 
 /// Extract content from a byte array.
-#[cfg_attr(feature = "otel", tracing::instrument(
-    skip(config, content),
-    fields(
-        extraction.mime_type = mime_type,
-        extraction.size_bytes = content.len(),
-    )
-))]
 pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
     use crate::core::mime;
 
-    let result = async {
-        let validated_mime = mime::validate_mime_type(mime_type)?;
-
-        match validated_mime.as_str() {
-            #[cfg(feature = "office")]
-            LEGACY_WORD_MIME_TYPE => {
-                let conversion = convert_doc_to_docx(content).await?;
-                let mut result =
-                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
-                apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
-                return Ok(result);
-            }
-            #[cfg(not(feature = "office"))]
-            LEGACY_WORD_MIME_TYPE => {
-                return Err(KreuzbergError::UnsupportedFormat(
-                    "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
-                ));
-            }
-            #[cfg(feature = "office")]
-            LEGACY_POWERPOINT_MIME_TYPE => {
-                let conversion = convert_ppt_to_pptx(content).await?;
-                let mut result =
-                    extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
-                apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
-                return Ok(result);
-            }
-            #[cfg(not(feature = "office"))]
-            LEGACY_POWERPOINT_MIME_TYPE => {
-                return Err(KreuzbergError::UnsupportedFormat(
-                    "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
-                ));
-            }
-            _ => {}
-        }
-
-        extract_bytes_with_extractor(content, &validated_mime, config).await
-    }
-    .await;
+    let validated_mime = mime::validate_mime_type(mime_type)?;
 
-    #[cfg(feature = "otel")]
-    if let Err(ref e) = result {
-        record_error(e);
+    match validated_mime.as_str() {
+        #[cfg(feature = "office")]
+        LEGACY_WORD_MIME_TYPE => {
+            let conversion = convert_doc_to_docx(content).await?;
+            let mut result =
+                extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+            apply_libreoffice_metadata(&mut result, LEGACY_WORD_MIME_TYPE, &conversion);
+            return Ok(result);
+        }
+        #[cfg(not(feature = "office"))]
+        LEGACY_WORD_MIME_TYPE => {
+            return Err(KreuzbergError::UnsupportedFormat(
+                "Legacy Word conversion requires the `office` feature or LibreOffice support".to_string(),
+            ));
+        }
+        #[cfg(feature = "office")]
+        LEGACY_POWERPOINT_MIME_TYPE => {
+            let conversion = convert_ppt_to_pptx(content).await?;
+            let mut result =
+                extract_bytes_with_extractor(&conversion.converted_bytes, &conversion.target_mime, config).await?;
+            apply_libreoffice_metadata(&mut result, LEGACY_POWERPOINT_MIME_TYPE, &conversion);
+            return Ok(result);
+        }
+        #[cfg(not(feature = "office"))]
+        LEGACY_POWERPOINT_MIME_TYPE => {
+            return Err(KreuzbergError::UnsupportedFormat(
+                "Legacy PowerPoint conversion requires the `office` feature or LibreOffice support".to_string(),
+            ));
+        }
+        _ => {}
     }
 
-    result
+    extract_bytes_with_extractor(content, &validated_mime, config).await
 }
 
 /// Extract content from multiple files concurrently.
@@ -310,13 +212,6 @@ pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionC
 ///
 /// Individual file errors are captured in the result metadata. System errors
 /// (IO, RuntimeError equivalents) will bubble up and fail the entire batch.
-#[cfg_attr(feature = "otel", tracing::instrument(
-    skip(config, paths),
-    fields(
-        extraction.batch_size = paths.len(),
-    )
-))]
-#[cfg(feature = "tokio-runtime")]
 pub async fn batch_extract_file(
     paths: Vec<impl AsRef<Path>>,
     config: &ExtractionConfig,
@@ -407,13 +302,6 @@ pub async fn batch_extract_file(
 /// # Returns
 ///
 /// A vector of `ExtractionResult` in the same order as the input.
-#[cfg_attr(feature = "otel", tracing::instrument(
-    skip(config, contents),
-    fields(
-        extraction.batch_size = contents.len(),
-    )
-))]
-#[cfg(feature = "tokio-runtime")]
 pub async fn batch_extract_bytes(
     contents: Vec<(&[u8], &str)>,
     config: &ExtractionConfig,
@@ -592,10 +480,6 @@ mod tests {
     use std::io::Write;
     use tempfile::tempdir;
 
-    fn assert_text_content(actual: &str, expected: &str) {
-        assert_eq!(actual.trim_end_matches('\n'), expected);
-    }
-
     #[tokio::test]
     async fn test_extract_file_basic() {
         let dir = tempdir().unwrap();
@@ -608,7 +492,7 @@ mod tests {
 
         assert!(result.is_ok());
         let result = result.unwrap();
-        assert_text_content(&result.content, "Hello, world!");
+        assert_eq!(result.content, "Hello, world!");
         assert_eq!(result.mime_type, "text/plain");
     }
 
@@ -641,7 +525,7 @@ mod tests {
 
         assert!(result.is_ok());
         let result = result.unwrap();
-        assert_text_content(&result.content, "test content");
+        assert_eq!(result.content, "test content");
         assert_eq!(result.mime_type, "text/plain");
     }
 
@@ -669,8 +553,8 @@ mod tests {
         assert!(results.is_ok());
         let results = results.unwrap();
         assert_eq!(results.len(), 2);
-        assert_text_content(&results[0].content, "content 1");
-        assert_text_content(&results[1].content, "content 2");
+        assert_eq!(results[0].content, "content 1");
+        assert_eq!(results[1].content, "content 2");
     }
 
     #[tokio::test]
@@ -695,8 +579,8 @@ mod tests {
         assert!(results.is_ok());
         let results = results.unwrap();
         assert_eq!(results.len(), 2);
-        assert_text_content(&results[0].content, "content 1");
-        assert_text_content(&results[1].content, "content 2");
+        assert_eq!(results[0].content, "content 1");
+        assert_eq!(results[1].content, "content 2");
     }
 
     #[test]
@@ -709,8 +593,7 @@ mod tests {
 
         let result = extract_file_sync(&file_path, None, &config);
         assert!(result.is_ok());
-        let result = result.unwrap();
-        assert_text_content(&result.content, "sync test");
+        assert_eq!(result.unwrap().content, "sync test");
 
         let result = extract_bytes_sync(b"test", "text/plain", &config);
         assert!(result.is_ok());
@@ -722,14 +605,12 @@ mod tests {
 
         let result1 = extract_bytes(b"test 1", "text/plain", &config).await;
         assert!(result1.is_ok());
-        let result1 = result1.unwrap();
 
         let result2 = extract_bytes(b"test 2", "text/plain", &config).await;
         assert!(result2.is_ok());
-        let result2 = result2.unwrap();
 
-        assert_text_content(&result1.content, "test 1");
-        assert_text_content(&result2.content, "test 2");
+        assert_eq!(result1.unwrap().content, "test 1");
+        assert_eq!(result2.unwrap().content, "test 2");
 
         let result3 = extract_bytes(b"# test 3", "text/markdown", &config).await;
         assert!(result3.is_ok());
@@ -795,8 +676,7 @@ mod tests {
         let result = extract_file(&file_path, None, &config).await;
 
         assert!(result.is_ok());
-        let result = result.unwrap();
-        assert_text_content(&result.content, "content");
+        assert_eq!(result.unwrap().content, "content");
     }
 
     #[tokio::test]
@@ -836,7 +716,7 @@ mod tests {
         assert!(results.is_ok());
         let results = results.unwrap();
         assert_eq!(results.len(), 2);
-        assert_text_content(&results[0].content, "valid content");
+        assert_eq!(results[0].content, "valid content");
         assert!(results[1].metadata.error.is_some());
     }
 
@@ -853,9 +733,9 @@ mod tests {
         assert!(results.is_ok());
         let results = results.unwrap();
         assert_eq!(results.len(), 3);
-        assert_text_content(&results[0].content, "valid 1");
+        assert_eq!(results[0].content, "valid 1");
         assert!(results[1].metadata.error.is_some());
-        assert_text_content(&results[2].content, "valid 2");
+        assert_eq!(results[2].content, "valid 2");
     }
 
     #[tokio::test]
@@ -882,8 +762,7 @@ mod tests {
 
         assert!(result.is_ok());
         let result = result.unwrap();
-        let trimmed_len = result.content.trim_end_matches('\n').len();
-        assert_eq!(trimmed_len, 10_000_000);
+        assert_eq!(result.content.len(), 10_000_000);
     }
 
     #[tokio::test]
@@ -908,7 +787,7 @@ mod tests {
         assert_eq!(results.len(), 100);
 
         for (i, result) in results.iter().enumerate() {
-            assert_text_content(&result.content, &format!("content {}", i));
+            assert_eq!(result.content, format!("content {}", i));
         }
     }
 

data/vendor/kreuzberg/src/core/io.rs

@@ -4,6 +4,7 @@
 
 use crate::{KreuzbergError, Result};
 use std::path::Path;
+use tokio::fs;
 
 /// Read a file asynchronously.
 ///
@@ -18,9 +19,8 @@ use std::path::Path;
 /// # Errors
 ///
 /// Returns `KreuzbergError::Io` for I/O errors (these always bubble up).
-#[cfg(feature = "tokio-runtime")]
 pub async fn read_file_async(path: impl AsRef<Path>) -> Result<Vec<u8>> {
-    tokio::fs::read(path.as_ref()).await.map_err(KreuzbergError::Io)
+    fs::read(path.as_ref()).await.map_err(KreuzbergError::Io)
 }
 
 /// Read a file synchronously.
@@ -181,7 +181,6 @@ mod tests {
     use std::io::Write;
     use tempfile::tempdir;
 
-    #[cfg(feature = "tokio-runtime")]
     #[tokio::test]
     async fn test_read_file_async() {
         let dir = tempdir().unwrap();
@@ -312,7 +311,6 @@ mod tests {
         assert!(result.is_err());
     }
 
-    #[cfg(feature = "tokio-runtime")]
     #[tokio::test]
     async fn test_read_file_async_io_error() {
         let result = read_file_async("/nonexistent/file.txt").await;

data/vendor/kreuzberg/src/core/mime.rs

@@ -152,7 +152,6 @@ static SUPPORTED_MIME_TYPES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
     set.insert("application/x-ipynb+json");
     set.insert("application/x-jats+xml");
     set.insert("application/x-latex");
-    set.insert("application/xml+opml");
     set.insert("application/x-opml+xml");
     set.insert("application/x-research-info-systems");
     set.insert("application/x-typst");
@@ -165,7 +164,6 @@ static SUPPORTED_MIME_TYPES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
     set.insert("text/x-markdown-extra");
     set.insert("text/x-mdoc");
     set.insert("text/x-multimarkdown");
-    set.insert("text/x-opml");
     set.insert("text/x-org");
     set.insert("text/x-pod");
     set.insert("text/x-rst");
@@ -329,35 +327,43 @@ pub fn detect_or_validate(path: Option<&Path>, mime_type: Option<&str>) -> Resul
 ///
 /// Returns `KreuzbergError::UnsupportedFormat` if MIME type cannot be determined.
 pub fn detect_mime_type_from_bytes(content: &[u8]) -> Result<String> {
+    // Use infer crate for magic byte detection
     if let Some(kind) = infer::get(content) {
         let mime_type = kind.mime_type();
 
+        // Validate that it's a supported type
         if SUPPORTED_MIME_TYPES.contains(mime_type) || mime_type.starts_with("image/") {
             return Ok(mime_type.to_string());
         }
     }
 
+    // Try to detect text-based formats
     if let Ok(text) = std::str::from_utf8(content) {
         let trimmed = text.trim_start();
 
+        // Detect JSON
         if (trimmed.starts_with('{') || trimmed.starts_with('['))
             && serde_json::from_str::<serde_json::Value>(text).is_ok()
         {
             return Ok(JSON_MIME_TYPE.to_string());
         }
 
+        // Detect XML
         if trimmed.starts_with("<?xml") || trimmed.starts_with('<') {
             return Ok(XML_MIME_TYPE.to_string());
         }
 
+        // Detect HTML
         if trimmed.starts_with("<!DOCTYPE html") || trimmed.starts_with("<html") {
             return Ok(HTML_MIME_TYPE.to_string());
         }
 
+        // Detect PDF header
         if trimmed.starts_with("%PDF") {
             return Ok(PDF_MIME_TYPE.to_string());
         }
 
+        // Default to plain text for valid UTF-8
         return Ok(PLAIN_TEXT_MIME_TYPE.to_string());
     }
 
@@ -392,21 +398,25 @@ pub fn detect_mime_type_from_bytes(content: &[u8]) -> Result<String> {
 pub fn get_extensions_for_mime(mime_type: &str) -> Result<Vec<String>> {
     let mut extensions = Vec::new();
 
+    // Search through EXT_TO_MIME for matching MIME types
     for (ext, mime) in EXT_TO_MIME.iter() {
         if *mime == mime_type {
             extensions.push(ext.to_string());
         }
     }
 
+    // If we found extensions, return them
     if !extensions.is_empty() {
         return Ok(extensions);
     }
 
+    // Try using mime_guess crate for reverse lookup
     let guessed = mime_guess::get_mime_extensions_str(mime_type);
     if let Some(exts) = guessed {
         return Ok(exts.iter().map(|s| s.to_string()).collect());
     }
 
+    // No extensions found
     Err(KreuzbergError::UnsupportedFormat(format!(
         "No known extensions for MIME type: {}",
         mime_type

data/vendor/kreuzberg/src/core/mod.rs

@@ -28,7 +28,6 @@
 //! # }
 //! ```
 
-#[cfg(feature = "tokio-runtime")]
 pub(crate) mod batch_mode;
 pub mod config;
 pub mod extractor;
@@ -40,6 +39,4 @@ pub use config::{
     ChunkingConfig, ExtractionConfig, ImageExtractionConfig, LanguageDetectionConfig, OcrConfig, PdfConfig,
     TokenReductionConfig,
 };
-#[cfg(feature = "tokio-runtime")]
-pub use extractor::{batch_extract_bytes, batch_extract_file};
-pub use extractor::{extract_bytes, extract_file};
+pub use extractor::{batch_extract_bytes, batch_extract_file, extract_bytes, extract_file};