kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/tests/csv_integration.rs

@@ -1,6 +1,6 @@
 //! CSV and spreadsheet integration tests.
 //!
-//! Tests for CSV and TSV extraction.
+//! Tests for CSV and TSV extraction via Pandoc.
 //! Validates data extraction, custom delimiters, quoted fields, and edge cases.
 
 use kreuzberg::core::config::ExtractionConfig;
@@ -15,13 +15,14 @@ async fn test_csv_basic_extraction() {
 
     let csv_content = b"Name,Age,City\nAlice,30,NYC\nBob,25,LA";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert_eq!(extraction.mime_type, "text/csv");
     assert!(
@@ -54,13 +55,14 @@ async fn test_csv_with_headers() {
 
     let csv_content = b"Product,Price,Quantity\nApple,1.50,100\nBanana,0.75,200\nOrange,2.00,150";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -103,13 +105,14 @@ async fn test_csv_custom_delimiter() {
 
     let csv_content = b"Name;Age;City\nAlice;30;NYC\nBob;25;LA";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -135,13 +138,14 @@ async fn test_tsv_file() {
 
     let tsv_content = b"Name\tAge\tCity\nAlice\t30\tNYC\nBob\t25\tLA";
 
-    let extraction = match extract_bytes(tsv_content, "text/tab-separated-values", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: TSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(tsv_content, "text/tab-separated-values", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert_eq!(extraction.mime_type, "text/tab-separated-values");
     assert!(
@@ -171,13 +175,14 @@ async fn test_csv_quoted_fields() {
     let csv_content =
         b"Name,Description,Price\n\"Smith, John\",\"Product A, premium\",100\n\"Doe, Jane\",\"Product B, standard\",50";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -207,13 +212,14 @@ async fn test_csv_special_characters() {
 
     let csv_content = "Name,City,Emoji\nAlice,Tokyo 東京,🎉\nBob,París,✅\nCarlos,Москва,🌍".as_bytes();
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -245,13 +251,14 @@ async fn test_csv_large_file() {
         csv_content.push_str(&format!("{},Item{},{}.00\n", i, i, i * 10));
     }
 
-    let extraction = match extract_bytes(csv_content.as_bytes(), "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content.as_bytes(), "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -315,13 +322,14 @@ async fn test_csv_headers_only() {
 
     let csv_content = b"Name,Age,City";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -346,13 +354,14 @@ async fn test_csv_blank_lines() {
 
     let csv_content = b"Name,Age\nAlice,30\n\nBob,25\n\nCarlos,35";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),
@@ -374,13 +383,14 @@ async fn test_csv_numeric_data() {
 
     let csv_content = b"ID,Price,Quantity,Discount\n1,19.99,100,0.15\n2,29.99,50,0.20\n3,9.99,200,0.10";
 
-    let extraction = match extract_bytes(csv_content, "text/csv", &config).await {
-        Ok(result) => result,
-        Err(_) => {
-            println!("Skipping test: CSV extraction not available");
-            return;
-        }
-    };
+    let result = extract_bytes(csv_content, "text/csv", &config).await;
+
+    if result.is_err() {
+        println!("Skipping test: Pandoc may not be installed");
+        return;
+    }
+
+    let extraction = result.unwrap();
 
     assert!(
         extraction.chunks.is_none(),

data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs

@@ -2,10 +2,15 @@
 
 #![cfg(feature = "office")]
 
-use kreuzberg::{ExtractionConfig, extract_file};
+use kreuzberg::extraction::pandoc::extract_file;
 
 #[tokio::test]
 async fn test_docx_full_metadata_extraction() {
+    if kreuzberg::extraction::pandoc::validate_pandoc_version().await.is_err() {
+        println!("Skipping test: Pandoc not available");
+        return;
+    }
+
     let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
         .parent()
         .unwrap()
@@ -18,7 +23,7 @@ async fn test_docx_full_metadata_extraction() {
         return;
     }
 
-    let result = extract_file(&test_file, None, &ExtractionConfig::default())
+    let result = extract_file(&test_file, "docx")
         .await
         .expect("Should extract DOCX successfully");
 
@@ -29,66 +34,63 @@ async fn test_docx_full_metadata_extraction() {
     );
 
     assert_eq!(
-        result.metadata.additional.get("created_by").and_then(|v| v.as_str()),
+        result.metadata.get("created_by").and_then(|v| v.as_str()),
         Some("Christoph Auer"),
         "Should have correct creator"
     );
     assert_eq!(
-        result.metadata.additional.get("modified_by").and_then(|v| v.as_str()),
+        result.metadata.get("modified_by").and_then(|v| v.as_str()),
         Some("Maxim Lysak"),
         "Should have correct last modified by"
     );
     assert_eq!(
-        result.metadata.additional.get("created_at").and_then(|v| v.as_str()),
+        result.metadata.get("created_at").and_then(|v| v.as_str()),
         Some("2024-10-09T12:43:00Z"),
         "Should have correct creation date"
     );
     assert_eq!(
-        result.metadata.additional.get("revision").and_then(|v| v.as_str()),
+        result.metadata.get("revision").and_then(|v| v.as_str()),
         Some("7"),
         "Should have revision number"
     );
 
     assert_eq!(
-        result.metadata.additional.get("page_count").and_then(|v| v.as_i64()),
+        result.metadata.get("page_count").and_then(|v| v.as_i64()),
         Some(2),
         "Should have 2 pages"
     );
     assert_eq!(
-        result.metadata.additional.get("word_count").and_then(|v| v.as_i64()),
+        result.metadata.get("word_count").and_then(|v| v.as_i64()),
         Some(108),
         "Should have 108 words"
     );
     assert_eq!(
-        result
-            .metadata
-            .additional
-            .get("character_count")
-            .and_then(|v| v.as_i64()),
+        result.metadata.get("character_count").and_then(|v| v.as_i64()),
         Some(620),
         "Should have 620 characters"
     );
     assert_eq!(
-        result.metadata.additional.get("line_count").and_then(|v| v.as_i64()),
+        result.metadata.get("line_count").and_then(|v| v.as_i64()),
         Some(5),
         "Should have 5 lines"
     );
     assert_eq!(
-        result
-            .metadata
-            .additional
-            .get("paragraph_count")
-            .and_then(|v| v.as_i64()),
+        result.metadata.get("paragraph_count").and_then(|v| v.as_i64()),
         Some(1),
         "Should have 1 paragraph"
     );
 
     println!("✅ DOCX metadata extraction test passed!");
-    println!("   Found {} metadata fields", result.metadata.additional.len());
+    println!("   Found {} metadata fields", result.metadata.len());
 }
 
 #[tokio::test]
 async fn test_docx_minimal_metadata_extraction() {
+    if kreuzberg::extraction::pandoc::validate_pandoc_version().await.is_err() {
+        println!("Skipping test: Pandoc not available");
+        return;
+    }
+
     let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
         .parent()
         .unwrap()
@@ -101,19 +103,19 @@ async fn test_docx_minimal_metadata_extraction() {
         return;
     }
 
-    let result = extract_file(&test_file, None, &ExtractionConfig::default())
+    let result = extract_file(&test_file, "docx")
         .await
         .expect("Should extract DOCX successfully");
 
     assert!(!result.content.is_empty(), "Content should not be empty");
 
     assert_eq!(
-        result.metadata.additional.get("page_count").and_then(|v| v.as_i64()),
+        result.metadata.get("page_count").and_then(|v| v.as_i64()),
         Some(1),
         "Should have 1 page"
     );
     assert_eq!(
-        result.metadata.additional.get("word_count").and_then(|v| v.as_i64()),
+        result.metadata.get("word_count").and_then(|v| v.as_i64()),
         Some(520),
         "Should have 520 words"
     );