kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14

data/vendor/kreuzberg/tests/format_integration.rs

@@ -1,164 +1,164 @@
-//! Binding-specific format integration tests.
-//!
-//! Positive-path scenarios are now covered by the shared fixture-based E2E
-//! suites. The tests here focus on behaviour that is specific to the Rust
-//! asynchronous APIs or to graceful handling when optional system
-//! dependencies are missing.
-
-#![cfg(any(feature = "pdf", feature = "office", feature = "ocr"))]
-
-mod helpers;
-
-use helpers::{assert_mime_type, get_test_file_path, test_documents_available};
-
-#[cfg(any(feature = "office", feature = "ocr"))]
-use helpers::assert_non_empty_content;
-use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::core::extractor::extract_file;
-
-#[cfg(feature = "ocr")]
-use kreuzberg::core::config::OcrConfig;
-
-#[cfg(feature = "pdf")]
-#[tokio::test]
-async fn test_pdf_password_protected_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("pdfs/copy_protected.pdf");
-    if !path.exists() {
-        tracing::debug!("Skipping test: protected PDF not available");
-        return;
-    }
-
-    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
-
-    match result {
-        Err(err) => {
-            tracing::debug!("Password protection detected (expected): {}", err);
-        }
-        Ok(res) => {
-            tracing::debug!("Protected PDF extracted; some files allow fallback");
-            assert_mime_type(&res, "application/pdf");
-            assert!(res.chunks.is_none(), "Chunks should be None without chunking config");
-            assert!(res.detected_languages.is_none(), "Language detection not enabled");
-        }
-    }
-}
-
-#[cfg(feature = "office")]
-#[tokio::test]
-async fn test_legacy_doc_extraction_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("legacy_office/simple.doc");
-    if !path.exists() {
-        tracing::debug!("Skipping test: legacy .doc file not available");
-        return;
-    }
-
-    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
-
-    match result {
-        Ok(extracted) => {
-            assert_mime_type(&extracted, "application/msword");
-            assert_non_empty_content(&extracted);
-            assert!(
-                extracted.chunks.is_none(),
-                "Chunks should be None without chunking config"
-            );
-            assert!(extracted.detected_languages.is_none(), "Language detection not enabled");
-        }
-        Err(err) => {
-            tracing::debug!(
-                "Legacy Office extraction failed (LibreOffice may not be installed): {}",
-                err
-            );
-        }
-    }
-}
-
-#[cfg(feature = "ocr")]
-#[tokio::test]
-async fn test_ocr_simple_english_image_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("images/test_hello_world.png");
-    if !path.exists() {
-        tracing::debug!("Skipping test: OCR sample image not available");
-        return;
-    }
-
-    let config = ExtractionConfig {
-        ocr: Some(OcrConfig {
-            backend: "tesseract".to_string(),
-            language: "eng".to_string(),
-            tesseract_config: None,
-        }),
-        force_ocr: true,
-        ..Default::default()
-    };
-
-    let result = extract_file(&path, None, &config).await;
-
-    match result {
-        Ok(res) => {
-            assert_mime_type(&res, "image/png");
-            assert_non_empty_content(&res);
-            let content_lower = res.content.to_lowercase();
-            assert!(
-                content_lower.contains("hello") || content_lower.contains("world"),
-                "OCR output {:?} should contain HELLO or WORLD",
-                res.content
-            );
-        }
-        Err(err) => {
-            tracing::debug!("OCR test failed (Tesseract may not be installed): {}", err);
-        }
-    }
-}
-
-#[cfg(feature = "ocr")]
-#[tokio::test]
-async fn test_ocr_image_without_text_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("images/flower_no_text.jpg");
-    if !path.exists() {
-        tracing::debug!("Skipping test: OCR flower image not available");
-        return;
-    }
-
-    let config = ExtractionConfig {
-        ocr: Some(OcrConfig {
-            backend: "tesseract".to_string(),
-            language: "eng".to_string(),
-            tesseract_config: None,
-        }),
-        force_ocr: true,
-        ..Default::default()
-    };
-
-    let result = extract_file(&path, None, &config).await;
-
-    match result {
-        Ok(res) => {
-            assert_mime_type(&res, "image/jpeg");
-            assert!(
-                res.content.len() < 200,
-                "Expected minimal OCR output, got {} bytes",
-                res.content.len()
-            );
-        }
-        Err(err) => {
-            tracing::debug!("OCR fallback test failed (Tesseract may not be installed): {}", err);
-        }
-    }
-}
+//! Binding-specific format integration tests.
+//!
+//! Positive-path scenarios are now covered by the shared fixture-based E2E
+//! suites. The tests here focus on behaviour that is specific to the Rust
+//! asynchronous APIs or to graceful handling when optional system
+//! dependencies are missing.
+
+#![cfg(any(feature = "pdf", feature = "office", feature = "ocr"))]
+
+mod helpers;
+
+use helpers::{assert_mime_type, get_test_file_path, test_documents_available};
+
+#[cfg(any(feature = "office", feature = "ocr"))]
+use helpers::assert_non_empty_content;
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::extractor::extract_file;
+
+#[cfg(feature = "ocr")]
+use kreuzberg::core::config::OcrConfig;
+
+#[cfg(feature = "pdf")]
+#[tokio::test]
+async fn test_pdf_password_protected_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("pdfs/copy_protected.pdf");
+    if !path.exists() {
+        tracing::debug!("Skipping test: protected PDF not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Err(err) => {
+            tracing::debug!("Password protection detected (expected): {}", err);
+        }
+        Ok(res) => {
+            tracing::debug!("Protected PDF extracted; some files allow fallback");
+            assert_mime_type(&res, "application/pdf");
+            assert!(res.chunks.is_none(), "Chunks should be None without chunking config");
+            assert!(res.detected_languages.is_none(), "Language detection not enabled");
+        }
+    }
+}
+
+#[cfg(feature = "office")]
+#[tokio::test]
+async fn test_legacy_doc_extraction_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("legacy_office/simple.doc");
+    if !path.exists() {
+        tracing::debug!("Skipping test: legacy .doc file not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Ok(extracted) => {
+            assert_mime_type(&extracted, "application/msword");
+            assert_non_empty_content(&extracted);
+            assert!(
+                extracted.chunks.is_none(),
+                "Chunks should be None without chunking config"
+            );
+            assert!(extracted.detected_languages.is_none(), "Language detection not enabled");
+        }
+        Err(err) => {
+            tracing::debug!(
+                "Legacy Office extraction failed (LibreOffice may not be installed): {}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_simple_english_image_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/test_hello_world.png");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR sample image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/png");
+            assert_non_empty_content(&res);
+            let content_lower = res.content.to_lowercase();
+            assert!(
+                content_lower.contains("hello") || content_lower.contains("world"),
+                "OCR output {:?} should contain HELLO or WORLD",
+                res.content
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_image_without_text_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/flower_no_text.jpg");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR flower image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/jpeg");
+            assert!(
+                res.content.len() < 200,
+                "Expected minimal OCR output, got {} bytes",
+                res.content.len()
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR fallback test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}

data/vendor/kreuzberg/tests/helpers/mod.rs

@@ -1,142 +1,142 @@
-//! Shared test helpers for integration tests.
-//!
-//! This module provides common utilities for loading test files,
-//! making assertions, and setting up test environments.
-
-#![allow(dead_code)]
-
-use kreuzberg::types::ExtractionResult;
-use std::path::PathBuf;
-
-/// Get the test_documents directory path.
-///
-/// This assumes the test is running from the workspace root.
-pub fn get_test_documents_dir() -> PathBuf {
-    let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap()
-        .to_path_buf();
-
-    workspace_root.join("test_documents")
-}
-
-/// Get the full path to a test file.
-///
-/// # Arguments
-///
-/// * `relative_path` - Path relative to test_documents/
-pub fn get_test_file_path(relative_path: &str) -> PathBuf {
-    get_test_documents_dir().join(relative_path)
-}
-
-/// Assert that extraction result contains non-empty content.
-///
-/// This is a common assertion for most extraction tests - we want
-/// to verify that *something* was extracted, even if we don't know
-/// the exact content.
-pub fn assert_non_empty_content(result: &ExtractionResult) {
-    assert!(
-        !result.content.trim().is_empty(),
-        "Extraction result should have non-empty content, got: '{}'",
-        result.content
-    );
-}
-
-/// Assert that extraction result has expected MIME type.
-pub fn assert_mime_type(result: &ExtractionResult, expected: &str) {
-    assert_eq!(
-        result.mime_type, expected,
-        "Expected MIME type '{}', got '{}'",
-        expected, result.mime_type
-    );
-}
-
-/// Skip test if file doesn't exist (for optional test files).
-///
-/// Returns true if test should be skipped.
-pub fn skip_if_missing(relative_path: &str) -> bool {
-    let path = get_test_file_path(relative_path);
-    if !path.exists() {
-        tracing::debug!("Skipping test: file not found at {}", path.display());
-        return true;
-    }
-    false
-}
-
-/// Check if test documents directory exists and has files.
-///
-/// This is useful for CI environments where test_documents might
-/// be a git submodule that hasn't been initialized.
-pub fn test_documents_available() -> bool {
-    let dir = get_test_documents_dir();
-    dir.exists() && dir.read_dir().map(|mut d| d.next().is_some()).unwrap_or(false)
-}
-
-/// Assert that content length is above a minimum threshold.
-///
-/// This is useful for smoke testing - ensuring substantial content
-/// was extracted without needing to verify exact text.
-pub fn assert_min_content_length(result: &ExtractionResult, min_length: usize) {
-    assert!(
-        result.content.len() >= min_length,
-        "Expected content length >= {}, got {}. Content preview: '{}'",
-        min_length,
-        result.content.len(),
-        result.content.chars().take(200).collect::<String>()
-    );
-}
-
-/// Assert that content contains at least one of the given substrings.
-pub fn assert_content_contains_any(result: &ExtractionResult, substrings: &[&str]) {
-    let found = substrings.iter().any(|s| result.content.contains(s));
-    assert!(
-        found,
-        "Expected content to contain at least one of {:?}, but found none",
-        substrings
-    );
-}
-
-/// Assert that extraction result has at least one table.
-pub fn assert_has_tables(result: &ExtractionResult) {
-    assert!(
-        !result.tables.is_empty(),
-        "Expected result to have tables, but found none"
-    );
-}
-
-/// Create a test configuration with OCR enabled.
-pub fn test_config_with_ocr() -> kreuzberg::core::config::ExtractionConfig {
-    use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
-
-    ExtractionConfig {
-        ocr: Some(OcrConfig {
-            backend: "tesseract".to_string(),
-            language: "eng".to_string(),
-            tesseract_config: None,
-        }),
-        force_ocr: false,
-        ..Default::default()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_get_test_documents_dir() {
-        let dir = get_test_documents_dir();
-        assert!(dir.to_string_lossy().ends_with("test_documents"));
-    }
-
-    #[test]
-    fn test_test_documents_available() {
-        let available = test_documents_available();
-        if !available {
-            tracing::debug!("Warning: test_documents directory not available");
-            tracing::debug!("This is expected in CI without git submodules initialized");
-        }
-    }
-}
+//! Shared test helpers for integration tests.
+//!
+//! This module provides common utilities for loading test files,
+//! making assertions, and setting up test environments.
+
+#![allow(dead_code)]
+
+use kreuzberg::types::ExtractionResult;
+use std::path::PathBuf;
+
+/// Get the test_documents directory path.
+///
+/// This assumes the test is running from the workspace root.
+pub fn get_test_documents_dir() -> PathBuf {
+    let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .to_path_buf();
+
+    workspace_root.join("test_documents")
+}
+
+/// Get the full path to a test file.
+///
+/// # Arguments
+///
+/// * `relative_path` - Path relative to test_documents/
+pub fn get_test_file_path(relative_path: &str) -> PathBuf {
+    get_test_documents_dir().join(relative_path)
+}
+
+/// Assert that extraction result contains non-empty content.
+///
+/// This is a common assertion for most extraction tests - we want
+/// to verify that *something* was extracted, even if we don't know
+/// the exact content.
+pub fn assert_non_empty_content(result: &ExtractionResult) {
+    assert!(
+        !result.content.trim().is_empty(),
+        "Extraction result should have non-empty content, got: '{}'",
+        result.content
+    );
+}
+
+/// Assert that extraction result has expected MIME type.
+pub fn assert_mime_type(result: &ExtractionResult, expected: &str) {
+    assert_eq!(
+        result.mime_type, expected,
+        "Expected MIME type '{}', got '{}'",
+        expected, result.mime_type
+    );
+}
+
+/// Skip test if file doesn't exist (for optional test files).
+///
+/// Returns true if test should be skipped.
+pub fn skip_if_missing(relative_path: &str) -> bool {
+    let path = get_test_file_path(relative_path);
+    if !path.exists() {
+        tracing::debug!("Skipping test: file not found at {}", path.display());
+        return true;
+    }
+    false
+}
+
+/// Check if test documents directory exists and has files.
+///
+/// This is useful for CI environments where test_documents might
+/// be a git submodule that hasn't been initialized.
+pub fn test_documents_available() -> bool {
+    let dir = get_test_documents_dir();
+    dir.exists() && dir.read_dir().map(|mut d| d.next().is_some()).unwrap_or(false)
+}
+
+/// Assert that content length is above a minimum threshold.
+///
+/// This is useful for smoke testing - ensuring substantial content
+/// was extracted without needing to verify exact text.
+pub fn assert_min_content_length(result: &ExtractionResult, min_length: usize) {
+    assert!(
+        result.content.len() >= min_length,
+        "Expected content length >= {}, got {}. Content preview: '{}'",
+        min_length,
+        result.content.len(),
+        result.content.chars().take(200).collect::<String>()
+    );
+}
+
+/// Assert that content contains at least one of the given substrings.
+pub fn assert_content_contains_any(result: &ExtractionResult, substrings: &[&str]) {
+    let found = substrings.iter().any(|s| result.content.contains(s));
+    assert!(
+        found,
+        "Expected content to contain at least one of {:?}, but found none",
+        substrings
+    );
+}
+
+/// Assert that extraction result has at least one table.
+pub fn assert_has_tables(result: &ExtractionResult) {
+    assert!(
+        !result.tables.is_empty(),
+        "Expected result to have tables, but found none"
+    );
+}
+
+/// Create a test configuration with OCR enabled.
+pub fn test_config_with_ocr() -> kreuzberg::core::config::ExtractionConfig {
+    use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
+
+    ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: false,
+        ..Default::default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_test_documents_dir() {
+        let dir = get_test_documents_dir();
+        assert!(dir.to_string_lossy().ends_with("test_documents"));
+    }
+
+    #[test]
+    fn test_test_documents_available() {
+        let available = test_documents_available();
+        if !available {
+            tracing::debug!("Warning: test_documents directory not available");
+            tracing::debug!("This is expected in CI without git submodules initialized");
+        }
+    }
+}