kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs

@@ -0,0 +1,228 @@
+#![cfg(feature = "office")]
+
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::plugins::DocumentExtractor;
+use std::path::PathBuf;
+
+/// Helper to get absolute path to test documents
+fn test_file_path(filename: &str) -> PathBuf {
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+    PathBuf::from(manifest_dir)
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join("test_documents")
+        .join("fictionbook")
+        .join(filename)
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_metadata_title() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("meta.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Book title"),
+        "Book title should be extracted from FB2 content"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_metadata_genre() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("meta.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.metadata.subject.is_none());
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_content_sections() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("titles.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Simple title"),
+        "Section titles should be extracted"
+    );
+    assert!(
+        result.content.contains("Emphasized"),
+        "Section with emphasis should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_section_hierarchy() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Top-level title"),
+        "Top-level section should be extracted"
+    );
+    assert!(result.content.contains("Section"), "Nested section should be extracted");
+    assert!(
+        result.content.contains("Subsection"),
+        "Nested subsection should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_inline_markup() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("emphasis.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    let content = result.content.to_lowercase();
+    assert!(content.contains("plain"), "Plain text should be extracted");
+    assert!(content.contains("strong"), "Strong emphasis should be extracted");
+    assert!(content.contains("emphasis"), "Emphasis should be extracted");
+    assert!(content.contains("strikethrough"), "Strikethrough should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_emphasis() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("emphasized"),
+        "Emphasized text should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_strong() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("strong"), "Strong text should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_code() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("verbatim"), "Code content should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_blockquote() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("Blockquote"), "Blockquote should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_tables() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("tables.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        !result.content.is_empty(),
+        "Content should be extracted from file with tables"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_markdown_formatting_preservation() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("emphasis.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("**strong**"),
+        "Strong text should be formatted as **bold** in markdown"
+    );
+    assert!(
+        result.content.contains("*emphasis*"),
+        "Emphasis text should be formatted as *italic* in markdown"
+    );
+    assert!(
+        result.content.contains("~~deleted~~"),
+        "Strikethrough text should be formatted as ~~strikethrough~~ in markdown"
+    );
+    assert!(
+        result.content.contains("`code`"),
+        "Code text should be wrapped in backticks in markdown"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_formatting_in_body_paragraphs() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("*emphasized*"),
+        "Emphasis formatting should be preserved in body content"
+    );
+    assert!(
+        result.content.contains("**strong**"),
+        "Strong formatting should be preserved in body content"
+    );
+    assert!(
+        result.content.contains("`verbatim`"),
+        "Code formatting should be preserved in body content"
+    );
+}

data/vendor/kreuzberg/tests/format_integration.rs

@@ -0,0 +1,159 @@
+//! Binding-specific format integration tests.
+//!
+//! Positive-path scenarios are now covered by the shared fixture-based E2E
+//! suites. The tests here focus on behaviour that is specific to the Rust
+//! asynchronous APIs or to graceful handling when optional system
+//! dependencies are missing.
+
+mod helpers;
+
+use helpers::{assert_mime_type, assert_non_empty_content, get_test_file_path, test_documents_available};
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::extractor::extract_file;
+
+#[cfg(feature = "ocr")]
+use kreuzberg::core::config::OcrConfig;
+
+#[cfg(feature = "pdf")]
+#[tokio::test]
+async fn test_pdf_password_protected_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("pdfs/copy_protected.pdf");
+    if !path.exists() {
+        tracing::debug!("Skipping test: protected PDF not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Err(err) => {
+            tracing::debug!("Password protection detected (expected): {}", err);
+        }
+        Ok(res) => {
+            tracing::debug!("Protected PDF extracted; some files allow fallback");
+            assert_mime_type(&res, "application/pdf");
+            assert!(res.chunks.is_none(), "Chunks should be None without chunking config");
+            assert!(res.detected_languages.is_none(), "Language detection not enabled");
+        }
+    }
+}
+
+#[cfg(feature = "office")]
+#[tokio::test]
+async fn test_legacy_doc_extraction_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("legacy_office/simple.doc");
+    if !path.exists() {
+        tracing::debug!("Skipping test: legacy .doc file not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Ok(extracted) => {
+            assert_mime_type(&extracted, "application/msword");
+            assert_non_empty_content(&extracted);
+            assert!(
+                extracted.chunks.is_none(),
+                "Chunks should be None without chunking config"
+            );
+            assert!(extracted.detected_languages.is_none(), "Language detection not enabled");
+        }
+        Err(err) => {
+            tracing::debug!(
+                "Legacy Office extraction failed (LibreOffice may not be installed): {}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_simple_english_image_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/test_hello_world.png");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR sample image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/png");
+            assert_non_empty_content(&res);
+            let content_lower = res.content.to_lowercase();
+            assert!(
+                content_lower.contains("hello") || content_lower.contains("world"),
+                "OCR output {:?} should contain HELLO or WORLD",
+                res.content
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_image_without_text_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/flower_no_text.jpg");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR flower image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/jpeg");
+            assert!(
+                res.content.len() < 200,
+                "Expected minimal OCR output, got {} bytes",
+                res.content.len()
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR fallback test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}

data/vendor/kreuzberg/tests/helpers/mod.rs

@@ -0,0 +1,142 @@
+//! Shared test helpers for integration tests.
+//!
+//! This module provides common utilities for loading test files,
+//! making assertions, and setting up test environments.
+
+#![allow(dead_code)]
+
+use kreuzberg::types::ExtractionResult;
+use std::path::PathBuf;
+
+/// Get the test_documents directory path.
+///
+/// This assumes the test is running from the workspace root.
+pub fn get_test_documents_dir() -> PathBuf {
+    let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .to_path_buf();
+
+    workspace_root.join("test_documents")
+}
+
+/// Get the full path to a test file.
+///
+/// # Arguments
+///
+/// * `relative_path` - Path relative to test_documents/
+pub fn get_test_file_path(relative_path: &str) -> PathBuf {
+    get_test_documents_dir().join(relative_path)
+}
+
+/// Assert that extraction result contains non-empty content.
+///
+/// This is a common assertion for most extraction tests - we want
+/// to verify that *something* was extracted, even if we don't know
+/// the exact content.
+pub fn assert_non_empty_content(result: &ExtractionResult) {
+    assert!(
+        !result.content.trim().is_empty(),
+        "Extraction result should have non-empty content, got: '{}'",
+        result.content
+    );
+}
+
+/// Assert that extraction result has expected MIME type.
+pub fn assert_mime_type(result: &ExtractionResult, expected: &str) {
+    assert_eq!(
+        result.mime_type, expected,
+        "Expected MIME type '{}', got '{}'",
+        expected, result.mime_type
+    );
+}
+
+/// Skip test if file doesn't exist (for optional test files).
+///
+/// Returns true if test should be skipped.
+pub fn skip_if_missing(relative_path: &str) -> bool {
+    let path = get_test_file_path(relative_path);
+    if !path.exists() {
+        tracing::debug!("Skipping test: file not found at {}", path.display());
+        return true;
+    }
+    false
+}
+
+/// Check if test documents directory exists and has files.
+///
+/// This is useful for CI environments where test_documents might
+/// be a git submodule that hasn't been initialized.
+pub fn test_documents_available() -> bool {
+    let dir = get_test_documents_dir();
+    dir.exists() && dir.read_dir().map(|mut d| d.next().is_some()).unwrap_or(false)
+}
+
+/// Assert that content length is above a minimum threshold.
+///
+/// This is useful for smoke testing - ensuring substantial content
+/// was extracted without needing to verify exact text.
+pub fn assert_min_content_length(result: &ExtractionResult, min_length: usize) {
+    assert!(
+        result.content.len() >= min_length,
+        "Expected content length >= {}, got {}. Content preview: '{}'",
+        min_length,
+        result.content.len(),
+        result.content.chars().take(200).collect::<String>()
+    );
+}
+
+/// Assert that content contains at least one of the given substrings.
+pub fn assert_content_contains_any(result: &ExtractionResult, substrings: &[&str]) {
+    let found = substrings.iter().any(|s| result.content.contains(s));
+    assert!(
+        found,
+        "Expected content to contain at least one of {:?}, but found none",
+        substrings
+    );
+}
+
+/// Assert that extraction result has at least one table.
+pub fn assert_has_tables(result: &ExtractionResult) {
+    assert!(
+        !result.tables.is_empty(),
+        "Expected result to have tables, but found none"
+    );
+}
+
+/// Create a test configuration with OCR enabled.
+pub fn test_config_with_ocr() -> kreuzberg::core::config::ExtractionConfig {
+    use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
+
+    ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: false,
+        ..Default::default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_test_documents_dir() {
+        let dir = get_test_documents_dir();
+        assert!(dir.to_string_lossy().ends_with("test_documents"));
+    }
+
+    #[test]
+    fn test_test_documents_available() {
+        let available = test_documents_available();
+        if !available {
+            tracing::debug!("Warning: test_documents directory not available");
+            tracing::debug!("This is expected in CI without git submodules initialized");
+        }
+    }
+}