kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs

@@ -0,0 +1,228 @@
+#![cfg(feature = "office")]
+
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::plugins::DocumentExtractor;
+use std::path::PathBuf;
+
+/// Helper to get absolute path to test documents
+fn test_file_path(filename: &str) -> PathBuf {
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+    PathBuf::from(manifest_dir)
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join("test_documents")
+        .join("fictionbook")
+        .join(filename)
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_metadata_title() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("meta.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Book title"),
+        "Book title should be extracted from FB2 content"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_metadata_genre() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("meta.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.metadata.subject.is_none());
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_content_sections() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("titles.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Simple title"),
+        "Section titles should be extracted"
+    );
+    assert!(
+        result.content.contains("Emphasized"),
+        "Section with emphasis should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_section_hierarchy() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("Top-level title"),
+        "Top-level section should be extracted"
+    );
+    assert!(result.content.contains("Section"), "Nested section should be extracted");
+    assert!(
+        result.content.contains("Subsection"),
+        "Nested subsection should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_inline_markup() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("emphasis.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    let content = result.content.to_lowercase();
+    assert!(content.contains("plain"), "Plain text should be extracted");
+    assert!(content.contains("strong"), "Strong emphasis should be extracted");
+    assert!(content.contains("emphasis"), "Emphasis should be extracted");
+    assert!(content.contains("strikethrough"), "Strikethrough should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_emphasis() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("emphasized"),
+        "Emphasized text should be extracted"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_strong() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("strong"), "Strong text should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_code() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("verbatim"), "Code content should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_blockquote() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(result.content.contains("Blockquote"), "Blockquote should be extracted");
+}
+
+#[tokio::test]
+async fn test_fictionbook_extract_tables() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("tables.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        !result.content.is_empty(),
+        "Content should be extracted from file with tables"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_markdown_formatting_preservation() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("emphasis.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("**strong**"),
+        "Strong text should be formatted as **bold** in markdown"
+    );
+    assert!(
+        result.content.contains("*emphasis*"),
+        "Emphasis text should be formatted as *italic* in markdown"
+    );
+    assert!(
+        result.content.contains("~~deleted~~"),
+        "Strikethrough text should be formatted as ~~strikethrough~~ in markdown"
+    );
+    assert!(
+        result.content.contains("`code`"),
+        "Code text should be wrapped in backticks in markdown"
+    );
+}
+
+#[tokio::test]
+async fn test_fictionbook_formatting_in_body_paragraphs() {
+    let extractor = kreuzberg::extractors::FictionBookExtractor::new();
+    let path = test_file_path("basic.fb2");
+
+    let result = extractor
+        .extract_file(&path, "application/x-fictionbook+xml", &ExtractionConfig::default())
+        .await
+        .expect("Failed to extract FB2 file");
+
+    assert!(
+        result.content.contains("*emphasized*"),
+        "Emphasis formatting should be preserved in body content"
+    );
+    assert!(
+        result.content.contains("**strong**"),
+        "Strong formatting should be preserved in body content"
+    );
+    assert!(
+        result.content.contains("`verbatim`"),
+        "Code formatting should be preserved in body content"
+    );
+}

data/vendor/kreuzberg/tests/format_integration.rs

@@ -1,159 +1,159 @@
-//! Binding-specific format integration tests.
-//!
-//! Positive-path scenarios are now covered by the shared fixture-based E2E
-//! suites. The tests here focus on behaviour that is specific to the Rust
-//! asynchronous APIs or to graceful handling when optional system
-//! dependencies are missing.
-
-mod helpers;
-
-use helpers::{assert_mime_type, assert_non_empty_content, get_test_file_path, test_documents_available};
-use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::core::extractor::extract_file;
-
-#[cfg(feature = "ocr")]
-use kreuzberg::core::config::OcrConfig;
-
-#[cfg(feature = "pdf")]
-#[tokio::test]
-async fn test_pdf_password_protected_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("pdfs/copy_protected.pdf");
-    if !path.exists() {
-        tracing::debug!("Skipping test: protected PDF not available");
-        return;
-    }
-
-    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
-
-    match result {
-        Err(err) => {
-            tracing::debug!("Password protection detected (expected): {}", err);
-        }
-        Ok(res) => {
-            tracing::debug!("Protected PDF extracted; some files allow fallback");
-            assert_mime_type(&res, "application/pdf");
-            assert!(res.chunks.is_none(), "Chunks should be None without chunking config");
-            assert!(res.detected_languages.is_none(), "Language detection not enabled");
-        }
-    }
-}
-
-#[cfg(feature = "office")]
-#[tokio::test]
-async fn test_legacy_doc_extraction_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("legacy_office/simple.doc");
-    if !path.exists() {
-        tracing::debug!("Skipping test: legacy .doc file not available");
-        return;
-    }
-
-    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
-
-    match result {
-        Ok(extracted) => {
-            assert_mime_type(&extracted, "application/msword");
-            assert_non_empty_content(&extracted);
-            assert!(
-                extracted.chunks.is_none(),
-                "Chunks should be None without chunking config"
-            );
-            assert!(extracted.detected_languages.is_none(), "Language detection not enabled");
-        }
-        Err(err) => {
-            tracing::debug!(
-                "Legacy Office extraction failed (LibreOffice may not be installed): {}",
-                err
-            );
-        }
-    }
-}
-
-#[cfg(feature = "ocr")]
-#[tokio::test]
-async fn test_ocr_simple_english_image_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("images/test_hello_world.png");
-    if !path.exists() {
-        tracing::debug!("Skipping test: OCR sample image not available");
-        return;
-    }
-
-    let config = ExtractionConfig {
-        ocr: Some(OcrConfig {
-            backend: "tesseract".to_string(),
-            language: "eng".to_string(),
-            tesseract_config: None,
-        }),
-        force_ocr: true,
-        ..Default::default()
-    };
-
-    let result = extract_file(&path, None, &config).await;
-
-    match result {
-        Ok(res) => {
-            assert_mime_type(&res, "image/png");
-            assert_non_empty_content(&res);
-            let content_lower = res.content.to_lowercase();
-            assert!(
-                content_lower.contains("hello") || content_lower.contains("world"),
-                "OCR output {:?} should contain HELLO or WORLD",
-                res.content
-            );
-        }
-        Err(err) => {
-            tracing::debug!("OCR test failed (Tesseract may not be installed): {}", err);
-        }
-    }
-}
-
-#[cfg(feature = "ocr")]
-#[tokio::test]
-async fn test_ocr_image_without_text_async() {
-    if !test_documents_available() {
-        return;
-    }
-
-    let path = get_test_file_path("images/flower_no_text.jpg");
-    if !path.exists() {
-        tracing::debug!("Skipping test: OCR flower image not available");
-        return;
-    }
-
-    let config = ExtractionConfig {
-        ocr: Some(OcrConfig {
-            backend: "tesseract".to_string(),
-            language: "eng".to_string(),
-            tesseract_config: None,
-        }),
-        force_ocr: true,
-        ..Default::default()
-    };
-
-    let result = extract_file(&path, None, &config).await;
-
-    match result {
-        Ok(res) => {
-            assert_mime_type(&res, "image/jpeg");
-            assert!(
-                res.content.len() < 200,
-                "Expected minimal OCR output, got {} bytes",
-                res.content.len()
-            );
-        }
-        Err(err) => {
-            tracing::debug!("OCR fallback test failed (Tesseract may not be installed): {}", err);
-        }
-    }
-}
+//! Binding-specific format integration tests.
+//!
+//! Positive-path scenarios are now covered by the shared fixture-based E2E
+//! suites. The tests here focus on behaviour that is specific to the Rust
+//! asynchronous APIs or to graceful handling when optional system
+//! dependencies are missing.
+
+mod helpers;
+
+use helpers::{assert_mime_type, assert_non_empty_content, get_test_file_path, test_documents_available};
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::extractor::extract_file;
+
+#[cfg(feature = "ocr")]
+use kreuzberg::core::config::OcrConfig;
+
+#[cfg(feature = "pdf")]
+#[tokio::test]
+async fn test_pdf_password_protected_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("pdfs/copy_protected.pdf");
+    if !path.exists() {
+        tracing::debug!("Skipping test: protected PDF not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Err(err) => {
+            tracing::debug!("Password protection detected (expected): {}", err);
+        }
+        Ok(res) => {
+            tracing::debug!("Protected PDF extracted; some files allow fallback");
+            assert_mime_type(&res, "application/pdf");
+            assert!(res.chunks.is_none(), "Chunks should be None without chunking config");
+            assert!(res.detected_languages.is_none(), "Language detection not enabled");
+        }
+    }
+}
+
+#[cfg(feature = "office")]
+#[tokio::test]
+async fn test_legacy_doc_extraction_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("legacy_office/simple.doc");
+    if !path.exists() {
+        tracing::debug!("Skipping test: legacy .doc file not available");
+        return;
+    }
+
+    let result = extract_file(&path, None, &ExtractionConfig::default()).await;
+
+    match result {
+        Ok(extracted) => {
+            assert_mime_type(&extracted, "application/msword");
+            assert_non_empty_content(&extracted);
+            assert!(
+                extracted.chunks.is_none(),
+                "Chunks should be None without chunking config"
+            );
+            assert!(extracted.detected_languages.is_none(), "Language detection not enabled");
+        }
+        Err(err) => {
+            tracing::debug!(
+                "Legacy Office extraction failed (LibreOffice may not be installed): {}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_simple_english_image_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/test_hello_world.png");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR sample image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/png");
+            assert_non_empty_content(&res);
+            let content_lower = res.content.to_lowercase();
+            assert!(
+                content_lower.contains("hello") || content_lower.contains("world"),
+                "OCR output {:?} should contain HELLO or WORLD",
+                res.content
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_image_without_text_async() {
+    if !test_documents_available() {
+        return;
+    }
+
+    let path = get_test_file_path("images/flower_no_text.jpg");
+    if !path.exists() {
+        tracing::debug!("Skipping test: OCR flower image not available");
+        return;
+    }
+
+    let config = ExtractionConfig {
+        ocr: Some(OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+            tesseract_config: None,
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+
+    let result = extract_file(&path, None, &config).await;
+
+    match result {
+        Ok(res) => {
+            assert_mime_type(&res, "image/jpeg");
+            assert!(
+                res.content.len() < 200,
+                "Expected minimal OCR output, got {} bytes",
+                res.content.len()
+            );
+        }
+        Err(err) => {
+            tracing::debug!("OCR fallback test failed (Tesseract may not be installed): {}", err);
+        }
+    }
+}