kreuzberg 4.0.0.pre.rc.6

data/vendor/kreuzberg/tests/image_integration.rs

@@ -0,0 +1,253 @@
+//! Image and OCR integration tests using real image files.
+//!
+//! This module tests image extraction and OCR processing end-to-end with real
+//! image files from the test_documents/ directory. Tests verify that both
+//! image metadata extraction and OCR text extraction work correctly.
+//!
+//! Test philosophy:
+//! - Use real images from test_documents/
+//! - Assert on behavior, not implementation
+//! - Test different image formats (PNG, JPG, BMP, etc.)
+//! - Test OCR with various languages and layouts
+//! - Verify graceful handling of images without text
+
+mod helpers;
+
+use helpers::*;
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::extract_file_sync;
+
+#[test]
+fn test_jpg_image_metadata() {
+    if skip_if_missing("images/example.jpg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/example.jpg");
+    let result = extract_file_sync(&file_path, None, &ExtractionConfig::default())
+        .expect("Should extract JPG image successfully");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_png_image_metadata() {
+    if skip_if_missing("images/test_hello_world.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/test_hello_world.png");
+    let result = extract_file_sync(&file_path, None, &ExtractionConfig::default())
+        .expect("Should extract PNG image successfully");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_bmp_image_format() {
+    if skip_if_missing("images/bmp_24.bmp") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/bmp_24.bmp");
+    let result = extract_file_sync(&file_path, None, &ExtractionConfig::default())
+        .expect("Should extract BMP image successfully");
+
+    assert_mime_type(&result, "image/bmp");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_ocr_simple_text() {
+    if skip_if_missing("images/test_hello_world.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/test_hello_world.png");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract text from image with OCR");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+
+    if !result.content.trim().is_empty() {
+        assert_min_content_length(&result, 5);
+    }
+}
+
+#[test]
+fn test_ocr_document_image() {
+    if skip_if_missing("images/ocr_image.jpg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/ocr_image.jpg");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract text from document image");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+
+    if !result.content.trim().is_empty() {
+        assert_min_content_length(&result, 10);
+    }
+}
+
+#[test]
+fn test_ocr_layout_parser() {
+    if skip_if_missing("images/layout_parser_ocr.jpg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/layout_parser_ocr.jpg");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract text from layout parser image");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+
+    if !result.content.trim().is_empty() {
+        assert_min_content_length(&result, 20);
+    }
+}
+
+#[test]
+fn test_ocr_invoice_image() {
+    if skip_if_missing("images/invoice_image.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/invoice_image.png");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract text from invoice image");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+
+    if !result.content.trim().is_empty() {
+        assert_min_content_length(&result, 10);
+    }
+}
+
+#[test]
+fn test_table_image_simple() {
+    if skip_if_missing("tables/simple_table.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("tables/simple_table.png");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract table image successfully");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_table_image_complex() {
+    if skip_if_missing("tables/complex_document.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("tables/complex_document.png");
+    let config = test_config_with_ocr();
+
+    let result =
+        extract_file_sync(&file_path, None, &config).expect("Should extract complex document image successfully");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_ocr_multilang_english_korean() {
+    if skip_if_missing("images/english_and_korean.png") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/english_and_korean.png");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract mixed language image");
+
+    assert_mime_type(&result, "image/png");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_ocr_chinese_simplified() {
+    if skip_if_missing("images/chi_sim_image.jpeg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/chi_sim_image.jpeg");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should process Chinese image");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_ocr_japanese_vertical() {
+    if skip_if_missing("images/jpn_vert.jpeg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/jpn_vert.jpeg");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should process Japanese vertical text image");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+
+#[test]
+fn test_image_no_text() {
+    if skip_if_missing("images/flower_no_text.jpg") {
+        return;
+    }
+
+    let file_path = get_test_file_path("images/flower_no_text.jpg");
+    let config = test_config_with_ocr();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should process image without text");
+
+    assert_mime_type(&result, "image/jpeg");
+
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}

data/vendor/kreuzberg/tests/instrumentation_test.rs

@@ -0,0 +1,139 @@
+#![cfg(feature = "otel")]
+
+use std::sync::{Arc, Mutex};
+use tracing::Subscriber;
+use tracing::span::{Attributes, Id};
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::{Context, SubscriberExt};
+use tracing_subscriber::registry::LookupSpan;
+
+/// Simple span name collector for testing.
+///
+/// This layer collects span names as they are created to verify
+/// that instrumentation is working correctly.
+struct SpanCollector {
+    spans: Arc<Mutex<Vec<String>>>,
+}
+
+impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for SpanCollector {
+    fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: Context<'_, S>) {
+        self.spans.lock().unwrap().push(attrs.metadata().name().to_string());
+    }
+}
+
+#[tokio::test]
+async fn test_cache_instrumentation() {
+    use kreuzberg::cache::GenericCache;
+    use tempfile::tempdir;
+
+    let spans = Arc::new(Mutex::new(Vec::new()));
+    let collector = SpanCollector { spans: spans.clone() };
+
+    let subscriber = tracing_subscriber::registry().with(collector);
+    let _guard = tracing::subscriber::set_default(subscriber);
+
+    let temp_dir = tempdir().unwrap();
+    let cache = GenericCache::new(
+        "test".to_string(),
+        Some(temp_dir.path().to_str().unwrap().to_string()),
+        30.0,
+        500.0,
+        1000.0,
+    )
+    .unwrap();
+
+    cache.set("test_key", b"test data".to_vec(), None).unwrap();
+
+    let _ = cache.get("test_key", None).unwrap();
+
+    let span_names = spans.lock().unwrap();
+    assert!(span_names.contains(&"set".to_string()), "Expected 'set' span");
+    assert!(span_names.contains(&"get".to_string()), "Expected 'get' span");
+}
+
+#[cfg(feature = "ocr")]
+#[tokio::test]
+async fn test_ocr_instrumentation() {
+    use kreuzberg::ocr::processor::OcrProcessor;
+    use kreuzberg::ocr::types::TesseractConfig;
+    use tempfile::tempdir;
+
+    let spans = Arc::new(Mutex::new(Vec::new()));
+    let collector = SpanCollector { spans: spans.clone() };
+
+    let subscriber = tracing_subscriber::registry().with(collector);
+    let _guard = tracing::subscriber::set_default(subscriber);
+
+    let temp_dir = tempdir().unwrap();
+    let processor = OcrProcessor::new(Some(temp_dir.path().to_path_buf())).unwrap();
+
+    let mut test_image = Vec::new();
+    let img = image::ImageBuffer::from_fn(1, 1, |_, _| image::Rgb([255u8, 255u8, 255u8]));
+    img.write_to(&mut std::io::Cursor::new(&mut test_image), image::ImageFormat::Png)
+        .unwrap();
+
+    let config = TesseractConfig {
+        output_format: "text".to_string(),
+        use_cache: false,
+        ..TesseractConfig::default()
+    };
+
+    let _ = processor.process_image(&test_image, &config);
+
+    let span_names = spans.lock().unwrap();
+    assert!(
+        span_names.contains(&"process_image".to_string()),
+        "Expected 'process_image' span"
+    );
+}
+
+#[tokio::test]
+async fn test_registry_instrumentation() {
+    use kreuzberg::plugins::registry::DocumentExtractorRegistry;
+
+    let spans = Arc::new(Mutex::new(Vec::new()));
+    let collector = SpanCollector { spans: spans.clone() };
+
+    let subscriber = tracing_subscriber::registry().with(collector);
+    let _guard = tracing::subscriber::set_default(subscriber);
+
+    let registry = DocumentExtractorRegistry::new();
+
+    let _ = registry.get("application/pdf");
+
+    let span_names = spans.lock().unwrap();
+    assert!(
+        span_names.contains(&"get".to_string()),
+        "Expected 'get' span from registry"
+    );
+}
+
+#[cfg(all(feature = "pdf", feature = "office"))]
+#[tokio::test]
+async fn test_span_hierarchy() {
+    use kreuzberg::core::config::ExtractionConfig;
+    use kreuzberg::core::extractor::extract_bytes;
+
+    let spans = Arc::new(Mutex::new(Vec::new()));
+    let collector = SpanCollector { spans: spans.clone() };
+
+    let subscriber = tracing_subscriber::registry().with(collector);
+    let _guard = tracing::subscriber::set_default(subscriber);
+
+    let test_content = b"Hello, World!";
+    let config = ExtractionConfig::default();
+
+    let _ = extract_bytes(test_content, "text/plain", &config).await;
+
+    let span_names = spans.lock().unwrap();
+    assert!(
+        span_names.contains(&"extract_bytes".to_string()),
+        "Expected 'extract_bytes' span"
+    );
+}
+
+#[test]
+fn test_span_collector_creation() {
+    let spans = Arc::new(Mutex::new(Vec::new()));
+    let _collector = SpanCollector { spans };
+}