kreuzberg 4.1.2 → 4.2.1

data/vendor/kreuzberg/tests/api_consistency.rs

@@ -0,0 +1,349 @@
+//! API consistency tests for ExtractionConfig and related types.
+//!
+//! This test suite validates that:
+//! 1. ExtractionConfig serialization is complete with all fields
+//! 2. All required configuration fields are present
+//! 3. Configuration types maintain consistency across different formats
+//! 4. No configuration fields are accidentally hidden or lost
+
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::config::OutputFormat;
+use serde_json::json;
+
+#[test]
+fn test_extraction_config_serialization_includes_all_fields() {
+    let config = ExtractionConfig::default();
+    let json = serde_json::to_value(&config).expect("Failed to serialize config");
+
+    // Verify core fields exist and are accessible
+    assert!(
+        json.get("use_cache").is_some(),
+        "Missing 'use_cache' field in serialized config"
+    );
+    assert!(
+        json.get("enable_quality_processing").is_some(),
+        "Missing 'enable_quality_processing' field"
+    );
+    assert!(
+        json.get("force_ocr").is_some(),
+        "Missing 'force_ocr' field in serialized config"
+    );
+    assert!(
+        json.get("max_concurrent_extractions").is_some(),
+        "Missing 'max_concurrent_extractions' field"
+    );
+    assert!(
+        json.get("result_format").is_some(),
+        "Missing 'result_format' field in serialized config"
+    );
+    assert!(
+        json.get("output_format").is_some(),
+        "Missing 'output_format' field in serialized config"
+    );
+}
+
+#[test]
+fn test_extraction_config_defaults_are_correct() {
+    let config = ExtractionConfig::default();
+
+    assert!(config.use_cache, "Default use_cache should be true");
+    assert!(
+        config.enable_quality_processing,
+        "Default enable_quality_processing should be true"
+    );
+    assert!(!config.force_ocr, "Default force_ocr should be false");
+    assert_eq!(
+        config.max_concurrent_extractions, None,
+        "Default max_concurrent_extractions should be None"
+    );
+}
+
+#[test]
+fn test_extraction_config_serialization_roundtrip() {
+    let config = ExtractionConfig::default();
+
+    // Serialize to JSON
+    let json_string = serde_json::to_string(&config).expect("Failed to serialize");
+
+    // Deserialize back
+    let deserialized: ExtractionConfig =
+        serde_json::from_str(&json_string).expect("Failed to deserialize config from JSON");
+
+    // Verify roundtrip integrity
+    assert_eq!(
+        config.use_cache, deserialized.use_cache,
+        "use_cache should survive roundtrip"
+    );
+    assert_eq!(
+        config.enable_quality_processing, deserialized.enable_quality_processing,
+        "enable_quality_processing should survive roundtrip"
+    );
+    assert_eq!(
+        config.force_ocr, deserialized.force_ocr,
+        "force_ocr should survive roundtrip"
+    );
+    assert_eq!(
+        config.result_format, deserialized.result_format,
+        "result_format should survive roundtrip"
+    );
+    assert_eq!(
+        config.output_format, deserialized.output_format,
+        "output_format should survive roundtrip"
+    );
+}
+
+#[test]
+fn test_extraction_config_json_structure() {
+    let config = ExtractionConfig::default();
+    let json = serde_json::to_value(&config).expect("Failed to serialize config");
+
+    let obj = json.as_object().expect("Config should serialize as object");
+
+    // Verify all expected fields are present as keys
+    let expected_fields = vec![
+        "use_cache",
+        "enable_quality_processing",
+        "force_ocr",
+        "max_concurrent_extractions",
+        "result_format",
+        "output_format",
+    ];
+
+    for field in expected_fields {
+        assert!(obj.contains_key(field), "Missing field in JSON: {}", field);
+    }
+}
+
+#[test]
+fn test_extraction_config_values_are_correct_types() {
+    let config = ExtractionConfig::default();
+    let json = serde_json::to_value(&config).expect("Failed to serialize config");
+
+    // Verify field types
+    assert!(
+        json.get("use_cache").expect("Value not found").is_boolean(),
+        "use_cache should be boolean"
+    );
+    assert!(
+        json.get("enable_quality_processing")
+            .expect("Value not found")
+            .is_boolean(),
+        "enable_quality_processing should be boolean"
+    );
+    assert!(
+        json.get("force_ocr").expect("Value not found").is_boolean(),
+        "force_ocr should be boolean"
+    );
+    assert!(
+        json.get("result_format").expect("Value not found").is_string(),
+        "result_format should be string"
+    );
+    assert!(
+        json.get("output_format").expect("Value not found").is_string(),
+        "output_format should be string"
+    );
+}
+
+#[test]
+fn test_extraction_config_with_custom_values() {
+    let config = ExtractionConfig {
+        use_cache: false,
+        force_ocr: true,
+        max_concurrent_extractions: Some(8),
+        ..ExtractionConfig::default()
+    };
+
+    let json = serde_json::to_value(&config).expect("Failed to serialize");
+
+    assert_eq!(json.get("use_cache").expect("Value not found"), &json!(false));
+    assert_eq!(json.get("force_ocr").expect("Value not found"), &json!(true));
+    assert_eq!(
+        json.get("max_concurrent_extractions").expect("Value not found"),
+        &json!(8)
+    );
+}
+
+#[test]
+fn test_extraction_config_partial_json_parsing() {
+    // Test that we can parse partial JSON and fields get defaults
+    let partial_json = json!({
+        "use_cache": false,
+    });
+
+    let config: ExtractionConfig = serde_json::from_value(partial_json).expect("Failed to parse partial config");
+
+    assert!(!config.use_cache, "Explicit use_cache should be respected");
+    assert!(
+        config.enable_quality_processing,
+        "Omitted enable_quality_processing should use default"
+    );
+    assert!(!config.force_ocr, "Omitted force_ocr should use default");
+}
+
+#[test]
+fn test_extraction_config_empty_json_uses_defaults() {
+    // Empty object should use all defaults
+    let empty_json = json!({});
+
+    let config: ExtractionConfig = serde_json::from_value(empty_json).expect("Failed to parse empty config");
+
+    let default_config = ExtractionConfig::default();
+    assert_eq!(config.use_cache, default_config.use_cache);
+    assert_eq!(
+        config.enable_quality_processing,
+        default_config.enable_quality_processing
+    );
+    assert_eq!(config.force_ocr, default_config.force_ocr);
+    assert_eq!(config.result_format, default_config.result_format);
+    assert_eq!(config.output_format, default_config.output_format);
+}
+
+#[test]
+fn test_extraction_config_output_format_valid_values() {
+    // Test that output_format accepts valid values (case-insensitive)
+    let json_plain = json!({"output_format": "plain"});
+    let config_plain: ExtractionConfig =
+        serde_json::from_value(json_plain).expect("Failed to parse plain output_format");
+    assert_eq!(config_plain.output_format, OutputFormat::Plain);
+
+    let json_markdown = json!({"output_format": "markdown"});
+    let config_markdown: ExtractionConfig =
+        serde_json::from_value(json_markdown).expect("Failed to parse markdown output_format");
+    assert_eq!(config_markdown.output_format, OutputFormat::Markdown);
+
+    let json_html = json!({"output_format": "html"});
+    let config_html: ExtractionConfig = serde_json::from_value(json_html).expect("Failed to parse html output_format");
+    assert_eq!(config_html.output_format, OutputFormat::Html);
+}
+
+#[test]
+fn test_extraction_config_result_format_valid_values() {
+    // Test that result_format accepts valid values
+    let json_unified = json!({"result_format": "unified"});
+    let config_unified: ExtractionConfig =
+        serde_json::from_value(json_unified).expect("Failed to parse unified result_format");
+    // result_format uses types::OutputFormat, not core::config::OutputFormat
+    let _ = config_unified.result_format;
+}
+
+#[test]
+fn test_extraction_config_no_unknown_fields_in_default() {
+    // Verify that the default config only has expected fields when serialized
+    let config = ExtractionConfig::default();
+    let json = serde_json::to_value(&config).expect("Failed to serialize");
+    let obj = json.as_object().expect("Should be object");
+
+    // These are the fields we expect (some may be null based on feature flags)
+    let expected_fields = vec![
+        "use_cache",
+        "enable_quality_processing",
+        "ocr",
+        "force_ocr",
+        "chunking",
+        "images",
+        "pdf_options",
+        "token_reduction",
+        "language_detection",
+        "pages",
+        "keywords",
+        "postprocessor",
+        "html_options",
+        "max_concurrent_extractions",
+        "result_format",
+        "output_format",
+    ];
+
+    for key in obj.keys() {
+        assert!(
+            expected_fields.contains(&key.as_str()),
+            "Unexpected field in config: {}",
+            key
+        );
+    }
+}
+
+#[test]
+fn test_extraction_config_needs_image_processing() {
+    // Test the needs_image_processing helper method
+    let mut config = ExtractionConfig::default();
+
+    // By default, should not need image processing
+    assert!(
+        !config.needs_image_processing(),
+        "Default config should not need image processing"
+    );
+
+    // With OCR enabled, should need image processing
+    config.ocr = Some(kreuzberg::OcrConfig {
+        backend: "tesseract".to_string(),
+        language: "eng".to_string(),
+        tesseract_config: None,
+        output_format: None,
+    });
+    assert!(
+        config.needs_image_processing(),
+        "Config with OCR should need image processing"
+    );
+
+    // Reset for next test
+    config.ocr = None;
+    config.images = Some(kreuzberg::ImageExtractionConfig {
+        extract_images: true,
+        target_dpi: 150,
+        max_image_dimension: 2000,
+        auto_adjust_dpi: true,
+        min_dpi: 72,
+        max_dpi: 600,
+    });
+    assert!(
+        config.needs_image_processing(),
+        "Config with image extraction should need image processing"
+    );
+}
+
+#[test]
+fn test_output_format_serialization_lowercase() {
+    // Verify that OutputFormat serializes to lowercase values
+    let json = serde_json::json!({"output_format": "markdown"});
+    let config: ExtractionConfig = serde_json::from_value(json).expect("Failed to parse");
+    let reserialized = serde_json::to_value(&config).expect("Failed to reserialize");
+
+    // Should serialize back to lowercase
+    assert_eq!(reserialized["output_format"], "markdown");
+}
+
+#[test]
+fn test_extraction_config_field_presence_consistency() {
+    // Test that all serialized configs have the expected top-level fields
+    let config = ExtractionConfig::default();
+    let json1 = serde_json::to_value(&config).expect("Failed to serialize");
+
+    let config2 = ExtractionConfig {
+        force_ocr: true,
+        ..ExtractionConfig::default()
+    };
+    let json2 = serde_json::to_value(&config2).expect("Failed to serialize");
+
+    // Both should have the same top-level keys
+    let keys1: Vec<_> = json1.as_object().expect("Expected object value").keys().collect();
+    let keys2: Vec<_> = json2.as_object().expect("Expected object value").keys().collect();
+
+    assert_eq!(keys1.len(), keys2.len(), "Configs should have same number of keys");
+}
+
+#[test]
+fn test_output_format_all_variants() {
+    // Test all output format variants can be serialized and deserialized
+    let formats = vec![
+        OutputFormat::Plain,
+        OutputFormat::Markdown,
+        OutputFormat::Html,
+        OutputFormat::Djot,
+    ];
+
+    for fmt in formats {
+        let serialized = serde_json::to_value(fmt).expect("Failed to serialize");
+        let deserialized: OutputFormat = serde_json::from_value(serialized).expect("Failed to deserialize");
+        assert_eq!(fmt, deserialized, "Format should survive roundtrip");
+    }
+}

data/vendor/kreuzberg/tests/api_embed.rs

@@ -29,16 +29,20 @@ async fn test_embed_valid_texts() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::OK);
 
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let embed_response: EmbedResponse = serde_json::from_slice(&body).unwrap();
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response: EmbedResponse = serde_json::from_slice(&body).expect("Failed to deserialize");
 
     assert_eq!(embed_response.count, 2);
     assert_eq!(embed_response.embeddings.len(), 2);
@@ -66,11 +70,13 @@ async fn test_embed_empty_texts() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::BAD_REQUEST);
 }
@@ -97,16 +103,20 @@ async fn test_embed_with_custom_config() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::OK);
 
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let embed_response: EmbedResponse = serde_json::from_slice(&body).unwrap();
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response: EmbedResponse = serde_json::from_slice(&body).expect("Failed to deserialize");
 
     assert_eq!(embed_response.count, 1);
     assert_eq!(embed_response.embeddings.len(), 1);
@@ -128,16 +138,20 @@ async fn test_embed_single_text() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::OK);
 
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let embed_response: EmbedResponse = serde_json::from_slice(&body).unwrap();
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response: EmbedResponse = serde_json::from_slice(&body).expect("Failed to deserialize");
 
     assert_eq!(embed_response.count, 1);
     assert_eq!(embed_response.embeddings.len(), 1);
@@ -160,16 +174,20 @@ async fn test_embed_batch() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::OK);
 
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let embed_response: EmbedResponse = serde_json::from_slice(&body).unwrap();
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response: EmbedResponse = serde_json::from_slice(&body).expect("Failed to deserialize");
 
     assert_eq!(embed_response.count, 10);
     assert_eq!(embed_response.embeddings.len(), 10);
@@ -198,16 +216,20 @@ async fn test_embed_long_text() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::OK);
 
-    let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
-    let embed_response: EmbedResponse = serde_json::from_slice(&body).unwrap();
+    let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response: EmbedResponse = serde_json::from_slice(&body).expect("Failed to deserialize");
 
     assert_eq!(embed_response.count, 1);
     assert_eq!(embed_response.embeddings.len(), 1);
@@ -225,10 +247,10 @@ async fn test_embed_malformed_json() {
                 .uri("/embed")
                 .header("content-type", "application/json")
                 .body(Body::from("{invalid json}"))
-                .unwrap(),
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response.status(), StatusCode::BAD_REQUEST);
 }
@@ -250,16 +272,20 @@ async fn test_embed_deterministic() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response1.status(), StatusCode::OK);
 
-    let body1 = axum::body::to_bytes(response1.into_body(), usize::MAX).await.unwrap();
-    let embed_response1: EmbedResponse = serde_json::from_slice(&body1).unwrap();
+    let body1 = axum::body::to_bytes(response1.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response1: EmbedResponse = serde_json::from_slice(&body1).expect("Failed to deserialize");
 
     // Second call with same text
     let response2 = app
@@ -268,16 +294,20 @@ async fn test_embed_deterministic() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_body).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_body).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response2.status(), StatusCode::OK);
 
-    let body2 = axum::body::to_bytes(response2.into_body(), usize::MAX).await.unwrap();
-    let embed_response2: EmbedResponse = serde_json::from_slice(&body2).unwrap();
+    let body2 = axum::body::to_bytes(response2.into_body(), usize::MAX)
+        .await
+        .expect("Failed to convert to bytes");
+    let embed_response2: EmbedResponse = serde_json::from_slice(&body2).expect("Failed to deserialize");
 
     // Compare embeddings - they should be identical
     assert_eq!(embed_response1.embeddings.len(), embed_response2.embeddings.len());
@@ -307,18 +337,20 @@ async fn test_embed_different_presets() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_fast).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_fast).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response_fast.status(), StatusCode::OK);
 
     let body_fast = axum::body::to_bytes(response_fast.into_body(), usize::MAX)
         .await
-        .unwrap();
-    let embed_fast: EmbedResponse = serde_json::from_slice(&body_fast).unwrap();
+        .expect("Operation failed");
+    let embed_fast: EmbedResponse = serde_json::from_slice(&body_fast).expect("Failed to deserialize");
 
     // Test with "balanced" preset
     let request_balanced = json!({
@@ -337,18 +369,20 @@ async fn test_embed_different_presets() {
                 .method("POST")
                 .uri("/embed")
                 .header("content-type", "application/json")
-                .body(Body::from(serde_json::to_string(&request_balanced).unwrap()))
-                .unwrap(),
+                .body(Body::from(
+                    serde_json::to_string(&request_balanced).expect("Operation failed"),
+                ))
+                .expect("Operation failed"),
         )
         .await
-        .unwrap();
+        .expect("Operation failed");
 
     assert_eq!(response_balanced.status(), StatusCode::OK);
 
     let body_balanced = axum::body::to_bytes(response_balanced.into_body(), usize::MAX)
         .await
-        .unwrap();
-    let embed_balanced: EmbedResponse = serde_json::from_slice(&body_balanced).unwrap();
+        .expect("Operation failed");
+    let embed_balanced: EmbedResponse = serde_json::from_slice(&body_balanced).expect("Failed to deserialize");
 
     // Different presets should have different dimensions
     assert_ne!(embed_fast.dimensions, embed_balanced.dimensions);

data/vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs

@@ -93,7 +93,10 @@ startxref
         .expect("Failed to read response body");
 
     let parsed: Value = serde_json::from_slice(&body).expect("Failed to parse response");
-    eprintln!("Extraction result: {}", serde_json::to_string_pretty(&parsed).unwrap());
+    eprintln!(
+        "Extraction result: {}",
+        serde_json::to_string_pretty(&parsed).expect("Failed to parse")
+    );
 }
 
 /// Test extracting a 1MB text file (control test without PDF).
@@ -187,7 +190,10 @@ async fn test_find_size_breaking_point() {
                 .expect("Failed to read response body");
 
             if let Ok(parsed) = serde_json::from_slice::<Value>(&body) {
-                eprintln!("Error response: {}", serde_json::to_string_pretty(&parsed).unwrap());
+                eprintln!(
+                    "Error response: {}",
+                    serde_json::to_string_pretty(&parsed).expect("Failed to parse")
+                );
             } else {
                 eprintln!("Response body (not JSON): {}", String::from_utf8_lossy(&body));
             }