kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/tests/html_table_test.rs

@@ -34,7 +34,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "HTML to markdown conversion should succeed");
 
         let markdown = result.unwrap();
@@ -76,7 +76,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should convert to markdown");
 
         let markdown = result.unwrap();
@@ -140,7 +140,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should convert complex table");
 
         let markdown = result.unwrap();
@@ -191,7 +191,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle merged cell table");
 
         let markdown = result.unwrap();
@@ -245,7 +245,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle multiple tables");
 
         let markdown = result.unwrap();
@@ -300,7 +300,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle mixed header cells");
 
         let markdown = result.unwrap();
@@ -346,7 +346,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle table with caption");
 
         let markdown = result.unwrap();
@@ -382,7 +382,7 @@ mod html_table_tests {
     fn test_simple_flat_table() {
         let html = r#"<table><tr><td>A</td><td>B</td></tr><tr><td>C</td><td>D</td></tr></table>"#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle flat table");
 
         let markdown = result.unwrap();
@@ -418,7 +418,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle empty cells");
 
         let markdown = result.unwrap();
@@ -456,7 +456,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle numeric table");
 
         let markdown = result.unwrap();
@@ -499,7 +499,7 @@ mod html_table_tests {
         </table>
         "#;
 
-        let result = convert_html_to_markdown(html, None);
+        let result = convert_html_to_markdown(html, None, None);
         assert!(result.is_ok(), "Should handle unicode characters");
 
         let markdown = result.unwrap();

data/vendor/kreuzberg/tests/ocr_configuration.rs

@@ -32,6 +32,7 @@ fn test_ocr_language_english() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -57,6 +58,7 @@ fn test_ocr_language_german() {
             backend: "tesseract".to_string(),
             language: "deu".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -95,6 +97,7 @@ fn test_ocr_language_multiple() {
             backend: "tesseract".to_string(),
             language: "eng+kor".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -136,6 +139,7 @@ fn test_ocr_psm_auto() {
                 psm: 3,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -164,6 +168,7 @@ fn test_ocr_psm_single_block() {
                 psm: 6,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -192,6 +197,7 @@ fn test_ocr_psm_single_line() {
                 psm: 7,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -218,6 +224,7 @@ fn test_force_ocr_on_text_pdf() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -248,6 +255,7 @@ fn test_force_ocr_disabled() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -283,6 +291,7 @@ fn test_table_detection_enabled() {
                 table_row_threshold_ratio: 0.5,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -311,6 +320,7 @@ fn test_table_detection_disabled() {
                 enable_table_detection: false,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -339,6 +349,7 @@ fn test_language_model_ngram_configuration() {
                 language_model_ngram_on: true,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -368,6 +379,7 @@ fn test_dictionary_correction_enabled() {
                 tessedit_enable_dict_correction: true,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -397,6 +409,7 @@ fn test_character_whitelist() {
                 tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz ".to_string(),
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -425,6 +438,7 @@ fn test_ocr_cache_enabled() {
                 use_cache: true,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -464,6 +478,7 @@ fn test_ocr_cache_disabled() {
                 use_cache: false,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: false,
@@ -498,6 +513,7 @@ fn test_complex_configuration_combination() {
                 use_cache: true,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,

data/vendor/kreuzberg/tests/ocr_errors.rs

@@ -34,6 +34,7 @@ fn test_ocr_invalid_language_code() {
             backend: "tesseract".to_string(),
             language: "invalid_lang_99999".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -74,6 +75,7 @@ fn test_ocr_invalid_psm_mode() {
                 psm: 999,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -112,6 +114,7 @@ fn test_ocr_invalid_backend_name() {
             backend: "nonexistent_ocr_backend_xyz".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -147,6 +150,7 @@ fn test_ocr_corrupted_image_data() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -177,6 +181,7 @@ fn test_ocr_empty_image() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -207,6 +212,7 @@ fn test_ocr_non_image_data() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -245,6 +251,7 @@ fn test_ocr_extreme_table_threshold() {
                 table_row_threshold_ratio: 10.0,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -281,6 +288,7 @@ fn test_ocr_negative_psm() {
                 psm: -5,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -313,6 +321,7 @@ fn test_ocr_empty_whitelist() {
                 tessedit_char_whitelist: "".to_string(),
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -349,6 +358,7 @@ fn test_ocr_conflicting_whitelist_blacklist() {
                 tessedit_char_blacklist: "abc".to_string(),
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -381,6 +391,7 @@ fn test_ocr_empty_language() {
             backend: "tesseract".to_string(),
             language: "".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -413,6 +424,7 @@ fn test_ocr_malformed_multi_language() {
             backend: "tesseract".to_string(),
             language: "eng++deu++fra".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -446,6 +458,7 @@ fn test_ocr_cache_disabled_then_enabled() {
                 use_cache: false,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: false,
@@ -466,6 +479,7 @@ fn test_ocr_cache_disabled_then_enabled() {
                 use_cache: true,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -497,6 +511,7 @@ fn test_ocr_concurrent_same_file() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -563,6 +578,7 @@ fn test_ocr_concurrent_different_files() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: true,
@@ -632,6 +648,7 @@ fn test_ocr_with_preprocessing_extreme_dpi() {
                 }),
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()
@@ -677,6 +694,7 @@ fn test_ocr_with_invalid_binarization_method() {
                 }),
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: false,
         ..Default::default()

data/vendor/kreuzberg/tests/ocr_quality.rs

@@ -167,6 +167,7 @@ fn test_ocr_quality_simple_text_high_accuracy() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -241,6 +242,7 @@ fn test_ocr_quality_numeric_accuracy() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -306,6 +308,7 @@ fn test_ocr_quality_layout_preservation() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -365,6 +368,7 @@ fn test_ocr_quality_technical_document() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -409,6 +413,7 @@ fn test_ocr_consistency_across_runs() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         use_cache: false,
@@ -474,6 +479,7 @@ fn test_ocr_consistency_with_different_psm() {
                 psm: 3,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -487,6 +493,7 @@ fn test_ocr_consistency_with_different_psm() {
                 psm: 6,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -557,6 +564,7 @@ fn test_ocr_quality_multi_page_consistency() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()
@@ -608,6 +616,7 @@ fn test_ocr_quality_with_tables() {
                 table_min_confidence: 0.5,
                 ..Default::default()
             }),
+            output_format: None,
         }),
         force_ocr: true,
         ..Default::default()

data/vendor/kreuzberg/tests/ocr_stress.rs

@@ -198,6 +198,7 @@ fn test_tesseract_api_thread_safety() {
             backend: "tesseract".to_string(),
             language: "eng".to_string(),
             tesseract_config: None,
+            output_format: None,
         }),
         force_ocr: false,
         use_cache: false,

data/vendor/kreuzberg/tests/pipeline_integration.rs

@@ -141,6 +141,8 @@ async fn test_pipeline_empty_no_processors() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -186,6 +188,8 @@ async fn test_pipeline_single_processor_per_stage() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -231,6 +235,8 @@ async fn test_pipeline_multiple_processors_per_stage() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -267,6 +273,8 @@ async fn test_pipeline_all_stages_enabled() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -301,6 +309,8 @@ async fn test_pipeline_postprocessing_disabled() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -350,6 +360,8 @@ async fn test_pipeline_early_stage_runs_first() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -390,6 +402,8 @@ async fn test_pipeline_middle_stage_runs_second() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -426,6 +440,8 @@ async fn test_pipeline_late_stage_runs_last() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -462,6 +478,8 @@ async fn test_pipeline_within_stage_priority_order() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -527,6 +545,8 @@ async fn test_pipeline_cross_stage_data_flow() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -584,6 +604,8 @@ async fn test_pipeline_early_stage_error_recorded() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -626,6 +648,8 @@ async fn test_pipeline_middle_stage_error_propagation() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -698,6 +722,8 @@ async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -786,6 +812,8 @@ async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -864,6 +892,8 @@ async fn test_pipeline_multiple_processor_errors() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -906,6 +936,8 @@ async fn test_pipeline_error_context_preservation() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -978,6 +1010,8 @@ async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1049,6 +1083,8 @@ async fn test_pipeline_content_modified_in_middle_visible_in_late() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1118,6 +1154,8 @@ async fn test_pipeline_multiple_processors_modifying_same_metadata() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1206,6 +1244,8 @@ async fn test_pipeline_processors_reading_previous_output() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1261,6 +1301,8 @@ async fn test_pipeline_large_content_modification() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1297,6 +1339,8 @@ async fn test_pipeline_enabled_processors_whitelist() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1344,6 +1388,8 @@ async fn test_pipeline_disabled_processors_blacklist() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1391,6 +1437,8 @@ async fn test_pipeline_no_filtering_runs_all() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1429,6 +1477,8 @@ async fn test_pipeline_empty_whitelist_runs_none() {
         chunks: None,
         images: None,
         pages: None,
+        djot_content: None,
+        elements: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {