kreuzberg 4.0.0.pre.rc.8 → 4.0.0.pre.rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +4 -104
- data/README.md +454 -432
- data/Rakefile +25 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6941 -6721
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3135
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -182
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -46
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -32
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -85
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -103
- data/lib/pdfium.dll +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -537
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +45 -0
- data/vendor/kreuzberg/Cargo.toml +61 -38
- data/vendor/kreuzberg/README.md +230 -221
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -891
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1171
- data/vendor/kreuzberg/src/embeddings.rs +500 -432
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -569
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -562
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -673
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -0
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -328
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -66
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -417
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -161
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +63 -0
- data/vendor/kreuzberg-ffi/README.md +851 -0
- data/vendor/kreuzberg-ffi/build.rs +176 -0
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -0
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +12 -0
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -0
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -0
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -0
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -0
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -0
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -0
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -0
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -0
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -0
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -0
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -0
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -0
- data/vendor/kreuzberg-tesseract/Cargo.toml +48 -0
- data/vendor/kreuzberg-tesseract/LICENSE +22 -0
- data/vendor/kreuzberg-tesseract/README.md +399 -0
- data/vendor/kreuzberg-tesseract/build.rs +1354 -0
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -0
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -0
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -0
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -0
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -0
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -0
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -0
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -0
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -0
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -0
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -0
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -0
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -0
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +44 -81
- data/vendor/rb-sys/bin/release.sh +0 -21
|
@@ -1,1436 +1,1436 @@
|
|
|
1
|
-
//! Comprehensive pipeline integration tests.
|
|
2
|
-
//!
|
|
3
|
-
//! Tests the post-processing pipeline orchestration: ordering, error handling,
|
|
4
|
-
//! metadata/content modifications, and integration with extractors.
|
|
5
|
-
//!
|
|
6
|
-
//! IMPORTANT: These tests use a global registry and must run serially to avoid interference.
|
|
7
|
-
|
|
8
|
-
use async_trait::async_trait;
|
|
9
|
-
use kreuzberg::core::config::{ExtractionConfig, PostProcessorConfig};
|
|
10
|
-
use kreuzberg::core::pipeline::run_pipeline;
|
|
11
|
-
use kreuzberg::plugins::registry::get_post_processor_registry;
|
|
12
|
-
use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
|
|
13
|
-
use kreuzberg::types::{ExtractionResult, Metadata};
|
|
14
|
-
use kreuzberg::{KreuzbergError, Result};
|
|
15
|
-
use serial_test::serial;
|
|
16
|
-
use std::sync::Arc;
|
|
17
|
-
|
|
18
|
-
struct OrderTrackingProcessor {
|
|
19
|
-
name: String,
|
|
20
|
-
stage: ProcessingStage,
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
impl Plugin for OrderTrackingProcessor {
|
|
24
|
-
fn name(&self) -> &str {
|
|
25
|
-
&self.name
|
|
26
|
-
}
|
|
27
|
-
fn version(&self) -> String {
|
|
28
|
-
"1.0.0".to_string()
|
|
29
|
-
}
|
|
30
|
-
fn initialize(&self) -> Result<()> {
|
|
31
|
-
Ok(())
|
|
32
|
-
}
|
|
33
|
-
fn shutdown(&self) -> Result<()> {
|
|
34
|
-
Ok(())
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
#[async_trait]
|
|
39
|
-
impl PostProcessor for OrderTrackingProcessor {
|
|
40
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
41
|
-
result.content.push_str(&format!("[{}]", self.name));
|
|
42
|
-
Ok(())
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
46
|
-
self.stage
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
struct MetadataAddingProcessor {
|
|
51
|
-
name: String,
|
|
52
|
-
key: String,
|
|
53
|
-
value: String,
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
impl Plugin for MetadataAddingProcessor {
|
|
57
|
-
fn name(&self) -> &str {
|
|
58
|
-
&self.name
|
|
59
|
-
}
|
|
60
|
-
fn version(&self) -> String {
|
|
61
|
-
"1.0.0".to_string()
|
|
62
|
-
}
|
|
63
|
-
fn initialize(&self) -> Result<()> {
|
|
64
|
-
Ok(())
|
|
65
|
-
}
|
|
66
|
-
fn shutdown(&self) -> Result<()> {
|
|
67
|
-
Ok(())
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
#[async_trait]
|
|
72
|
-
impl PostProcessor for MetadataAddingProcessor {
|
|
73
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
74
|
-
result
|
|
75
|
-
.metadata
|
|
76
|
-
.additional
|
|
77
|
-
.insert(self.key.clone(), serde_json::json!(self.value));
|
|
78
|
-
Ok(())
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
82
|
-
ProcessingStage::Early
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
struct FailingProcessor {
|
|
87
|
-
name: String,
|
|
88
|
-
error_message: String,
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
impl Plugin for FailingProcessor {
|
|
92
|
-
fn name(&self) -> &str {
|
|
93
|
-
&self.name
|
|
94
|
-
}
|
|
95
|
-
fn version(&self) -> String {
|
|
96
|
-
"1.0.0".to_string()
|
|
97
|
-
}
|
|
98
|
-
fn initialize(&self) -> Result<()> {
|
|
99
|
-
Ok(())
|
|
100
|
-
}
|
|
101
|
-
fn shutdown(&self) -> Result<()> {
|
|
102
|
-
Ok(())
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
#[async_trait]
|
|
107
|
-
impl PostProcessor for FailingProcessor {
|
|
108
|
-
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
109
|
-
Err(KreuzbergError::Plugin {
|
|
110
|
-
message: self.error_message.clone(),
|
|
111
|
-
plugin_name: self.name.clone(),
|
|
112
|
-
})
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
116
|
-
ProcessingStage::Middle
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
fn clear_processor_registry() {
|
|
121
|
-
let registry = get_post_processor_registry();
|
|
122
|
-
let mut reg = registry
|
|
123
|
-
.write()
|
|
124
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
125
|
-
let _ = reg.shutdown_all();
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
#[tokio::test]
|
|
129
|
-
#[serial]
|
|
130
|
-
async fn test_pipeline_empty_no_processors() {
|
|
131
|
-
clear_processor_registry();
|
|
132
|
-
|
|
133
|
-
let result = ExtractionResult {
|
|
134
|
-
content: "original content".to_string(),
|
|
135
|
-
mime_type: "text/plain".to_string(),
|
|
136
|
-
metadata: Metadata::default(),
|
|
137
|
-
tables: vec![],
|
|
138
|
-
detected_languages: None,
|
|
139
|
-
chunks: None,
|
|
140
|
-
images: None,
|
|
141
|
-
pages: None,
|
|
142
|
-
};
|
|
143
|
-
let config = ExtractionConfig::default();
|
|
144
|
-
|
|
145
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
146
|
-
assert_eq!(processed.content, "original content");
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
#[tokio::test]
|
|
150
|
-
#[serial]
|
|
151
|
-
async fn test_pipeline_single_processor_per_stage() {
|
|
152
|
-
clear_processor_registry();
|
|
153
|
-
|
|
154
|
-
let registry = get_post_processor_registry();
|
|
155
|
-
{
|
|
156
|
-
let mut reg = registry
|
|
157
|
-
.write()
|
|
158
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
159
|
-
|
|
160
|
-
let early = Arc::new(OrderTrackingProcessor {
|
|
161
|
-
name: "early".to_string(),
|
|
162
|
-
stage: ProcessingStage::Early,
|
|
163
|
-
});
|
|
164
|
-
let middle = Arc::new(OrderTrackingProcessor {
|
|
165
|
-
name: "middle".to_string(),
|
|
166
|
-
stage: ProcessingStage::Middle,
|
|
167
|
-
});
|
|
168
|
-
let late = Arc::new(OrderTrackingProcessor {
|
|
169
|
-
name: "late".to_string(),
|
|
170
|
-
stage: ProcessingStage::Late,
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
reg.register(early, 50).unwrap();
|
|
174
|
-
reg.register(middle, 50).unwrap();
|
|
175
|
-
reg.register(late, 50).unwrap();
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
let result = ExtractionResult {
|
|
179
|
-
content: "start".to_string(),
|
|
180
|
-
mime_type: "text/plain".to_string(),
|
|
181
|
-
metadata: Metadata::default(),
|
|
182
|
-
tables: vec![],
|
|
183
|
-
detected_languages: None,
|
|
184
|
-
chunks: None,
|
|
185
|
-
images: None,
|
|
186
|
-
pages: None,
|
|
187
|
-
};
|
|
188
|
-
let config = ExtractionConfig::default();
|
|
189
|
-
|
|
190
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
191
|
-
assert_eq!(processed.content, "start[early][middle][late]");
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
#[tokio::test]
|
|
195
|
-
#[serial]
|
|
196
|
-
async fn test_pipeline_multiple_processors_per_stage() {
|
|
197
|
-
clear_processor_registry();
|
|
198
|
-
|
|
199
|
-
let registry = get_post_processor_registry();
|
|
200
|
-
{
|
|
201
|
-
let mut reg = registry
|
|
202
|
-
.write()
|
|
203
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
204
|
-
|
|
205
|
-
let early_high = Arc::new(OrderTrackingProcessor {
|
|
206
|
-
name: "early-high".to_string(),
|
|
207
|
-
stage: ProcessingStage::Early,
|
|
208
|
-
});
|
|
209
|
-
let early_medium = Arc::new(OrderTrackingProcessor {
|
|
210
|
-
name: "early-medium".to_string(),
|
|
211
|
-
stage: ProcessingStage::Early,
|
|
212
|
-
});
|
|
213
|
-
let early_low = Arc::new(OrderTrackingProcessor {
|
|
214
|
-
name: "early-low".to_string(),
|
|
215
|
-
stage: ProcessingStage::Early,
|
|
216
|
-
});
|
|
217
|
-
|
|
218
|
-
reg.register(early_low, 10).unwrap();
|
|
219
|
-
reg.register(early_high, 100).unwrap();
|
|
220
|
-
reg.register(early_medium, 50).unwrap();
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
let result = ExtractionResult {
|
|
224
|
-
content: "start".to_string(),
|
|
225
|
-
mime_type: "text/plain".to_string(),
|
|
226
|
-
metadata: Metadata::default(),
|
|
227
|
-
tables: vec![],
|
|
228
|
-
detected_languages: None,
|
|
229
|
-
chunks: None,
|
|
230
|
-
images: None,
|
|
231
|
-
pages: None,
|
|
232
|
-
};
|
|
233
|
-
let config = ExtractionConfig::default();
|
|
234
|
-
|
|
235
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
236
|
-
assert_eq!(processed.content, "start[early-high][early-medium][early-low]");
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
#[tokio::test]
|
|
240
|
-
#[serial]
|
|
241
|
-
async fn test_pipeline_all_stages_enabled() {
|
|
242
|
-
clear_processor_registry();
|
|
243
|
-
|
|
244
|
-
let registry = get_post_processor_registry();
|
|
245
|
-
{
|
|
246
|
-
let mut reg = registry
|
|
247
|
-
.write()
|
|
248
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
249
|
-
|
|
250
|
-
for stage in [ProcessingStage::Early, ProcessingStage::Middle, ProcessingStage::Late] {
|
|
251
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
252
|
-
name: format!("{:?}", stage),
|
|
253
|
-
stage,
|
|
254
|
-
});
|
|
255
|
-
reg.register(processor, 50).unwrap();
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
let result = ExtractionResult {
|
|
260
|
-
content: "start".to_string(),
|
|
261
|
-
mime_type: "text/plain".to_string(),
|
|
262
|
-
metadata: Metadata::default(),
|
|
263
|
-
tables: vec![],
|
|
264
|
-
detected_languages: None,
|
|
265
|
-
chunks: None,
|
|
266
|
-
images: None,
|
|
267
|
-
pages: None,
|
|
268
|
-
};
|
|
269
|
-
let config = ExtractionConfig::default();
|
|
270
|
-
|
|
271
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
272
|
-
assert_eq!(processed.content, "start[Early][Middle][Late]");
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
#[tokio::test]
|
|
276
|
-
#[serial]
|
|
277
|
-
async fn test_pipeline_postprocessing_disabled() {
|
|
278
|
-
clear_processor_registry();
|
|
279
|
-
|
|
280
|
-
let registry = get_post_processor_registry();
|
|
281
|
-
{
|
|
282
|
-
let mut reg = registry
|
|
283
|
-
.write()
|
|
284
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
285
|
-
|
|
286
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
287
|
-
name: "processor".to_string(),
|
|
288
|
-
stage: ProcessingStage::Early,
|
|
289
|
-
});
|
|
290
|
-
reg.register(processor, 50).unwrap();
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
let result = ExtractionResult {
|
|
294
|
-
content: "start".to_string(),
|
|
295
|
-
mime_type: "text/plain".to_string(),
|
|
296
|
-
metadata: Metadata::default(),
|
|
297
|
-
tables: vec![],
|
|
298
|
-
detected_languages: None,
|
|
299
|
-
chunks: None,
|
|
300
|
-
images: None,
|
|
301
|
-
pages: None,
|
|
302
|
-
};
|
|
303
|
-
let config = ExtractionConfig {
|
|
304
|
-
postprocessor: Some(PostProcessorConfig {
|
|
305
|
-
enabled: false,
|
|
306
|
-
enabled_processors: None,
|
|
307
|
-
disabled_processors: None,
|
|
308
|
-
}),
|
|
309
|
-
..Default::default()
|
|
310
|
-
};
|
|
311
|
-
|
|
312
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
313
|
-
assert_eq!(processed.content, "start");
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
#[tokio::test]
|
|
317
|
-
#[serial]
|
|
318
|
-
async fn test_pipeline_early_stage_runs_first() {
|
|
319
|
-
clear_processor_registry();
|
|
320
|
-
|
|
321
|
-
let registry = get_post_processor_registry();
|
|
322
|
-
{
|
|
323
|
-
let mut reg = registry
|
|
324
|
-
.write()
|
|
325
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
326
|
-
|
|
327
|
-
let late = Arc::new(OrderTrackingProcessor {
|
|
328
|
-
name: "late".to_string(),
|
|
329
|
-
stage: ProcessingStage::Late,
|
|
330
|
-
});
|
|
331
|
-
let early = Arc::new(OrderTrackingProcessor {
|
|
332
|
-
name: "early".to_string(),
|
|
333
|
-
stage: ProcessingStage::Early,
|
|
334
|
-
});
|
|
335
|
-
|
|
336
|
-
reg.register(late, 50).unwrap();
|
|
337
|
-
reg.register(early, 50).unwrap();
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
let result = ExtractionResult {
|
|
341
|
-
content: "start".to_string(),
|
|
342
|
-
mime_type: "text/plain".to_string(),
|
|
343
|
-
metadata: Metadata::default(),
|
|
344
|
-
tables: vec![],
|
|
345
|
-
detected_languages: None,
|
|
346
|
-
chunks: None,
|
|
347
|
-
images: None,
|
|
348
|
-
pages: None,
|
|
349
|
-
};
|
|
350
|
-
let config = ExtractionConfig::default();
|
|
351
|
-
|
|
352
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
353
|
-
assert_eq!(processed.content, "start[early][late]");
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
#[tokio::test]
|
|
357
|
-
#[serial]
|
|
358
|
-
async fn test_pipeline_middle_stage_runs_second() {
|
|
359
|
-
clear_processor_registry();
|
|
360
|
-
|
|
361
|
-
let registry = get_post_processor_registry();
|
|
362
|
-
{
|
|
363
|
-
let mut reg = registry
|
|
364
|
-
.write()
|
|
365
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
366
|
-
|
|
367
|
-
let early = Arc::new(OrderTrackingProcessor {
|
|
368
|
-
name: "early".to_string(),
|
|
369
|
-
stage: ProcessingStage::Early,
|
|
370
|
-
});
|
|
371
|
-
let middle = Arc::new(OrderTrackingProcessor {
|
|
372
|
-
name: "middle".to_string(),
|
|
373
|
-
stage: ProcessingStage::Middle,
|
|
374
|
-
});
|
|
375
|
-
|
|
376
|
-
reg.register(middle, 50).unwrap();
|
|
377
|
-
reg.register(early, 50).unwrap();
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
let result = ExtractionResult {
|
|
381
|
-
content: "start".to_string(),
|
|
382
|
-
mime_type: "text/plain".to_string(),
|
|
383
|
-
metadata: Metadata::default(),
|
|
384
|
-
tables: vec![],
|
|
385
|
-
detected_languages: None,
|
|
386
|
-
chunks: None,
|
|
387
|
-
images: None,
|
|
388
|
-
pages: None,
|
|
389
|
-
};
|
|
390
|
-
let config = ExtractionConfig::default();
|
|
391
|
-
|
|
392
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
393
|
-
assert_eq!(processed.content, "start[early][middle]");
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
#[tokio::test]
|
|
397
|
-
#[serial]
|
|
398
|
-
async fn test_pipeline_late_stage_runs_last() {
|
|
399
|
-
clear_processor_registry();
|
|
400
|
-
|
|
401
|
-
let registry = get_post_processor_registry();
|
|
402
|
-
{
|
|
403
|
-
let mut reg = registry
|
|
404
|
-
.write()
|
|
405
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
406
|
-
|
|
407
|
-
for stage in [ProcessingStage::Late, ProcessingStage::Early, ProcessingStage::Middle] {
|
|
408
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
409
|
-
name: format!("{:?}", stage),
|
|
410
|
-
stage,
|
|
411
|
-
});
|
|
412
|
-
reg.register(processor, 50).unwrap();
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
let result = ExtractionResult {
|
|
417
|
-
content: "start".to_string(),
|
|
418
|
-
mime_type: "text/plain".to_string(),
|
|
419
|
-
metadata: Metadata::default(),
|
|
420
|
-
tables: vec![],
|
|
421
|
-
detected_languages: None,
|
|
422
|
-
chunks: None,
|
|
423
|
-
images: None,
|
|
424
|
-
pages: None,
|
|
425
|
-
};
|
|
426
|
-
let config = ExtractionConfig::default();
|
|
427
|
-
|
|
428
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
429
|
-
assert_eq!(processed.content, "start[Early][Middle][Late]");
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
#[tokio::test]
|
|
433
|
-
#[serial]
|
|
434
|
-
async fn test_pipeline_within_stage_priority_order() {
|
|
435
|
-
clear_processor_registry();
|
|
436
|
-
|
|
437
|
-
let registry = get_post_processor_registry();
|
|
438
|
-
{
|
|
439
|
-
let mut reg = registry
|
|
440
|
-
.write()
|
|
441
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
442
|
-
|
|
443
|
-
for (name, priority) in [("p1", 100), ("p2", 10), ("p3", 50), ("p4", 75)] {
|
|
444
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
445
|
-
name: name.to_string(),
|
|
446
|
-
stage: ProcessingStage::Early,
|
|
447
|
-
});
|
|
448
|
-
reg.register(processor, priority).unwrap();
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
let result = ExtractionResult {
|
|
453
|
-
content: "start".to_string(),
|
|
454
|
-
mime_type: "text/plain".to_string(),
|
|
455
|
-
metadata: Metadata::default(),
|
|
456
|
-
tables: vec![],
|
|
457
|
-
detected_languages: None,
|
|
458
|
-
chunks: None,
|
|
459
|
-
images: None,
|
|
460
|
-
pages: None,
|
|
461
|
-
};
|
|
462
|
-
let config = ExtractionConfig::default();
|
|
463
|
-
|
|
464
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
465
|
-
assert_eq!(processed.content, "start[p1][p4][p3][p2]");
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
#[tokio::test]
|
|
469
|
-
#[serial]
|
|
470
|
-
async fn test_pipeline_cross_stage_data_flow() {
|
|
471
|
-
clear_processor_registry();
|
|
472
|
-
|
|
473
|
-
let registry = get_post_processor_registry();
|
|
474
|
-
{
|
|
475
|
-
let mut reg = registry
|
|
476
|
-
.write()
|
|
477
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
478
|
-
|
|
479
|
-
let early = Arc::new(MetadataAddingProcessor {
|
|
480
|
-
name: "early".to_string(),
|
|
481
|
-
key: "stage".to_string(),
|
|
482
|
-
value: "early".to_string(),
|
|
483
|
-
});
|
|
484
|
-
|
|
485
|
-
struct MiddleProcessor;
|
|
486
|
-
impl Plugin for MiddleProcessor {
|
|
487
|
-
fn name(&self) -> &str {
|
|
488
|
-
"middle"
|
|
489
|
-
}
|
|
490
|
-
fn version(&self) -> String {
|
|
491
|
-
"1.0.0".to_string()
|
|
492
|
-
}
|
|
493
|
-
fn initialize(&self) -> Result<()> {
|
|
494
|
-
Ok(())
|
|
495
|
-
}
|
|
496
|
-
fn shutdown(&self) -> Result<()> {
|
|
497
|
-
Ok(())
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
#[async_trait]
|
|
501
|
-
impl PostProcessor for MiddleProcessor {
|
|
502
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
503
|
-
if let Some(stage) = result.metadata.additional.get("stage") {
|
|
504
|
-
result.content.push_str(&format!("[saw:{}]", stage.as_str().unwrap()));
|
|
505
|
-
}
|
|
506
|
-
Ok(())
|
|
507
|
-
}
|
|
508
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
509
|
-
ProcessingStage::Middle
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
reg.register(early, 50).unwrap();
|
|
514
|
-
reg.register(Arc::new(MiddleProcessor), 50).unwrap();
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
let result = ExtractionResult {
|
|
518
|
-
content: "start".to_string(),
|
|
519
|
-
mime_type: "text/plain".to_string(),
|
|
520
|
-
metadata: Metadata::default(),
|
|
521
|
-
tables: vec![],
|
|
522
|
-
detected_languages: None,
|
|
523
|
-
chunks: None,
|
|
524
|
-
images: None,
|
|
525
|
-
pages: None,
|
|
526
|
-
};
|
|
527
|
-
let config = ExtractionConfig::default();
|
|
528
|
-
|
|
529
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
530
|
-
assert!(processed.content.contains("[saw:early]"));
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
#[tokio::test]
|
|
534
|
-
#[serial]
|
|
535
|
-
async fn test_pipeline_early_stage_error_recorded() {
|
|
536
|
-
clear_processor_registry();
|
|
537
|
-
|
|
538
|
-
struct EarlyFailingProcessor;
|
|
539
|
-
impl Plugin for EarlyFailingProcessor {
|
|
540
|
-
fn name(&self) -> &str {
|
|
541
|
-
"early-failing"
|
|
542
|
-
}
|
|
543
|
-
fn version(&self) -> String {
|
|
544
|
-
"1.0.0".to_string()
|
|
545
|
-
}
|
|
546
|
-
fn initialize(&self) -> Result<()> {
|
|
547
|
-
Ok(())
|
|
548
|
-
}
|
|
549
|
-
fn shutdown(&self) -> Result<()> {
|
|
550
|
-
Ok(())
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
#[async_trait]
|
|
554
|
-
impl PostProcessor for EarlyFailingProcessor {
|
|
555
|
-
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
556
|
-
Err(KreuzbergError::Plugin {
|
|
557
|
-
message: "Early error".to_string(),
|
|
558
|
-
plugin_name: "early-failing".to_string(),
|
|
559
|
-
})
|
|
560
|
-
}
|
|
561
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
562
|
-
ProcessingStage::Early
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
let registry = get_post_processor_registry();
|
|
567
|
-
{
|
|
568
|
-
let mut reg = registry
|
|
569
|
-
.write()
|
|
570
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
571
|
-
reg.register(Arc::new(EarlyFailingProcessor), 50).unwrap();
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
let result = ExtractionResult {
|
|
575
|
-
content: "content".to_string(),
|
|
576
|
-
mime_type: "text/plain".to_string(),
|
|
577
|
-
metadata: Metadata::default(),
|
|
578
|
-
tables: vec![],
|
|
579
|
-
detected_languages: None,
|
|
580
|
-
chunks: None,
|
|
581
|
-
images: None,
|
|
582
|
-
pages: None,
|
|
583
|
-
};
|
|
584
|
-
let config = ExtractionConfig::default();
|
|
585
|
-
|
|
586
|
-
let result = run_pipeline(result, &config).await;
|
|
587
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
588
|
-
match result {
|
|
589
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
590
|
-
assert_eq!(message, "Early error");
|
|
591
|
-
assert_eq!(plugin_name, "early-failing");
|
|
592
|
-
}
|
|
593
|
-
_ => panic!("Expected Plugin error"),
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
#[tokio::test]
|
|
598
|
-
#[serial]
|
|
599
|
-
async fn test_pipeline_middle_stage_error_propagation() {
|
|
600
|
-
clear_processor_registry();
|
|
601
|
-
|
|
602
|
-
let registry = get_post_processor_registry();
|
|
603
|
-
{
|
|
604
|
-
let mut reg = registry
|
|
605
|
-
.write()
|
|
606
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
607
|
-
|
|
608
|
-
let failing = Arc::new(FailingProcessor {
|
|
609
|
-
name: "middle-failing".to_string(),
|
|
610
|
-
error_message: "Middle stage error".to_string(),
|
|
611
|
-
});
|
|
612
|
-
|
|
613
|
-
reg.register(failing, 50).unwrap();
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
let result = ExtractionResult {
|
|
617
|
-
content: "content".to_string(),
|
|
618
|
-
mime_type: "text/plain".to_string(),
|
|
619
|
-
metadata: Metadata::default(),
|
|
620
|
-
tables: vec![],
|
|
621
|
-
detected_languages: None,
|
|
622
|
-
chunks: None,
|
|
623
|
-
images: None,
|
|
624
|
-
pages: None,
|
|
625
|
-
};
|
|
626
|
-
let config = ExtractionConfig::default();
|
|
627
|
-
|
|
628
|
-
let result = run_pipeline(result, &config).await;
|
|
629
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
630
|
-
match result {
|
|
631
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
632
|
-
assert_eq!(message, "Middle stage error");
|
|
633
|
-
assert_eq!(plugin_name, "middle-failing");
|
|
634
|
-
}
|
|
635
|
-
_ => panic!("Expected Plugin error"),
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
#[tokio::test]
|
|
640
|
-
#[serial]
|
|
641
|
-
async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
|
|
642
|
-
clear_processor_registry();
|
|
643
|
-
|
|
644
|
-
struct LateFailingProcessor;
|
|
645
|
-
impl Plugin for LateFailingProcessor {
|
|
646
|
-
fn name(&self) -> &str {
|
|
647
|
-
"late-failing"
|
|
648
|
-
}
|
|
649
|
-
fn version(&self) -> String {
|
|
650
|
-
"1.0.0".to_string()
|
|
651
|
-
}
|
|
652
|
-
fn initialize(&self) -> Result<()> {
|
|
653
|
-
Ok(())
|
|
654
|
-
}
|
|
655
|
-
fn shutdown(&self) -> Result<()> {
|
|
656
|
-
Ok(())
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
#[async_trait]
|
|
660
|
-
impl PostProcessor for LateFailingProcessor {
|
|
661
|
-
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
662
|
-
Err(KreuzbergError::Plugin {
|
|
663
|
-
message: "Late error".to_string(),
|
|
664
|
-
plugin_name: "late-failing".to_string(),
|
|
665
|
-
})
|
|
666
|
-
}
|
|
667
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
668
|
-
ProcessingStage::Late
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
let registry = get_post_processor_registry();
|
|
673
|
-
{
|
|
674
|
-
let mut reg = registry
|
|
675
|
-
.write()
|
|
676
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
677
|
-
|
|
678
|
-
let early = Arc::new(OrderTrackingProcessor {
|
|
679
|
-
name: "early".to_string(),
|
|
680
|
-
stage: ProcessingStage::Early,
|
|
681
|
-
});
|
|
682
|
-
let late_failing = Arc::new(LateFailingProcessor);
|
|
683
|
-
|
|
684
|
-
reg.register(early, 50).unwrap();
|
|
685
|
-
reg.register(late_failing, 50).unwrap();
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
let result = ExtractionResult {
|
|
689
|
-
content: "start".to_string(),
|
|
690
|
-
mime_type: "text/plain".to_string(),
|
|
691
|
-
metadata: Metadata::default(),
|
|
692
|
-
tables: vec![],
|
|
693
|
-
detected_languages: None,
|
|
694
|
-
chunks: None,
|
|
695
|
-
images: None,
|
|
696
|
-
pages: None,
|
|
697
|
-
};
|
|
698
|
-
let config = ExtractionConfig::default();
|
|
699
|
-
|
|
700
|
-
let result = run_pipeline(result, &config).await;
|
|
701
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
702
|
-
match result {
|
|
703
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
704
|
-
assert_eq!(message, "Late error");
|
|
705
|
-
assert_eq!(plugin_name, "late-failing");
|
|
706
|
-
}
|
|
707
|
-
_ => panic!("Expected Plugin error"),
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
#[tokio::test]
|
|
712
|
-
#[serial]
|
|
713
|
-
async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
|
|
714
|
-
clear_processor_registry();
|
|
715
|
-
|
|
716
|
-
let registry = get_post_processor_registry();
|
|
717
|
-
{
|
|
718
|
-
let mut reg = registry
|
|
719
|
-
.write()
|
|
720
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
721
|
-
|
|
722
|
-
let p1 = Arc::new(OrderTrackingProcessor {
|
|
723
|
-
name: "p1".to_string(),
|
|
724
|
-
stage: ProcessingStage::Early,
|
|
725
|
-
});
|
|
726
|
-
let _p2_failing = Arc::new(FailingProcessor {
|
|
727
|
-
name: "p2-failing".to_string(),
|
|
728
|
-
error_message: "Test error".to_string(),
|
|
729
|
-
});
|
|
730
|
-
let p3 = Arc::new(OrderTrackingProcessor {
|
|
731
|
-
name: "p3".to_string(),
|
|
732
|
-
stage: ProcessingStage::Late,
|
|
733
|
-
});
|
|
734
|
-
|
|
735
|
-
struct EarlyFailingProcessor {
|
|
736
|
-
name: String,
|
|
737
|
-
}
|
|
738
|
-
impl Plugin for EarlyFailingProcessor {
|
|
739
|
-
fn name(&self) -> &str {
|
|
740
|
-
&self.name
|
|
741
|
-
}
|
|
742
|
-
fn version(&self) -> String {
|
|
743
|
-
"1.0.0".to_string()
|
|
744
|
-
}
|
|
745
|
-
fn initialize(&self) -> Result<()> {
|
|
746
|
-
Ok(())
|
|
747
|
-
}
|
|
748
|
-
fn shutdown(&self) -> Result<()> {
|
|
749
|
-
Ok(())
|
|
750
|
-
}
|
|
751
|
-
}
|
|
752
|
-
#[async_trait]
|
|
753
|
-
impl PostProcessor for EarlyFailingProcessor {
|
|
754
|
-
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
755
|
-
Err(KreuzbergError::Plugin {
|
|
756
|
-
message: "Test error".to_string(),
|
|
757
|
-
plugin_name: self.name.clone(),
|
|
758
|
-
})
|
|
759
|
-
}
|
|
760
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
761
|
-
ProcessingStage::Early
|
|
762
|
-
}
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
reg.register(p1, 100).unwrap();
|
|
766
|
-
reg.register(
|
|
767
|
-
Arc::new(EarlyFailingProcessor {
|
|
768
|
-
name: "p2-failing".to_string(),
|
|
769
|
-
}),
|
|
770
|
-
50,
|
|
771
|
-
)
|
|
772
|
-
.unwrap();
|
|
773
|
-
reg.register(p3, 50).unwrap();
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
let result = ExtractionResult {
|
|
777
|
-
content: "start".to_string(),
|
|
778
|
-
mime_type: "text/plain".to_string(),
|
|
779
|
-
metadata: Metadata::default(),
|
|
780
|
-
tables: vec![],
|
|
781
|
-
detected_languages: None,
|
|
782
|
-
chunks: None,
|
|
783
|
-
images: None,
|
|
784
|
-
pages: None,
|
|
785
|
-
};
|
|
786
|
-
let config = ExtractionConfig::default();
|
|
787
|
-
|
|
788
|
-
let result = run_pipeline(result, &config).await;
|
|
789
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
790
|
-
match result {
|
|
791
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
792
|
-
assert_eq!(message, "Test error");
|
|
793
|
-
assert_eq!(plugin_name, "p2-failing");
|
|
794
|
-
}
|
|
795
|
-
_ => panic!("Expected Plugin error"),
|
|
796
|
-
}
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
#[tokio::test]
|
|
800
|
-
#[serial]
|
|
801
|
-
async fn test_pipeline_multiple_processor_errors() {
|
|
802
|
-
clear_processor_registry();
|
|
803
|
-
|
|
804
|
-
struct MultiFailingProcessor {
|
|
805
|
-
name: String,
|
|
806
|
-
stage: ProcessingStage,
|
|
807
|
-
}
|
|
808
|
-
impl Plugin for MultiFailingProcessor {
|
|
809
|
-
fn name(&self) -> &str {
|
|
810
|
-
&self.name
|
|
811
|
-
}
|
|
812
|
-
fn version(&self) -> String {
|
|
813
|
-
"1.0.0".to_string()
|
|
814
|
-
}
|
|
815
|
-
fn initialize(&self) -> Result<()> {
|
|
816
|
-
Ok(())
|
|
817
|
-
}
|
|
818
|
-
fn shutdown(&self) -> Result<()> {
|
|
819
|
-
Ok(())
|
|
820
|
-
}
|
|
821
|
-
}
|
|
822
|
-
#[async_trait]
|
|
823
|
-
impl PostProcessor for MultiFailingProcessor {
|
|
824
|
-
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
825
|
-
Err(KreuzbergError::Plugin {
|
|
826
|
-
message: format!("{} error", self.name),
|
|
827
|
-
plugin_name: self.name.clone(),
|
|
828
|
-
})
|
|
829
|
-
}
|
|
830
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
831
|
-
self.stage
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
let registry = get_post_processor_registry();
|
|
836
|
-
{
|
|
837
|
-
let mut reg = registry
|
|
838
|
-
.write()
|
|
839
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
840
|
-
|
|
841
|
-
for (name, stage) in [
|
|
842
|
-
("fail1", ProcessingStage::Early),
|
|
843
|
-
("fail2", ProcessingStage::Middle),
|
|
844
|
-
("fail3", ProcessingStage::Late),
|
|
845
|
-
] {
|
|
846
|
-
let processor = Arc::new(MultiFailingProcessor {
|
|
847
|
-
name: name.to_string(),
|
|
848
|
-
stage,
|
|
849
|
-
});
|
|
850
|
-
reg.register(processor, 50).unwrap();
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
let result = ExtractionResult {
|
|
855
|
-
content: "start".to_string(),
|
|
856
|
-
mime_type: "text/plain".to_string(),
|
|
857
|
-
metadata: Metadata::default(),
|
|
858
|
-
tables: vec![],
|
|
859
|
-
detected_languages: None,
|
|
860
|
-
chunks: None,
|
|
861
|
-
images: None,
|
|
862
|
-
pages: None,
|
|
863
|
-
};
|
|
864
|
-
let config = ExtractionConfig::default();
|
|
865
|
-
|
|
866
|
-
let result = run_pipeline(result, &config).await;
|
|
867
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
868
|
-
match result {
|
|
869
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
870
|
-
assert_eq!(message, "fail1 error");
|
|
871
|
-
assert_eq!(plugin_name, "fail1");
|
|
872
|
-
}
|
|
873
|
-
_ => panic!("Expected Plugin error"),
|
|
874
|
-
}
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
#[tokio::test]
|
|
878
|
-
#[serial]
|
|
879
|
-
async fn test_pipeline_error_context_preservation() {
|
|
880
|
-
clear_processor_registry();
|
|
881
|
-
|
|
882
|
-
let registry = get_post_processor_registry();
|
|
883
|
-
{
|
|
884
|
-
let mut reg = registry
|
|
885
|
-
.write()
|
|
886
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
887
|
-
|
|
888
|
-
let failing = Arc::new(FailingProcessor {
|
|
889
|
-
name: "context-test".to_string(),
|
|
890
|
-
error_message: "Detailed error message with context".to_string(),
|
|
891
|
-
});
|
|
892
|
-
|
|
893
|
-
reg.register(failing, 50).unwrap();
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
let result = ExtractionResult {
|
|
897
|
-
content: "content".to_string(),
|
|
898
|
-
mime_type: "text/plain".to_string(),
|
|
899
|
-
metadata: Metadata::default(),
|
|
900
|
-
tables: vec![],
|
|
901
|
-
detected_languages: None,
|
|
902
|
-
chunks: None,
|
|
903
|
-
images: None,
|
|
904
|
-
pages: None,
|
|
905
|
-
};
|
|
906
|
-
let config = ExtractionConfig::default();
|
|
907
|
-
|
|
908
|
-
let result = run_pipeline(result, &config).await;
|
|
909
|
-
assert!(result.is_err(), "Expected pipeline to return error");
|
|
910
|
-
match result {
|
|
911
|
-
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
912
|
-
assert_eq!(message, "Detailed error message with context");
|
|
913
|
-
assert_eq!(plugin_name, "context-test");
|
|
914
|
-
}
|
|
915
|
-
_ => panic!("Expected Plugin error"),
|
|
916
|
-
}
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
#[tokio::test]
|
|
920
|
-
#[serial]
|
|
921
|
-
async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
|
|
922
|
-
clear_processor_registry();
|
|
923
|
-
|
|
924
|
-
let registry = get_post_processor_registry();
|
|
925
|
-
{
|
|
926
|
-
let mut reg = registry
|
|
927
|
-
.write()
|
|
928
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
929
|
-
|
|
930
|
-
let early = Arc::new(MetadataAddingProcessor {
|
|
931
|
-
name: "early".to_string(),
|
|
932
|
-
key: "early_key".to_string(),
|
|
933
|
-
value: "early_value".to_string(),
|
|
934
|
-
});
|
|
935
|
-
|
|
936
|
-
struct MiddleReadingProcessor;
|
|
937
|
-
impl Plugin for MiddleReadingProcessor {
|
|
938
|
-
fn name(&self) -> &str {
|
|
939
|
-
"middle"
|
|
940
|
-
}
|
|
941
|
-
fn version(&self) -> String {
|
|
942
|
-
"1.0.0".to_string()
|
|
943
|
-
}
|
|
944
|
-
fn initialize(&self) -> Result<()> {
|
|
945
|
-
Ok(())
|
|
946
|
-
}
|
|
947
|
-
fn shutdown(&self) -> Result<()> {
|
|
948
|
-
Ok(())
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
#[async_trait]
|
|
952
|
-
impl PostProcessor for MiddleReadingProcessor {
|
|
953
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
954
|
-
if let Some(val) = result.metadata.additional.get("early_key") {
|
|
955
|
-
result.metadata.additional.insert("middle_saw".to_string(), val.clone());
|
|
956
|
-
}
|
|
957
|
-
Ok(())
|
|
958
|
-
}
|
|
959
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
960
|
-
ProcessingStage::Middle
|
|
961
|
-
}
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
reg.register(early, 50).unwrap();
|
|
965
|
-
reg.register(Arc::new(MiddleReadingProcessor), 50).unwrap();
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
let result = ExtractionResult {
|
|
969
|
-
content: "content".to_string(),
|
|
970
|
-
mime_type: "text/plain".to_string(),
|
|
971
|
-
metadata: Metadata::default(),
|
|
972
|
-
tables: vec![],
|
|
973
|
-
detected_languages: None,
|
|
974
|
-
chunks: None,
|
|
975
|
-
images: None,
|
|
976
|
-
pages: None,
|
|
977
|
-
};
|
|
978
|
-
let config = ExtractionConfig::default();
|
|
979
|
-
|
|
980
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
981
|
-
assert_eq!(
|
|
982
|
-
processed
|
|
983
|
-
.metadata
|
|
984
|
-
.additional
|
|
985
|
-
.get("middle_saw")
|
|
986
|
-
.unwrap()
|
|
987
|
-
.as_str()
|
|
988
|
-
.unwrap(),
|
|
989
|
-
"early_value"
|
|
990
|
-
);
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
#[tokio::test]
|
|
994
|
-
#[serial]
|
|
995
|
-
async fn test_pipeline_content_modified_in_middle_visible_in_late() {
|
|
996
|
-
clear_processor_registry();
|
|
997
|
-
|
|
998
|
-
let registry = get_post_processor_registry();
|
|
999
|
-
{
|
|
1000
|
-
let mut reg = registry
|
|
1001
|
-
.write()
|
|
1002
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1003
|
-
|
|
1004
|
-
let middle = Arc::new(OrderTrackingProcessor {
|
|
1005
|
-
name: "middle-content".to_string(),
|
|
1006
|
-
stage: ProcessingStage::Middle,
|
|
1007
|
-
});
|
|
1008
|
-
|
|
1009
|
-
struct LateReadingProcessor;
|
|
1010
|
-
impl Plugin for LateReadingProcessor {
|
|
1011
|
-
fn name(&self) -> &str {
|
|
1012
|
-
"late"
|
|
1013
|
-
}
|
|
1014
|
-
fn version(&self) -> String {
|
|
1015
|
-
"1.0.0".to_string()
|
|
1016
|
-
}
|
|
1017
|
-
fn initialize(&self) -> Result<()> {
|
|
1018
|
-
Ok(())
|
|
1019
|
-
}
|
|
1020
|
-
fn shutdown(&self) -> Result<()> {
|
|
1021
|
-
Ok(())
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
|
-
#[async_trait]
|
|
1025
|
-
impl PostProcessor for LateReadingProcessor {
|
|
1026
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1027
|
-
result.content.push_str("[late-saw-middle]");
|
|
1028
|
-
Ok(())
|
|
1029
|
-
}
|
|
1030
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
1031
|
-
ProcessingStage::Late
|
|
1032
|
-
}
|
|
1033
|
-
}
|
|
1034
|
-
|
|
1035
|
-
reg.register(middle, 50).unwrap();
|
|
1036
|
-
reg.register(Arc::new(LateReadingProcessor), 50).unwrap();
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
let result = ExtractionResult {
|
|
1040
|
-
content: "start".to_string(),
|
|
1041
|
-
mime_type: "text/plain".to_string(),
|
|
1042
|
-
metadata: Metadata::default(),
|
|
1043
|
-
tables: vec![],
|
|
1044
|
-
detected_languages: None,
|
|
1045
|
-
chunks: None,
|
|
1046
|
-
images: None,
|
|
1047
|
-
pages: None,
|
|
1048
|
-
};
|
|
1049
|
-
let config = ExtractionConfig::default();
|
|
1050
|
-
|
|
1051
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1052
|
-
assert!(processed.content.contains("[middle-content]"));
|
|
1053
|
-
assert!(processed.content.contains("[late-saw-middle]"));
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
#[tokio::test]
|
|
1057
|
-
#[serial]
|
|
1058
|
-
async fn test_pipeline_multiple_processors_modifying_same_metadata() {
|
|
1059
|
-
clear_processor_registry();
|
|
1060
|
-
|
|
1061
|
-
let registry = get_post_processor_registry();
|
|
1062
|
-
{
|
|
1063
|
-
let mut reg = registry
|
|
1064
|
-
.write()
|
|
1065
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1066
|
-
|
|
1067
|
-
for i in 1..=3 {
|
|
1068
|
-
struct MetadataOverwritingProcessor {
|
|
1069
|
-
name: String,
|
|
1070
|
-
value: String,
|
|
1071
|
-
}
|
|
1072
|
-
impl Plugin for MetadataOverwritingProcessor {
|
|
1073
|
-
fn name(&self) -> &str {
|
|
1074
|
-
&self.name
|
|
1075
|
-
}
|
|
1076
|
-
fn version(&self) -> String {
|
|
1077
|
-
"1.0.0".to_string()
|
|
1078
|
-
}
|
|
1079
|
-
fn initialize(&self) -> Result<()> {
|
|
1080
|
-
Ok(())
|
|
1081
|
-
}
|
|
1082
|
-
fn shutdown(&self) -> Result<()> {
|
|
1083
|
-
Ok(())
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
#[async_trait]
|
|
1087
|
-
impl PostProcessor for MetadataOverwritingProcessor {
|
|
1088
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1089
|
-
result
|
|
1090
|
-
.metadata
|
|
1091
|
-
.additional
|
|
1092
|
-
.insert("shared_key".to_string(), serde_json::json!(self.value));
|
|
1093
|
-
Ok(())
|
|
1094
|
-
}
|
|
1095
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
1096
|
-
ProcessingStage::Early
|
|
1097
|
-
}
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
let processor = Arc::new(MetadataOverwritingProcessor {
|
|
1101
|
-
name: format!("proc{}", i),
|
|
1102
|
-
value: format!("value{}", i),
|
|
1103
|
-
});
|
|
1104
|
-
reg.register(processor, 100 - i * 10).unwrap();
|
|
1105
|
-
}
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
let result = ExtractionResult {
|
|
1109
|
-
content: "content".to_string(),
|
|
1110
|
-
mime_type: "text/plain".to_string(),
|
|
1111
|
-
metadata: Metadata::default(),
|
|
1112
|
-
tables: vec![],
|
|
1113
|
-
detected_languages: None,
|
|
1114
|
-
chunks: None,
|
|
1115
|
-
images: None,
|
|
1116
|
-
pages: None,
|
|
1117
|
-
};
|
|
1118
|
-
let config = ExtractionConfig::default();
|
|
1119
|
-
|
|
1120
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1121
|
-
assert_eq!(
|
|
1122
|
-
processed
|
|
1123
|
-
.metadata
|
|
1124
|
-
.additional
|
|
1125
|
-
.get("shared_key")
|
|
1126
|
-
.unwrap()
|
|
1127
|
-
.as_str()
|
|
1128
|
-
.unwrap(),
|
|
1129
|
-
"value3"
|
|
1130
|
-
);
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
#[tokio::test]
|
|
1134
|
-
#[serial]
|
|
1135
|
-
async fn test_pipeline_processors_reading_previous_output() {
|
|
1136
|
-
clear_processor_registry();
|
|
1137
|
-
|
|
1138
|
-
let registry = get_post_processor_registry();
|
|
1139
|
-
{
|
|
1140
|
-
let mut reg = registry
|
|
1141
|
-
.write()
|
|
1142
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1143
|
-
|
|
1144
|
-
struct CountingProcessor {
|
|
1145
|
-
name: String,
|
|
1146
|
-
stage: ProcessingStage,
|
|
1147
|
-
}
|
|
1148
|
-
impl Plugin for CountingProcessor {
|
|
1149
|
-
fn name(&self) -> &str {
|
|
1150
|
-
&self.name
|
|
1151
|
-
}
|
|
1152
|
-
fn version(&self) -> String {
|
|
1153
|
-
"1.0.0".to_string()
|
|
1154
|
-
}
|
|
1155
|
-
fn initialize(&self) -> Result<()> {
|
|
1156
|
-
Ok(())
|
|
1157
|
-
}
|
|
1158
|
-
fn shutdown(&self) -> Result<()> {
|
|
1159
|
-
Ok(())
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
#[async_trait]
|
|
1163
|
-
impl PostProcessor for CountingProcessor {
|
|
1164
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1165
|
-
let current_count = result
|
|
1166
|
-
.metadata
|
|
1167
|
-
.additional
|
|
1168
|
-
.get("count")
|
|
1169
|
-
.and_then(|v| v.as_i64())
|
|
1170
|
-
.unwrap_or(0);
|
|
1171
|
-
result
|
|
1172
|
-
.metadata
|
|
1173
|
-
.additional
|
|
1174
|
-
.insert("count".to_string(), serde_json::json!(current_count + 1));
|
|
1175
|
-
Ok(())
|
|
1176
|
-
}
|
|
1177
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
1178
|
-
self.stage
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
for (name, stage) in [
|
|
1183
|
-
("early1", ProcessingStage::Early),
|
|
1184
|
-
("early2", ProcessingStage::Early),
|
|
1185
|
-
("middle1", ProcessingStage::Middle),
|
|
1186
|
-
("late1", ProcessingStage::Late),
|
|
1187
|
-
] {
|
|
1188
|
-
let processor = Arc::new(CountingProcessor {
|
|
1189
|
-
name: name.to_string(),
|
|
1190
|
-
stage,
|
|
1191
|
-
});
|
|
1192
|
-
reg.register(processor, 50).unwrap();
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
let result = ExtractionResult {
|
|
1197
|
-
content: "content".to_string(),
|
|
1198
|
-
mime_type: "text/plain".to_string(),
|
|
1199
|
-
metadata: Metadata::default(),
|
|
1200
|
-
tables: vec![],
|
|
1201
|
-
detected_languages: None,
|
|
1202
|
-
chunks: None,
|
|
1203
|
-
images: None,
|
|
1204
|
-
pages: None,
|
|
1205
|
-
};
|
|
1206
|
-
let config = ExtractionConfig::default();
|
|
1207
|
-
|
|
1208
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1209
|
-
assert_eq!(processed.metadata.additional.get("count").unwrap().as_i64().unwrap(), 4);
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
#[tokio::test]
|
|
1213
|
-
#[serial]
|
|
1214
|
-
async fn test_pipeline_large_content_modification() {
|
|
1215
|
-
clear_processor_registry();
|
|
1216
|
-
|
|
1217
|
-
struct LargeContentProcessor;
|
|
1218
|
-
impl Plugin for LargeContentProcessor {
|
|
1219
|
-
fn name(&self) -> &str {
|
|
1220
|
-
"large"
|
|
1221
|
-
}
|
|
1222
|
-
fn version(&self) -> String {
|
|
1223
|
-
"1.0.0".to_string()
|
|
1224
|
-
}
|
|
1225
|
-
fn initialize(&self) -> Result<()> {
|
|
1226
|
-
Ok(())
|
|
1227
|
-
}
|
|
1228
|
-
fn shutdown(&self) -> Result<()> {
|
|
1229
|
-
Ok(())
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
#[async_trait]
|
|
1233
|
-
impl PostProcessor for LargeContentProcessor {
|
|
1234
|
-
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1235
|
-
result.content.push_str(&"x".repeat(10000));
|
|
1236
|
-
Ok(())
|
|
1237
|
-
}
|
|
1238
|
-
fn processing_stage(&self) -> ProcessingStage {
|
|
1239
|
-
ProcessingStage::Early
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
let registry = get_post_processor_registry();
|
|
1244
|
-
{
|
|
1245
|
-
let mut reg = registry
|
|
1246
|
-
.write()
|
|
1247
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1248
|
-
reg.register(Arc::new(LargeContentProcessor), 50).unwrap();
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
let result = ExtractionResult {
|
|
1252
|
-
content: "start".to_string(),
|
|
1253
|
-
mime_type: "text/plain".to_string(),
|
|
1254
|
-
metadata: Metadata::default(),
|
|
1255
|
-
tables: vec![],
|
|
1256
|
-
detected_languages: None,
|
|
1257
|
-
chunks: None,
|
|
1258
|
-
images: None,
|
|
1259
|
-
pages: None,
|
|
1260
|
-
};
|
|
1261
|
-
let config = ExtractionConfig::default();
|
|
1262
|
-
|
|
1263
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1264
|
-
assert!(processed.content.len() > 10000);
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
#[tokio::test]
|
|
1268
|
-
#[serial]
|
|
1269
|
-
async fn test_pipeline_enabled_processors_whitelist() {
|
|
1270
|
-
clear_processor_registry();
|
|
1271
|
-
|
|
1272
|
-
let registry = get_post_processor_registry();
|
|
1273
|
-
{
|
|
1274
|
-
let mut reg = registry
|
|
1275
|
-
.write()
|
|
1276
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1277
|
-
|
|
1278
|
-
for name in ["proc1", "proc2", "proc3"] {
|
|
1279
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
1280
|
-
name: name.to_string(),
|
|
1281
|
-
stage: ProcessingStage::Early,
|
|
1282
|
-
});
|
|
1283
|
-
reg.register(processor, 50).unwrap();
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
|
-
|
|
1287
|
-
let result = ExtractionResult {
|
|
1288
|
-
content: "start".to_string(),
|
|
1289
|
-
mime_type: "text/plain".to_string(),
|
|
1290
|
-
metadata: Metadata::default(),
|
|
1291
|
-
tables: vec![],
|
|
1292
|
-
detected_languages: None,
|
|
1293
|
-
chunks: None,
|
|
1294
|
-
images: None,
|
|
1295
|
-
pages: None,
|
|
1296
|
-
};
|
|
1297
|
-
let config = ExtractionConfig {
|
|
1298
|
-
postprocessor: Some(PostProcessorConfig {
|
|
1299
|
-
enabled: true,
|
|
1300
|
-
enabled_processors: Some(vec!["proc1".to_string(), "proc3".to_string()]),
|
|
1301
|
-
disabled_processors: None,
|
|
1302
|
-
}),
|
|
1303
|
-
..Default::default()
|
|
1304
|
-
};
|
|
1305
|
-
|
|
1306
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1307
|
-
assert!(processed.content.contains("[proc1]"));
|
|
1308
|
-
assert!(!processed.content.contains("[proc2]"));
|
|
1309
|
-
assert!(processed.content.contains("[proc3]"));
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
#[tokio::test]
|
|
1313
|
-
#[serial]
|
|
1314
|
-
async fn test_pipeline_disabled_processors_blacklist() {
|
|
1315
|
-
clear_processor_registry();
|
|
1316
|
-
|
|
1317
|
-
let registry = get_post_processor_registry();
|
|
1318
|
-
{
|
|
1319
|
-
let mut reg = registry
|
|
1320
|
-
.write()
|
|
1321
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1322
|
-
|
|
1323
|
-
for name in ["proc1", "proc2", "proc3"] {
|
|
1324
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
1325
|
-
name: name.to_string(),
|
|
1326
|
-
stage: ProcessingStage::Early,
|
|
1327
|
-
});
|
|
1328
|
-
reg.register(processor, 50).unwrap();
|
|
1329
|
-
}
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
let result = ExtractionResult {
|
|
1333
|
-
content: "start".to_string(),
|
|
1334
|
-
mime_type: "text/plain".to_string(),
|
|
1335
|
-
metadata: Metadata::default(),
|
|
1336
|
-
tables: vec![],
|
|
1337
|
-
detected_languages: None,
|
|
1338
|
-
chunks: None,
|
|
1339
|
-
images: None,
|
|
1340
|
-
pages: None,
|
|
1341
|
-
};
|
|
1342
|
-
let config = ExtractionConfig {
|
|
1343
|
-
postprocessor: Some(PostProcessorConfig {
|
|
1344
|
-
enabled: true,
|
|
1345
|
-
enabled_processors: None,
|
|
1346
|
-
disabled_processors: Some(vec!["proc2".to_string()]),
|
|
1347
|
-
}),
|
|
1348
|
-
..Default::default()
|
|
1349
|
-
};
|
|
1350
|
-
|
|
1351
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1352
|
-
assert!(processed.content.contains("[proc1]"));
|
|
1353
|
-
assert!(!processed.content.contains("[proc2]"));
|
|
1354
|
-
assert!(processed.content.contains("[proc3]"));
|
|
1355
|
-
}
|
|
1356
|
-
|
|
1357
|
-
#[tokio::test]
|
|
1358
|
-
#[serial]
|
|
1359
|
-
async fn test_pipeline_no_filtering_runs_all() {
|
|
1360
|
-
clear_processor_registry();
|
|
1361
|
-
|
|
1362
|
-
let registry = get_post_processor_registry();
|
|
1363
|
-
{
|
|
1364
|
-
let mut reg = registry
|
|
1365
|
-
.write()
|
|
1366
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1367
|
-
|
|
1368
|
-
for name in ["proc1", "proc2", "proc3"] {
|
|
1369
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
1370
|
-
name: name.to_string(),
|
|
1371
|
-
stage: ProcessingStage::Early,
|
|
1372
|
-
});
|
|
1373
|
-
reg.register(processor, 50).unwrap();
|
|
1374
|
-
}
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
|
-
let result = ExtractionResult {
|
|
1378
|
-
content: "start".to_string(),
|
|
1379
|
-
mime_type: "text/plain".to_string(),
|
|
1380
|
-
metadata: Metadata::default(),
|
|
1381
|
-
tables: vec![],
|
|
1382
|
-
detected_languages: None,
|
|
1383
|
-
chunks: None,
|
|
1384
|
-
images: None,
|
|
1385
|
-
pages: None,
|
|
1386
|
-
};
|
|
1387
|
-
let config = ExtractionConfig::default();
|
|
1388
|
-
|
|
1389
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1390
|
-
assert!(processed.content.contains("[proc1]"));
|
|
1391
|
-
assert!(processed.content.contains("[proc2]"));
|
|
1392
|
-
assert!(processed.content.contains("[proc3]"));
|
|
1393
|
-
}
|
|
1394
|
-
|
|
1395
|
-
#[tokio::test]
|
|
1396
|
-
#[serial]
|
|
1397
|
-
async fn test_pipeline_empty_whitelist_runs_none() {
|
|
1398
|
-
clear_processor_registry();
|
|
1399
|
-
|
|
1400
|
-
let registry = get_post_processor_registry();
|
|
1401
|
-
{
|
|
1402
|
-
let mut reg = registry
|
|
1403
|
-
.write()
|
|
1404
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
1405
|
-
|
|
1406
|
-
for name in ["proc1", "proc2"] {
|
|
1407
|
-
let processor = Arc::new(OrderTrackingProcessor {
|
|
1408
|
-
name: name.to_string(),
|
|
1409
|
-
stage: ProcessingStage::Early,
|
|
1410
|
-
});
|
|
1411
|
-
reg.register(processor, 50).unwrap();
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
let result = ExtractionResult {
|
|
1416
|
-
content: "start".to_string(),
|
|
1417
|
-
mime_type: "text/plain".to_string(),
|
|
1418
|
-
metadata: Metadata::default(),
|
|
1419
|
-
tables: vec![],
|
|
1420
|
-
detected_languages: None,
|
|
1421
|
-
chunks: None,
|
|
1422
|
-
images: None,
|
|
1423
|
-
pages: None,
|
|
1424
|
-
};
|
|
1425
|
-
let config = ExtractionConfig {
|
|
1426
|
-
postprocessor: Some(PostProcessorConfig {
|
|
1427
|
-
enabled: true,
|
|
1428
|
-
enabled_processors: Some(vec![]),
|
|
1429
|
-
disabled_processors: None,
|
|
1430
|
-
}),
|
|
1431
|
-
..Default::default()
|
|
1432
|
-
};
|
|
1433
|
-
|
|
1434
|
-
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1435
|
-
assert_eq!(processed.content, "start");
|
|
1436
|
-
}
|
|
1
|
+
//! Comprehensive pipeline integration tests.
|
|
2
|
+
//!
|
|
3
|
+
//! Tests the post-processing pipeline orchestration: ordering, error handling,
|
|
4
|
+
//! metadata/content modifications, and integration with extractors.
|
|
5
|
+
//!
|
|
6
|
+
//! IMPORTANT: These tests use a global registry and must run serially to avoid interference.
|
|
7
|
+
|
|
8
|
+
use async_trait::async_trait;
|
|
9
|
+
use kreuzberg::core::config::{ExtractionConfig, PostProcessorConfig};
|
|
10
|
+
use kreuzberg::core::pipeline::run_pipeline;
|
|
11
|
+
use kreuzberg::plugins::registry::get_post_processor_registry;
|
|
12
|
+
use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
|
|
13
|
+
use kreuzberg::types::{ExtractionResult, Metadata};
|
|
14
|
+
use kreuzberg::{KreuzbergError, Result};
|
|
15
|
+
use serial_test::serial;
|
|
16
|
+
use std::sync::Arc;
|
|
17
|
+
|
|
18
|
+
struct OrderTrackingProcessor {
|
|
19
|
+
name: String,
|
|
20
|
+
stage: ProcessingStage,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
impl Plugin for OrderTrackingProcessor {
|
|
24
|
+
fn name(&self) -> &str {
|
|
25
|
+
&self.name
|
|
26
|
+
}
|
|
27
|
+
fn version(&self) -> String {
|
|
28
|
+
"1.0.0".to_string()
|
|
29
|
+
}
|
|
30
|
+
fn initialize(&self) -> Result<()> {
|
|
31
|
+
Ok(())
|
|
32
|
+
}
|
|
33
|
+
fn shutdown(&self) -> Result<()> {
|
|
34
|
+
Ok(())
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
#[async_trait]
|
|
39
|
+
impl PostProcessor for OrderTrackingProcessor {
|
|
40
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
41
|
+
result.content.push_str(&format!("[{}]", self.name));
|
|
42
|
+
Ok(())
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
46
|
+
self.stage
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
struct MetadataAddingProcessor {
|
|
51
|
+
name: String,
|
|
52
|
+
key: String,
|
|
53
|
+
value: String,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
impl Plugin for MetadataAddingProcessor {
|
|
57
|
+
fn name(&self) -> &str {
|
|
58
|
+
&self.name
|
|
59
|
+
}
|
|
60
|
+
fn version(&self) -> String {
|
|
61
|
+
"1.0.0".to_string()
|
|
62
|
+
}
|
|
63
|
+
fn initialize(&self) -> Result<()> {
|
|
64
|
+
Ok(())
|
|
65
|
+
}
|
|
66
|
+
fn shutdown(&self) -> Result<()> {
|
|
67
|
+
Ok(())
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#[async_trait]
|
|
72
|
+
impl PostProcessor for MetadataAddingProcessor {
|
|
73
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
74
|
+
result
|
|
75
|
+
.metadata
|
|
76
|
+
.additional
|
|
77
|
+
.insert(self.key.clone(), serde_json::json!(self.value));
|
|
78
|
+
Ok(())
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
82
|
+
ProcessingStage::Early
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
struct FailingProcessor {
|
|
87
|
+
name: String,
|
|
88
|
+
error_message: String,
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
impl Plugin for FailingProcessor {
|
|
92
|
+
fn name(&self) -> &str {
|
|
93
|
+
&self.name
|
|
94
|
+
}
|
|
95
|
+
fn version(&self) -> String {
|
|
96
|
+
"1.0.0".to_string()
|
|
97
|
+
}
|
|
98
|
+
fn initialize(&self) -> Result<()> {
|
|
99
|
+
Ok(())
|
|
100
|
+
}
|
|
101
|
+
fn shutdown(&self) -> Result<()> {
|
|
102
|
+
Ok(())
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
#[async_trait]
|
|
107
|
+
impl PostProcessor for FailingProcessor {
|
|
108
|
+
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
109
|
+
Err(KreuzbergError::Plugin {
|
|
110
|
+
message: self.error_message.clone(),
|
|
111
|
+
plugin_name: self.name.clone(),
|
|
112
|
+
})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
116
|
+
ProcessingStage::Middle
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
fn clear_processor_registry() {
|
|
121
|
+
let registry = get_post_processor_registry();
|
|
122
|
+
let mut reg = registry
|
|
123
|
+
.write()
|
|
124
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
125
|
+
let _ = reg.shutdown_all();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
#[tokio::test]
|
|
129
|
+
#[serial]
|
|
130
|
+
async fn test_pipeline_empty_no_processors() {
|
|
131
|
+
clear_processor_registry();
|
|
132
|
+
|
|
133
|
+
let result = ExtractionResult {
|
|
134
|
+
content: "original content".to_string(),
|
|
135
|
+
mime_type: "text/plain".to_string(),
|
|
136
|
+
metadata: Metadata::default(),
|
|
137
|
+
tables: vec![],
|
|
138
|
+
detected_languages: None,
|
|
139
|
+
chunks: None,
|
|
140
|
+
images: None,
|
|
141
|
+
pages: None,
|
|
142
|
+
};
|
|
143
|
+
let config = ExtractionConfig::default();
|
|
144
|
+
|
|
145
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
146
|
+
assert_eq!(processed.content, "original content");
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
#[tokio::test]
|
|
150
|
+
#[serial]
|
|
151
|
+
async fn test_pipeline_single_processor_per_stage() {
|
|
152
|
+
clear_processor_registry();
|
|
153
|
+
|
|
154
|
+
let registry = get_post_processor_registry();
|
|
155
|
+
{
|
|
156
|
+
let mut reg = registry
|
|
157
|
+
.write()
|
|
158
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
159
|
+
|
|
160
|
+
let early = Arc::new(OrderTrackingProcessor {
|
|
161
|
+
name: "early".to_string(),
|
|
162
|
+
stage: ProcessingStage::Early,
|
|
163
|
+
});
|
|
164
|
+
let middle = Arc::new(OrderTrackingProcessor {
|
|
165
|
+
name: "middle".to_string(),
|
|
166
|
+
stage: ProcessingStage::Middle,
|
|
167
|
+
});
|
|
168
|
+
let late = Arc::new(OrderTrackingProcessor {
|
|
169
|
+
name: "late".to_string(),
|
|
170
|
+
stage: ProcessingStage::Late,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
reg.register(early, 50).unwrap();
|
|
174
|
+
reg.register(middle, 50).unwrap();
|
|
175
|
+
reg.register(late, 50).unwrap();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
let result = ExtractionResult {
|
|
179
|
+
content: "start".to_string(),
|
|
180
|
+
mime_type: "text/plain".to_string(),
|
|
181
|
+
metadata: Metadata::default(),
|
|
182
|
+
tables: vec![],
|
|
183
|
+
detected_languages: None,
|
|
184
|
+
chunks: None,
|
|
185
|
+
images: None,
|
|
186
|
+
pages: None,
|
|
187
|
+
};
|
|
188
|
+
let config = ExtractionConfig::default();
|
|
189
|
+
|
|
190
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
191
|
+
assert_eq!(processed.content, "start[early][middle][late]");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
#[tokio::test]
|
|
195
|
+
#[serial]
|
|
196
|
+
async fn test_pipeline_multiple_processors_per_stage() {
|
|
197
|
+
clear_processor_registry();
|
|
198
|
+
|
|
199
|
+
let registry = get_post_processor_registry();
|
|
200
|
+
{
|
|
201
|
+
let mut reg = registry
|
|
202
|
+
.write()
|
|
203
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
204
|
+
|
|
205
|
+
let early_high = Arc::new(OrderTrackingProcessor {
|
|
206
|
+
name: "early-high".to_string(),
|
|
207
|
+
stage: ProcessingStage::Early,
|
|
208
|
+
});
|
|
209
|
+
let early_medium = Arc::new(OrderTrackingProcessor {
|
|
210
|
+
name: "early-medium".to_string(),
|
|
211
|
+
stage: ProcessingStage::Early,
|
|
212
|
+
});
|
|
213
|
+
let early_low = Arc::new(OrderTrackingProcessor {
|
|
214
|
+
name: "early-low".to_string(),
|
|
215
|
+
stage: ProcessingStage::Early,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
reg.register(early_low, 10).unwrap();
|
|
219
|
+
reg.register(early_high, 100).unwrap();
|
|
220
|
+
reg.register(early_medium, 50).unwrap();
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
let result = ExtractionResult {
|
|
224
|
+
content: "start".to_string(),
|
|
225
|
+
mime_type: "text/plain".to_string(),
|
|
226
|
+
metadata: Metadata::default(),
|
|
227
|
+
tables: vec![],
|
|
228
|
+
detected_languages: None,
|
|
229
|
+
chunks: None,
|
|
230
|
+
images: None,
|
|
231
|
+
pages: None,
|
|
232
|
+
};
|
|
233
|
+
let config = ExtractionConfig::default();
|
|
234
|
+
|
|
235
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
236
|
+
assert_eq!(processed.content, "start[early-high][early-medium][early-low]");
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
#[tokio::test]
|
|
240
|
+
#[serial]
|
|
241
|
+
async fn test_pipeline_all_stages_enabled() {
|
|
242
|
+
clear_processor_registry();
|
|
243
|
+
|
|
244
|
+
let registry = get_post_processor_registry();
|
|
245
|
+
{
|
|
246
|
+
let mut reg = registry
|
|
247
|
+
.write()
|
|
248
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
249
|
+
|
|
250
|
+
for stage in [ProcessingStage::Early, ProcessingStage::Middle, ProcessingStage::Late] {
|
|
251
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
252
|
+
name: format!("{:?}", stage),
|
|
253
|
+
stage,
|
|
254
|
+
});
|
|
255
|
+
reg.register(processor, 50).unwrap();
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
let result = ExtractionResult {
|
|
260
|
+
content: "start".to_string(),
|
|
261
|
+
mime_type: "text/plain".to_string(),
|
|
262
|
+
metadata: Metadata::default(),
|
|
263
|
+
tables: vec![],
|
|
264
|
+
detected_languages: None,
|
|
265
|
+
chunks: None,
|
|
266
|
+
images: None,
|
|
267
|
+
pages: None,
|
|
268
|
+
};
|
|
269
|
+
let config = ExtractionConfig::default();
|
|
270
|
+
|
|
271
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
272
|
+
assert_eq!(processed.content, "start[Early][Middle][Late]");
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
#[tokio::test]
|
|
276
|
+
#[serial]
|
|
277
|
+
async fn test_pipeline_postprocessing_disabled() {
|
|
278
|
+
clear_processor_registry();
|
|
279
|
+
|
|
280
|
+
let registry = get_post_processor_registry();
|
|
281
|
+
{
|
|
282
|
+
let mut reg = registry
|
|
283
|
+
.write()
|
|
284
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
285
|
+
|
|
286
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
287
|
+
name: "processor".to_string(),
|
|
288
|
+
stage: ProcessingStage::Early,
|
|
289
|
+
});
|
|
290
|
+
reg.register(processor, 50).unwrap();
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
let result = ExtractionResult {
|
|
294
|
+
content: "start".to_string(),
|
|
295
|
+
mime_type: "text/plain".to_string(),
|
|
296
|
+
metadata: Metadata::default(),
|
|
297
|
+
tables: vec![],
|
|
298
|
+
detected_languages: None,
|
|
299
|
+
chunks: None,
|
|
300
|
+
images: None,
|
|
301
|
+
pages: None,
|
|
302
|
+
};
|
|
303
|
+
let config = ExtractionConfig {
|
|
304
|
+
postprocessor: Some(PostProcessorConfig {
|
|
305
|
+
enabled: false,
|
|
306
|
+
enabled_processors: None,
|
|
307
|
+
disabled_processors: None,
|
|
308
|
+
}),
|
|
309
|
+
..Default::default()
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
313
|
+
assert_eq!(processed.content, "start");
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
#[tokio::test]
|
|
317
|
+
#[serial]
|
|
318
|
+
async fn test_pipeline_early_stage_runs_first() {
|
|
319
|
+
clear_processor_registry();
|
|
320
|
+
|
|
321
|
+
let registry = get_post_processor_registry();
|
|
322
|
+
{
|
|
323
|
+
let mut reg = registry
|
|
324
|
+
.write()
|
|
325
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
326
|
+
|
|
327
|
+
let late = Arc::new(OrderTrackingProcessor {
|
|
328
|
+
name: "late".to_string(),
|
|
329
|
+
stage: ProcessingStage::Late,
|
|
330
|
+
});
|
|
331
|
+
let early = Arc::new(OrderTrackingProcessor {
|
|
332
|
+
name: "early".to_string(),
|
|
333
|
+
stage: ProcessingStage::Early,
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
reg.register(late, 50).unwrap();
|
|
337
|
+
reg.register(early, 50).unwrap();
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
let result = ExtractionResult {
|
|
341
|
+
content: "start".to_string(),
|
|
342
|
+
mime_type: "text/plain".to_string(),
|
|
343
|
+
metadata: Metadata::default(),
|
|
344
|
+
tables: vec![],
|
|
345
|
+
detected_languages: None,
|
|
346
|
+
chunks: None,
|
|
347
|
+
images: None,
|
|
348
|
+
pages: None,
|
|
349
|
+
};
|
|
350
|
+
let config = ExtractionConfig::default();
|
|
351
|
+
|
|
352
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
353
|
+
assert_eq!(processed.content, "start[early][late]");
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
#[tokio::test]
|
|
357
|
+
#[serial]
|
|
358
|
+
async fn test_pipeline_middle_stage_runs_second() {
|
|
359
|
+
clear_processor_registry();
|
|
360
|
+
|
|
361
|
+
let registry = get_post_processor_registry();
|
|
362
|
+
{
|
|
363
|
+
let mut reg = registry
|
|
364
|
+
.write()
|
|
365
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
366
|
+
|
|
367
|
+
let early = Arc::new(OrderTrackingProcessor {
|
|
368
|
+
name: "early".to_string(),
|
|
369
|
+
stage: ProcessingStage::Early,
|
|
370
|
+
});
|
|
371
|
+
let middle = Arc::new(OrderTrackingProcessor {
|
|
372
|
+
name: "middle".to_string(),
|
|
373
|
+
stage: ProcessingStage::Middle,
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
reg.register(middle, 50).unwrap();
|
|
377
|
+
reg.register(early, 50).unwrap();
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
let result = ExtractionResult {
|
|
381
|
+
content: "start".to_string(),
|
|
382
|
+
mime_type: "text/plain".to_string(),
|
|
383
|
+
metadata: Metadata::default(),
|
|
384
|
+
tables: vec![],
|
|
385
|
+
detected_languages: None,
|
|
386
|
+
chunks: None,
|
|
387
|
+
images: None,
|
|
388
|
+
pages: None,
|
|
389
|
+
};
|
|
390
|
+
let config = ExtractionConfig::default();
|
|
391
|
+
|
|
392
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
393
|
+
assert_eq!(processed.content, "start[early][middle]");
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
#[tokio::test]
|
|
397
|
+
#[serial]
|
|
398
|
+
async fn test_pipeline_late_stage_runs_last() {
|
|
399
|
+
clear_processor_registry();
|
|
400
|
+
|
|
401
|
+
let registry = get_post_processor_registry();
|
|
402
|
+
{
|
|
403
|
+
let mut reg = registry
|
|
404
|
+
.write()
|
|
405
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
406
|
+
|
|
407
|
+
for stage in [ProcessingStage::Late, ProcessingStage::Early, ProcessingStage::Middle] {
|
|
408
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
409
|
+
name: format!("{:?}", stage),
|
|
410
|
+
stage,
|
|
411
|
+
});
|
|
412
|
+
reg.register(processor, 50).unwrap();
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
let result = ExtractionResult {
|
|
417
|
+
content: "start".to_string(),
|
|
418
|
+
mime_type: "text/plain".to_string(),
|
|
419
|
+
metadata: Metadata::default(),
|
|
420
|
+
tables: vec![],
|
|
421
|
+
detected_languages: None,
|
|
422
|
+
chunks: None,
|
|
423
|
+
images: None,
|
|
424
|
+
pages: None,
|
|
425
|
+
};
|
|
426
|
+
let config = ExtractionConfig::default();
|
|
427
|
+
|
|
428
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
429
|
+
assert_eq!(processed.content, "start[Early][Middle][Late]");
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
#[tokio::test]
|
|
433
|
+
#[serial]
|
|
434
|
+
async fn test_pipeline_within_stage_priority_order() {
|
|
435
|
+
clear_processor_registry();
|
|
436
|
+
|
|
437
|
+
let registry = get_post_processor_registry();
|
|
438
|
+
{
|
|
439
|
+
let mut reg = registry
|
|
440
|
+
.write()
|
|
441
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
442
|
+
|
|
443
|
+
for (name, priority) in [("p1", 100), ("p2", 10), ("p3", 50), ("p4", 75)] {
|
|
444
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
445
|
+
name: name.to_string(),
|
|
446
|
+
stage: ProcessingStage::Early,
|
|
447
|
+
});
|
|
448
|
+
reg.register(processor, priority).unwrap();
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
let result = ExtractionResult {
|
|
453
|
+
content: "start".to_string(),
|
|
454
|
+
mime_type: "text/plain".to_string(),
|
|
455
|
+
metadata: Metadata::default(),
|
|
456
|
+
tables: vec![],
|
|
457
|
+
detected_languages: None,
|
|
458
|
+
chunks: None,
|
|
459
|
+
images: None,
|
|
460
|
+
pages: None,
|
|
461
|
+
};
|
|
462
|
+
let config = ExtractionConfig::default();
|
|
463
|
+
|
|
464
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
465
|
+
assert_eq!(processed.content, "start[p1][p4][p3][p2]");
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
#[tokio::test]
|
|
469
|
+
#[serial]
|
|
470
|
+
async fn test_pipeline_cross_stage_data_flow() {
|
|
471
|
+
clear_processor_registry();
|
|
472
|
+
|
|
473
|
+
let registry = get_post_processor_registry();
|
|
474
|
+
{
|
|
475
|
+
let mut reg = registry
|
|
476
|
+
.write()
|
|
477
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
478
|
+
|
|
479
|
+
let early = Arc::new(MetadataAddingProcessor {
|
|
480
|
+
name: "early".to_string(),
|
|
481
|
+
key: "stage".to_string(),
|
|
482
|
+
value: "early".to_string(),
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
struct MiddleProcessor;
|
|
486
|
+
impl Plugin for MiddleProcessor {
|
|
487
|
+
fn name(&self) -> &str {
|
|
488
|
+
"middle"
|
|
489
|
+
}
|
|
490
|
+
fn version(&self) -> String {
|
|
491
|
+
"1.0.0".to_string()
|
|
492
|
+
}
|
|
493
|
+
fn initialize(&self) -> Result<()> {
|
|
494
|
+
Ok(())
|
|
495
|
+
}
|
|
496
|
+
fn shutdown(&self) -> Result<()> {
|
|
497
|
+
Ok(())
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
#[async_trait]
|
|
501
|
+
impl PostProcessor for MiddleProcessor {
|
|
502
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
503
|
+
if let Some(stage) = result.metadata.additional.get("stage") {
|
|
504
|
+
result.content.push_str(&format!("[saw:{}]", stage.as_str().unwrap()));
|
|
505
|
+
}
|
|
506
|
+
Ok(())
|
|
507
|
+
}
|
|
508
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
509
|
+
ProcessingStage::Middle
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
reg.register(early, 50).unwrap();
|
|
514
|
+
reg.register(Arc::new(MiddleProcessor), 50).unwrap();
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
let result = ExtractionResult {
|
|
518
|
+
content: "start".to_string(),
|
|
519
|
+
mime_type: "text/plain".to_string(),
|
|
520
|
+
metadata: Metadata::default(),
|
|
521
|
+
tables: vec![],
|
|
522
|
+
detected_languages: None,
|
|
523
|
+
chunks: None,
|
|
524
|
+
images: None,
|
|
525
|
+
pages: None,
|
|
526
|
+
};
|
|
527
|
+
let config = ExtractionConfig::default();
|
|
528
|
+
|
|
529
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
530
|
+
assert!(processed.content.contains("[saw:early]"));
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
#[tokio::test]
|
|
534
|
+
#[serial]
|
|
535
|
+
async fn test_pipeline_early_stage_error_recorded() {
|
|
536
|
+
clear_processor_registry();
|
|
537
|
+
|
|
538
|
+
struct EarlyFailingProcessor;
|
|
539
|
+
impl Plugin for EarlyFailingProcessor {
|
|
540
|
+
fn name(&self) -> &str {
|
|
541
|
+
"early-failing"
|
|
542
|
+
}
|
|
543
|
+
fn version(&self) -> String {
|
|
544
|
+
"1.0.0".to_string()
|
|
545
|
+
}
|
|
546
|
+
fn initialize(&self) -> Result<()> {
|
|
547
|
+
Ok(())
|
|
548
|
+
}
|
|
549
|
+
fn shutdown(&self) -> Result<()> {
|
|
550
|
+
Ok(())
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
#[async_trait]
|
|
554
|
+
impl PostProcessor for EarlyFailingProcessor {
|
|
555
|
+
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
556
|
+
Err(KreuzbergError::Plugin {
|
|
557
|
+
message: "Early error".to_string(),
|
|
558
|
+
plugin_name: "early-failing".to_string(),
|
|
559
|
+
})
|
|
560
|
+
}
|
|
561
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
562
|
+
ProcessingStage::Early
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
let registry = get_post_processor_registry();
|
|
567
|
+
{
|
|
568
|
+
let mut reg = registry
|
|
569
|
+
.write()
|
|
570
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
571
|
+
reg.register(Arc::new(EarlyFailingProcessor), 50).unwrap();
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
let result = ExtractionResult {
|
|
575
|
+
content: "content".to_string(),
|
|
576
|
+
mime_type: "text/plain".to_string(),
|
|
577
|
+
metadata: Metadata::default(),
|
|
578
|
+
tables: vec![],
|
|
579
|
+
detected_languages: None,
|
|
580
|
+
chunks: None,
|
|
581
|
+
images: None,
|
|
582
|
+
pages: None,
|
|
583
|
+
};
|
|
584
|
+
let config = ExtractionConfig::default();
|
|
585
|
+
|
|
586
|
+
let result = run_pipeline(result, &config).await;
|
|
587
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
588
|
+
match result {
|
|
589
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
590
|
+
assert_eq!(message, "Early error");
|
|
591
|
+
assert_eq!(plugin_name, "early-failing");
|
|
592
|
+
}
|
|
593
|
+
_ => panic!("Expected Plugin error"),
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
#[tokio::test]
|
|
598
|
+
#[serial]
|
|
599
|
+
async fn test_pipeline_middle_stage_error_propagation() {
|
|
600
|
+
clear_processor_registry();
|
|
601
|
+
|
|
602
|
+
let registry = get_post_processor_registry();
|
|
603
|
+
{
|
|
604
|
+
let mut reg = registry
|
|
605
|
+
.write()
|
|
606
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
607
|
+
|
|
608
|
+
let failing = Arc::new(FailingProcessor {
|
|
609
|
+
name: "middle-failing".to_string(),
|
|
610
|
+
error_message: "Middle stage error".to_string(),
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
reg.register(failing, 50).unwrap();
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
let result = ExtractionResult {
|
|
617
|
+
content: "content".to_string(),
|
|
618
|
+
mime_type: "text/plain".to_string(),
|
|
619
|
+
metadata: Metadata::default(),
|
|
620
|
+
tables: vec![],
|
|
621
|
+
detected_languages: None,
|
|
622
|
+
chunks: None,
|
|
623
|
+
images: None,
|
|
624
|
+
pages: None,
|
|
625
|
+
};
|
|
626
|
+
let config = ExtractionConfig::default();
|
|
627
|
+
|
|
628
|
+
let result = run_pipeline(result, &config).await;
|
|
629
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
630
|
+
match result {
|
|
631
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
632
|
+
assert_eq!(message, "Middle stage error");
|
|
633
|
+
assert_eq!(plugin_name, "middle-failing");
|
|
634
|
+
}
|
|
635
|
+
_ => panic!("Expected Plugin error"),
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
#[tokio::test]
|
|
640
|
+
#[serial]
|
|
641
|
+
async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
|
|
642
|
+
clear_processor_registry();
|
|
643
|
+
|
|
644
|
+
struct LateFailingProcessor;
|
|
645
|
+
impl Plugin for LateFailingProcessor {
|
|
646
|
+
fn name(&self) -> &str {
|
|
647
|
+
"late-failing"
|
|
648
|
+
}
|
|
649
|
+
fn version(&self) -> String {
|
|
650
|
+
"1.0.0".to_string()
|
|
651
|
+
}
|
|
652
|
+
fn initialize(&self) -> Result<()> {
|
|
653
|
+
Ok(())
|
|
654
|
+
}
|
|
655
|
+
fn shutdown(&self) -> Result<()> {
|
|
656
|
+
Ok(())
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
#[async_trait]
|
|
660
|
+
impl PostProcessor for LateFailingProcessor {
|
|
661
|
+
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
662
|
+
Err(KreuzbergError::Plugin {
|
|
663
|
+
message: "Late error".to_string(),
|
|
664
|
+
plugin_name: "late-failing".to_string(),
|
|
665
|
+
})
|
|
666
|
+
}
|
|
667
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
668
|
+
ProcessingStage::Late
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
let registry = get_post_processor_registry();
|
|
673
|
+
{
|
|
674
|
+
let mut reg = registry
|
|
675
|
+
.write()
|
|
676
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
677
|
+
|
|
678
|
+
let early = Arc::new(OrderTrackingProcessor {
|
|
679
|
+
name: "early".to_string(),
|
|
680
|
+
stage: ProcessingStage::Early,
|
|
681
|
+
});
|
|
682
|
+
let late_failing = Arc::new(LateFailingProcessor);
|
|
683
|
+
|
|
684
|
+
reg.register(early, 50).unwrap();
|
|
685
|
+
reg.register(late_failing, 50).unwrap();
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
let result = ExtractionResult {
|
|
689
|
+
content: "start".to_string(),
|
|
690
|
+
mime_type: "text/plain".to_string(),
|
|
691
|
+
metadata: Metadata::default(),
|
|
692
|
+
tables: vec![],
|
|
693
|
+
detected_languages: None,
|
|
694
|
+
chunks: None,
|
|
695
|
+
images: None,
|
|
696
|
+
pages: None,
|
|
697
|
+
};
|
|
698
|
+
let config = ExtractionConfig::default();
|
|
699
|
+
|
|
700
|
+
let result = run_pipeline(result, &config).await;
|
|
701
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
702
|
+
match result {
|
|
703
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
704
|
+
assert_eq!(message, "Late error");
|
|
705
|
+
assert_eq!(plugin_name, "late-failing");
|
|
706
|
+
}
|
|
707
|
+
_ => panic!("Expected Plugin error"),
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
#[tokio::test]
|
|
712
|
+
#[serial]
|
|
713
|
+
async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
|
|
714
|
+
clear_processor_registry();
|
|
715
|
+
|
|
716
|
+
let registry = get_post_processor_registry();
|
|
717
|
+
{
|
|
718
|
+
let mut reg = registry
|
|
719
|
+
.write()
|
|
720
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
721
|
+
|
|
722
|
+
let p1 = Arc::new(OrderTrackingProcessor {
|
|
723
|
+
name: "p1".to_string(),
|
|
724
|
+
stage: ProcessingStage::Early,
|
|
725
|
+
});
|
|
726
|
+
let _p2_failing = Arc::new(FailingProcessor {
|
|
727
|
+
name: "p2-failing".to_string(),
|
|
728
|
+
error_message: "Test error".to_string(),
|
|
729
|
+
});
|
|
730
|
+
let p3 = Arc::new(OrderTrackingProcessor {
|
|
731
|
+
name: "p3".to_string(),
|
|
732
|
+
stage: ProcessingStage::Late,
|
|
733
|
+
});
|
|
734
|
+
|
|
735
|
+
struct EarlyFailingProcessor {
|
|
736
|
+
name: String,
|
|
737
|
+
}
|
|
738
|
+
impl Plugin for EarlyFailingProcessor {
|
|
739
|
+
fn name(&self) -> &str {
|
|
740
|
+
&self.name
|
|
741
|
+
}
|
|
742
|
+
fn version(&self) -> String {
|
|
743
|
+
"1.0.0".to_string()
|
|
744
|
+
}
|
|
745
|
+
fn initialize(&self) -> Result<()> {
|
|
746
|
+
Ok(())
|
|
747
|
+
}
|
|
748
|
+
fn shutdown(&self) -> Result<()> {
|
|
749
|
+
Ok(())
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
#[async_trait]
|
|
753
|
+
impl PostProcessor for EarlyFailingProcessor {
|
|
754
|
+
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
755
|
+
Err(KreuzbergError::Plugin {
|
|
756
|
+
message: "Test error".to_string(),
|
|
757
|
+
plugin_name: self.name.clone(),
|
|
758
|
+
})
|
|
759
|
+
}
|
|
760
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
761
|
+
ProcessingStage::Early
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
reg.register(p1, 100).unwrap();
|
|
766
|
+
reg.register(
|
|
767
|
+
Arc::new(EarlyFailingProcessor {
|
|
768
|
+
name: "p2-failing".to_string(),
|
|
769
|
+
}),
|
|
770
|
+
50,
|
|
771
|
+
)
|
|
772
|
+
.unwrap();
|
|
773
|
+
reg.register(p3, 50).unwrap();
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
let result = ExtractionResult {
|
|
777
|
+
content: "start".to_string(),
|
|
778
|
+
mime_type: "text/plain".to_string(),
|
|
779
|
+
metadata: Metadata::default(),
|
|
780
|
+
tables: vec![],
|
|
781
|
+
detected_languages: None,
|
|
782
|
+
chunks: None,
|
|
783
|
+
images: None,
|
|
784
|
+
pages: None,
|
|
785
|
+
};
|
|
786
|
+
let config = ExtractionConfig::default();
|
|
787
|
+
|
|
788
|
+
let result = run_pipeline(result, &config).await;
|
|
789
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
790
|
+
match result {
|
|
791
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
792
|
+
assert_eq!(message, "Test error");
|
|
793
|
+
assert_eq!(plugin_name, "p2-failing");
|
|
794
|
+
}
|
|
795
|
+
_ => panic!("Expected Plugin error"),
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
#[tokio::test]
|
|
800
|
+
#[serial]
|
|
801
|
+
async fn test_pipeline_multiple_processor_errors() {
|
|
802
|
+
clear_processor_registry();
|
|
803
|
+
|
|
804
|
+
struct MultiFailingProcessor {
|
|
805
|
+
name: String,
|
|
806
|
+
stage: ProcessingStage,
|
|
807
|
+
}
|
|
808
|
+
impl Plugin for MultiFailingProcessor {
|
|
809
|
+
fn name(&self) -> &str {
|
|
810
|
+
&self.name
|
|
811
|
+
}
|
|
812
|
+
fn version(&self) -> String {
|
|
813
|
+
"1.0.0".to_string()
|
|
814
|
+
}
|
|
815
|
+
fn initialize(&self) -> Result<()> {
|
|
816
|
+
Ok(())
|
|
817
|
+
}
|
|
818
|
+
fn shutdown(&self) -> Result<()> {
|
|
819
|
+
Ok(())
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
#[async_trait]
|
|
823
|
+
impl PostProcessor for MultiFailingProcessor {
|
|
824
|
+
async fn process(&self, _: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
825
|
+
Err(KreuzbergError::Plugin {
|
|
826
|
+
message: format!("{} error", self.name),
|
|
827
|
+
plugin_name: self.name.clone(),
|
|
828
|
+
})
|
|
829
|
+
}
|
|
830
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
831
|
+
self.stage
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
let registry = get_post_processor_registry();
|
|
836
|
+
{
|
|
837
|
+
let mut reg = registry
|
|
838
|
+
.write()
|
|
839
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
840
|
+
|
|
841
|
+
for (name, stage) in [
|
|
842
|
+
("fail1", ProcessingStage::Early),
|
|
843
|
+
("fail2", ProcessingStage::Middle),
|
|
844
|
+
("fail3", ProcessingStage::Late),
|
|
845
|
+
] {
|
|
846
|
+
let processor = Arc::new(MultiFailingProcessor {
|
|
847
|
+
name: name.to_string(),
|
|
848
|
+
stage,
|
|
849
|
+
});
|
|
850
|
+
reg.register(processor, 50).unwrap();
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
let result = ExtractionResult {
|
|
855
|
+
content: "start".to_string(),
|
|
856
|
+
mime_type: "text/plain".to_string(),
|
|
857
|
+
metadata: Metadata::default(),
|
|
858
|
+
tables: vec![],
|
|
859
|
+
detected_languages: None,
|
|
860
|
+
chunks: None,
|
|
861
|
+
images: None,
|
|
862
|
+
pages: None,
|
|
863
|
+
};
|
|
864
|
+
let config = ExtractionConfig::default();
|
|
865
|
+
|
|
866
|
+
let result = run_pipeline(result, &config).await;
|
|
867
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
868
|
+
match result {
|
|
869
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
870
|
+
assert_eq!(message, "fail1 error");
|
|
871
|
+
assert_eq!(plugin_name, "fail1");
|
|
872
|
+
}
|
|
873
|
+
_ => panic!("Expected Plugin error"),
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
#[tokio::test]
|
|
878
|
+
#[serial]
|
|
879
|
+
async fn test_pipeline_error_context_preservation() {
|
|
880
|
+
clear_processor_registry();
|
|
881
|
+
|
|
882
|
+
let registry = get_post_processor_registry();
|
|
883
|
+
{
|
|
884
|
+
let mut reg = registry
|
|
885
|
+
.write()
|
|
886
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
887
|
+
|
|
888
|
+
let failing = Arc::new(FailingProcessor {
|
|
889
|
+
name: "context-test".to_string(),
|
|
890
|
+
error_message: "Detailed error message with context".to_string(),
|
|
891
|
+
});
|
|
892
|
+
|
|
893
|
+
reg.register(failing, 50).unwrap();
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
let result = ExtractionResult {
|
|
897
|
+
content: "content".to_string(),
|
|
898
|
+
mime_type: "text/plain".to_string(),
|
|
899
|
+
metadata: Metadata::default(),
|
|
900
|
+
tables: vec![],
|
|
901
|
+
detected_languages: None,
|
|
902
|
+
chunks: None,
|
|
903
|
+
images: None,
|
|
904
|
+
pages: None,
|
|
905
|
+
};
|
|
906
|
+
let config = ExtractionConfig::default();
|
|
907
|
+
|
|
908
|
+
let result = run_pipeline(result, &config).await;
|
|
909
|
+
assert!(result.is_err(), "Expected pipeline to return error");
|
|
910
|
+
match result {
|
|
911
|
+
Err(KreuzbergError::Plugin { message, plugin_name }) => {
|
|
912
|
+
assert_eq!(message, "Detailed error message with context");
|
|
913
|
+
assert_eq!(plugin_name, "context-test");
|
|
914
|
+
}
|
|
915
|
+
_ => panic!("Expected Plugin error"),
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
#[tokio::test]
|
|
920
|
+
#[serial]
|
|
921
|
+
async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
|
|
922
|
+
clear_processor_registry();
|
|
923
|
+
|
|
924
|
+
let registry = get_post_processor_registry();
|
|
925
|
+
{
|
|
926
|
+
let mut reg = registry
|
|
927
|
+
.write()
|
|
928
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
929
|
+
|
|
930
|
+
let early = Arc::new(MetadataAddingProcessor {
|
|
931
|
+
name: "early".to_string(),
|
|
932
|
+
key: "early_key".to_string(),
|
|
933
|
+
value: "early_value".to_string(),
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
struct MiddleReadingProcessor;
|
|
937
|
+
impl Plugin for MiddleReadingProcessor {
|
|
938
|
+
fn name(&self) -> &str {
|
|
939
|
+
"middle"
|
|
940
|
+
}
|
|
941
|
+
fn version(&self) -> String {
|
|
942
|
+
"1.0.0".to_string()
|
|
943
|
+
}
|
|
944
|
+
fn initialize(&self) -> Result<()> {
|
|
945
|
+
Ok(())
|
|
946
|
+
}
|
|
947
|
+
fn shutdown(&self) -> Result<()> {
|
|
948
|
+
Ok(())
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
#[async_trait]
|
|
952
|
+
impl PostProcessor for MiddleReadingProcessor {
|
|
953
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
954
|
+
if let Some(val) = result.metadata.additional.get("early_key") {
|
|
955
|
+
result.metadata.additional.insert("middle_saw".to_string(), val.clone());
|
|
956
|
+
}
|
|
957
|
+
Ok(())
|
|
958
|
+
}
|
|
959
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
960
|
+
ProcessingStage::Middle
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
reg.register(early, 50).unwrap();
|
|
965
|
+
reg.register(Arc::new(MiddleReadingProcessor), 50).unwrap();
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
let result = ExtractionResult {
|
|
969
|
+
content: "content".to_string(),
|
|
970
|
+
mime_type: "text/plain".to_string(),
|
|
971
|
+
metadata: Metadata::default(),
|
|
972
|
+
tables: vec![],
|
|
973
|
+
detected_languages: None,
|
|
974
|
+
chunks: None,
|
|
975
|
+
images: None,
|
|
976
|
+
pages: None,
|
|
977
|
+
};
|
|
978
|
+
let config = ExtractionConfig::default();
|
|
979
|
+
|
|
980
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
981
|
+
assert_eq!(
|
|
982
|
+
processed
|
|
983
|
+
.metadata
|
|
984
|
+
.additional
|
|
985
|
+
.get("middle_saw")
|
|
986
|
+
.unwrap()
|
|
987
|
+
.as_str()
|
|
988
|
+
.unwrap(),
|
|
989
|
+
"early_value"
|
|
990
|
+
);
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
#[tokio::test]
|
|
994
|
+
#[serial]
|
|
995
|
+
async fn test_pipeline_content_modified_in_middle_visible_in_late() {
|
|
996
|
+
clear_processor_registry();
|
|
997
|
+
|
|
998
|
+
let registry = get_post_processor_registry();
|
|
999
|
+
{
|
|
1000
|
+
let mut reg = registry
|
|
1001
|
+
.write()
|
|
1002
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1003
|
+
|
|
1004
|
+
let middle = Arc::new(OrderTrackingProcessor {
|
|
1005
|
+
name: "middle-content".to_string(),
|
|
1006
|
+
stage: ProcessingStage::Middle,
|
|
1007
|
+
});
|
|
1008
|
+
|
|
1009
|
+
struct LateReadingProcessor;
|
|
1010
|
+
impl Plugin for LateReadingProcessor {
|
|
1011
|
+
fn name(&self) -> &str {
|
|
1012
|
+
"late"
|
|
1013
|
+
}
|
|
1014
|
+
fn version(&self) -> String {
|
|
1015
|
+
"1.0.0".to_string()
|
|
1016
|
+
}
|
|
1017
|
+
fn initialize(&self) -> Result<()> {
|
|
1018
|
+
Ok(())
|
|
1019
|
+
}
|
|
1020
|
+
fn shutdown(&self) -> Result<()> {
|
|
1021
|
+
Ok(())
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
#[async_trait]
|
|
1025
|
+
impl PostProcessor for LateReadingProcessor {
|
|
1026
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1027
|
+
result.content.push_str("[late-saw-middle]");
|
|
1028
|
+
Ok(())
|
|
1029
|
+
}
|
|
1030
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
1031
|
+
ProcessingStage::Late
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
reg.register(middle, 50).unwrap();
|
|
1036
|
+
reg.register(Arc::new(LateReadingProcessor), 50).unwrap();
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
let result = ExtractionResult {
|
|
1040
|
+
content: "start".to_string(),
|
|
1041
|
+
mime_type: "text/plain".to_string(),
|
|
1042
|
+
metadata: Metadata::default(),
|
|
1043
|
+
tables: vec![],
|
|
1044
|
+
detected_languages: None,
|
|
1045
|
+
chunks: None,
|
|
1046
|
+
images: None,
|
|
1047
|
+
pages: None,
|
|
1048
|
+
};
|
|
1049
|
+
let config = ExtractionConfig::default();
|
|
1050
|
+
|
|
1051
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1052
|
+
assert!(processed.content.contains("[middle-content]"));
|
|
1053
|
+
assert!(processed.content.contains("[late-saw-middle]"));
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
#[tokio::test]
|
|
1057
|
+
#[serial]
|
|
1058
|
+
async fn test_pipeline_multiple_processors_modifying_same_metadata() {
|
|
1059
|
+
clear_processor_registry();
|
|
1060
|
+
|
|
1061
|
+
let registry = get_post_processor_registry();
|
|
1062
|
+
{
|
|
1063
|
+
let mut reg = registry
|
|
1064
|
+
.write()
|
|
1065
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1066
|
+
|
|
1067
|
+
for i in 1..=3 {
|
|
1068
|
+
struct MetadataOverwritingProcessor {
|
|
1069
|
+
name: String,
|
|
1070
|
+
value: String,
|
|
1071
|
+
}
|
|
1072
|
+
impl Plugin for MetadataOverwritingProcessor {
|
|
1073
|
+
fn name(&self) -> &str {
|
|
1074
|
+
&self.name
|
|
1075
|
+
}
|
|
1076
|
+
fn version(&self) -> String {
|
|
1077
|
+
"1.0.0".to_string()
|
|
1078
|
+
}
|
|
1079
|
+
fn initialize(&self) -> Result<()> {
|
|
1080
|
+
Ok(())
|
|
1081
|
+
}
|
|
1082
|
+
fn shutdown(&self) -> Result<()> {
|
|
1083
|
+
Ok(())
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
#[async_trait]
|
|
1087
|
+
impl PostProcessor for MetadataOverwritingProcessor {
|
|
1088
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1089
|
+
result
|
|
1090
|
+
.metadata
|
|
1091
|
+
.additional
|
|
1092
|
+
.insert("shared_key".to_string(), serde_json::json!(self.value));
|
|
1093
|
+
Ok(())
|
|
1094
|
+
}
|
|
1095
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
1096
|
+
ProcessingStage::Early
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
let processor = Arc::new(MetadataOverwritingProcessor {
|
|
1101
|
+
name: format!("proc{}", i),
|
|
1102
|
+
value: format!("value{}", i),
|
|
1103
|
+
});
|
|
1104
|
+
reg.register(processor, 100 - i * 10).unwrap();
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
let result = ExtractionResult {
|
|
1109
|
+
content: "content".to_string(),
|
|
1110
|
+
mime_type: "text/plain".to_string(),
|
|
1111
|
+
metadata: Metadata::default(),
|
|
1112
|
+
tables: vec![],
|
|
1113
|
+
detected_languages: None,
|
|
1114
|
+
chunks: None,
|
|
1115
|
+
images: None,
|
|
1116
|
+
pages: None,
|
|
1117
|
+
};
|
|
1118
|
+
let config = ExtractionConfig::default();
|
|
1119
|
+
|
|
1120
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1121
|
+
assert_eq!(
|
|
1122
|
+
processed
|
|
1123
|
+
.metadata
|
|
1124
|
+
.additional
|
|
1125
|
+
.get("shared_key")
|
|
1126
|
+
.unwrap()
|
|
1127
|
+
.as_str()
|
|
1128
|
+
.unwrap(),
|
|
1129
|
+
"value3"
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
#[tokio::test]
|
|
1134
|
+
#[serial]
|
|
1135
|
+
async fn test_pipeline_processors_reading_previous_output() {
|
|
1136
|
+
clear_processor_registry();
|
|
1137
|
+
|
|
1138
|
+
let registry = get_post_processor_registry();
|
|
1139
|
+
{
|
|
1140
|
+
let mut reg = registry
|
|
1141
|
+
.write()
|
|
1142
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1143
|
+
|
|
1144
|
+
struct CountingProcessor {
|
|
1145
|
+
name: String,
|
|
1146
|
+
stage: ProcessingStage,
|
|
1147
|
+
}
|
|
1148
|
+
impl Plugin for CountingProcessor {
|
|
1149
|
+
fn name(&self) -> &str {
|
|
1150
|
+
&self.name
|
|
1151
|
+
}
|
|
1152
|
+
fn version(&self) -> String {
|
|
1153
|
+
"1.0.0".to_string()
|
|
1154
|
+
}
|
|
1155
|
+
fn initialize(&self) -> Result<()> {
|
|
1156
|
+
Ok(())
|
|
1157
|
+
}
|
|
1158
|
+
fn shutdown(&self) -> Result<()> {
|
|
1159
|
+
Ok(())
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
#[async_trait]
|
|
1163
|
+
impl PostProcessor for CountingProcessor {
|
|
1164
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1165
|
+
let current_count = result
|
|
1166
|
+
.metadata
|
|
1167
|
+
.additional
|
|
1168
|
+
.get("count")
|
|
1169
|
+
.and_then(|v| v.as_i64())
|
|
1170
|
+
.unwrap_or(0);
|
|
1171
|
+
result
|
|
1172
|
+
.metadata
|
|
1173
|
+
.additional
|
|
1174
|
+
.insert("count".to_string(), serde_json::json!(current_count + 1));
|
|
1175
|
+
Ok(())
|
|
1176
|
+
}
|
|
1177
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
1178
|
+
self.stage
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
for (name, stage) in [
|
|
1183
|
+
("early1", ProcessingStage::Early),
|
|
1184
|
+
("early2", ProcessingStage::Early),
|
|
1185
|
+
("middle1", ProcessingStage::Middle),
|
|
1186
|
+
("late1", ProcessingStage::Late),
|
|
1187
|
+
] {
|
|
1188
|
+
let processor = Arc::new(CountingProcessor {
|
|
1189
|
+
name: name.to_string(),
|
|
1190
|
+
stage,
|
|
1191
|
+
});
|
|
1192
|
+
reg.register(processor, 50).unwrap();
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
let result = ExtractionResult {
|
|
1197
|
+
content: "content".to_string(),
|
|
1198
|
+
mime_type: "text/plain".to_string(),
|
|
1199
|
+
metadata: Metadata::default(),
|
|
1200
|
+
tables: vec![],
|
|
1201
|
+
detected_languages: None,
|
|
1202
|
+
chunks: None,
|
|
1203
|
+
images: None,
|
|
1204
|
+
pages: None,
|
|
1205
|
+
};
|
|
1206
|
+
let config = ExtractionConfig::default();
|
|
1207
|
+
|
|
1208
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1209
|
+
assert_eq!(processed.metadata.additional.get("count").unwrap().as_i64().unwrap(), 4);
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
#[tokio::test]
|
|
1213
|
+
#[serial]
|
|
1214
|
+
async fn test_pipeline_large_content_modification() {
|
|
1215
|
+
clear_processor_registry();
|
|
1216
|
+
|
|
1217
|
+
struct LargeContentProcessor;
|
|
1218
|
+
impl Plugin for LargeContentProcessor {
|
|
1219
|
+
fn name(&self) -> &str {
|
|
1220
|
+
"large"
|
|
1221
|
+
}
|
|
1222
|
+
fn version(&self) -> String {
|
|
1223
|
+
"1.0.0".to_string()
|
|
1224
|
+
}
|
|
1225
|
+
fn initialize(&self) -> Result<()> {
|
|
1226
|
+
Ok(())
|
|
1227
|
+
}
|
|
1228
|
+
fn shutdown(&self) -> Result<()> {
|
|
1229
|
+
Ok(())
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
#[async_trait]
|
|
1233
|
+
impl PostProcessor for LargeContentProcessor {
|
|
1234
|
+
async fn process(&self, result: &mut ExtractionResult, _: &ExtractionConfig) -> Result<()> {
|
|
1235
|
+
result.content.push_str(&"x".repeat(10000));
|
|
1236
|
+
Ok(())
|
|
1237
|
+
}
|
|
1238
|
+
fn processing_stage(&self) -> ProcessingStage {
|
|
1239
|
+
ProcessingStage::Early
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
let registry = get_post_processor_registry();
|
|
1244
|
+
{
|
|
1245
|
+
let mut reg = registry
|
|
1246
|
+
.write()
|
|
1247
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1248
|
+
reg.register(Arc::new(LargeContentProcessor), 50).unwrap();
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
let result = ExtractionResult {
|
|
1252
|
+
content: "start".to_string(),
|
|
1253
|
+
mime_type: "text/plain".to_string(),
|
|
1254
|
+
metadata: Metadata::default(),
|
|
1255
|
+
tables: vec![],
|
|
1256
|
+
detected_languages: None,
|
|
1257
|
+
chunks: None,
|
|
1258
|
+
images: None,
|
|
1259
|
+
pages: None,
|
|
1260
|
+
};
|
|
1261
|
+
let config = ExtractionConfig::default();
|
|
1262
|
+
|
|
1263
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1264
|
+
assert!(processed.content.len() > 10000);
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
#[tokio::test]
|
|
1268
|
+
#[serial]
|
|
1269
|
+
async fn test_pipeline_enabled_processors_whitelist() {
|
|
1270
|
+
clear_processor_registry();
|
|
1271
|
+
|
|
1272
|
+
let registry = get_post_processor_registry();
|
|
1273
|
+
{
|
|
1274
|
+
let mut reg = registry
|
|
1275
|
+
.write()
|
|
1276
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1277
|
+
|
|
1278
|
+
for name in ["proc1", "proc2", "proc3"] {
|
|
1279
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
1280
|
+
name: name.to_string(),
|
|
1281
|
+
stage: ProcessingStage::Early,
|
|
1282
|
+
});
|
|
1283
|
+
reg.register(processor, 50).unwrap();
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
let result = ExtractionResult {
|
|
1288
|
+
content: "start".to_string(),
|
|
1289
|
+
mime_type: "text/plain".to_string(),
|
|
1290
|
+
metadata: Metadata::default(),
|
|
1291
|
+
tables: vec![],
|
|
1292
|
+
detected_languages: None,
|
|
1293
|
+
chunks: None,
|
|
1294
|
+
images: None,
|
|
1295
|
+
pages: None,
|
|
1296
|
+
};
|
|
1297
|
+
let config = ExtractionConfig {
|
|
1298
|
+
postprocessor: Some(PostProcessorConfig {
|
|
1299
|
+
enabled: true,
|
|
1300
|
+
enabled_processors: Some(vec!["proc1".to_string(), "proc3".to_string()]),
|
|
1301
|
+
disabled_processors: None,
|
|
1302
|
+
}),
|
|
1303
|
+
..Default::default()
|
|
1304
|
+
};
|
|
1305
|
+
|
|
1306
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1307
|
+
assert!(processed.content.contains("[proc1]"));
|
|
1308
|
+
assert!(!processed.content.contains("[proc2]"));
|
|
1309
|
+
assert!(processed.content.contains("[proc3]"));
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
#[tokio::test]
|
|
1313
|
+
#[serial]
|
|
1314
|
+
async fn test_pipeline_disabled_processors_blacklist() {
|
|
1315
|
+
clear_processor_registry();
|
|
1316
|
+
|
|
1317
|
+
let registry = get_post_processor_registry();
|
|
1318
|
+
{
|
|
1319
|
+
let mut reg = registry
|
|
1320
|
+
.write()
|
|
1321
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1322
|
+
|
|
1323
|
+
for name in ["proc1", "proc2", "proc3"] {
|
|
1324
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
1325
|
+
name: name.to_string(),
|
|
1326
|
+
stage: ProcessingStage::Early,
|
|
1327
|
+
});
|
|
1328
|
+
reg.register(processor, 50).unwrap();
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
let result = ExtractionResult {
|
|
1333
|
+
content: "start".to_string(),
|
|
1334
|
+
mime_type: "text/plain".to_string(),
|
|
1335
|
+
metadata: Metadata::default(),
|
|
1336
|
+
tables: vec![],
|
|
1337
|
+
detected_languages: None,
|
|
1338
|
+
chunks: None,
|
|
1339
|
+
images: None,
|
|
1340
|
+
pages: None,
|
|
1341
|
+
};
|
|
1342
|
+
let config = ExtractionConfig {
|
|
1343
|
+
postprocessor: Some(PostProcessorConfig {
|
|
1344
|
+
enabled: true,
|
|
1345
|
+
enabled_processors: None,
|
|
1346
|
+
disabled_processors: Some(vec!["proc2".to_string()]),
|
|
1347
|
+
}),
|
|
1348
|
+
..Default::default()
|
|
1349
|
+
};
|
|
1350
|
+
|
|
1351
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1352
|
+
assert!(processed.content.contains("[proc1]"));
|
|
1353
|
+
assert!(!processed.content.contains("[proc2]"));
|
|
1354
|
+
assert!(processed.content.contains("[proc3]"));
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
#[tokio::test]
|
|
1358
|
+
#[serial]
|
|
1359
|
+
async fn test_pipeline_no_filtering_runs_all() {
|
|
1360
|
+
clear_processor_registry();
|
|
1361
|
+
|
|
1362
|
+
let registry = get_post_processor_registry();
|
|
1363
|
+
{
|
|
1364
|
+
let mut reg = registry
|
|
1365
|
+
.write()
|
|
1366
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1367
|
+
|
|
1368
|
+
for name in ["proc1", "proc2", "proc3"] {
|
|
1369
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
1370
|
+
name: name.to_string(),
|
|
1371
|
+
stage: ProcessingStage::Early,
|
|
1372
|
+
});
|
|
1373
|
+
reg.register(processor, 50).unwrap();
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
let result = ExtractionResult {
|
|
1378
|
+
content: "start".to_string(),
|
|
1379
|
+
mime_type: "text/plain".to_string(),
|
|
1380
|
+
metadata: Metadata::default(),
|
|
1381
|
+
tables: vec![],
|
|
1382
|
+
detected_languages: None,
|
|
1383
|
+
chunks: None,
|
|
1384
|
+
images: None,
|
|
1385
|
+
pages: None,
|
|
1386
|
+
};
|
|
1387
|
+
let config = ExtractionConfig::default();
|
|
1388
|
+
|
|
1389
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1390
|
+
assert!(processed.content.contains("[proc1]"));
|
|
1391
|
+
assert!(processed.content.contains("[proc2]"));
|
|
1392
|
+
assert!(processed.content.contains("[proc3]"));
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
#[tokio::test]
|
|
1396
|
+
#[serial]
|
|
1397
|
+
async fn test_pipeline_empty_whitelist_runs_none() {
|
|
1398
|
+
clear_processor_registry();
|
|
1399
|
+
|
|
1400
|
+
let registry = get_post_processor_registry();
|
|
1401
|
+
{
|
|
1402
|
+
let mut reg = registry
|
|
1403
|
+
.write()
|
|
1404
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
1405
|
+
|
|
1406
|
+
for name in ["proc1", "proc2"] {
|
|
1407
|
+
let processor = Arc::new(OrderTrackingProcessor {
|
|
1408
|
+
name: name.to_string(),
|
|
1409
|
+
stage: ProcessingStage::Early,
|
|
1410
|
+
});
|
|
1411
|
+
reg.register(processor, 50).unwrap();
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
let result = ExtractionResult {
|
|
1416
|
+
content: "start".to_string(),
|
|
1417
|
+
mime_type: "text/plain".to_string(),
|
|
1418
|
+
metadata: Metadata::default(),
|
|
1419
|
+
tables: vec![],
|
|
1420
|
+
detected_languages: None,
|
|
1421
|
+
chunks: None,
|
|
1422
|
+
images: None,
|
|
1423
|
+
pages: None,
|
|
1424
|
+
};
|
|
1425
|
+
let config = ExtractionConfig {
|
|
1426
|
+
postprocessor: Some(PostProcessorConfig {
|
|
1427
|
+
enabled: true,
|
|
1428
|
+
enabled_processors: Some(vec![]),
|
|
1429
|
+
disabled_processors: None,
|
|
1430
|
+
}),
|
|
1431
|
+
..Default::default()
|
|
1432
|
+
};
|
|
1433
|
+
|
|
1434
|
+
let processed = run_pipeline(result, &config).await.unwrap();
|
|
1435
|
+
assert_eq!(processed.content, "start");
|
|
1436
|
+
}
|