kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +105 -2
- data/README.md +454 -454
- data/Rakefile +33 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6940 -6941
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3158
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -214
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -81
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -80
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -340
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -109
- data/lib/{pdfium.dll → libpdfium.dylib} +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -546
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +1 -1
- data/vendor/kreuzberg/Cargo.toml +5 -5
- data/vendor/kreuzberg/README.md +230 -230
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -843
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1184
- data/vendor/kreuzberg/src/embeddings.rs +500 -500
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -601
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -574
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -749
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -44
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -346
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -68
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -420
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -164
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +1 -1
- data/vendor/kreuzberg-ffi/README.md +851 -851
- data/vendor/kreuzberg-ffi/build.rs +176 -176
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -27
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -1087
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -3616
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -247
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -470
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
- data/vendor/kreuzberg-tesseract/LICENSE +22 -22
- data/vendor/kreuzberg-tesseract/README.md +399 -399
- data/vendor/kreuzberg-tesseract/build.rs +1354 -1354
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -211
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +73 -4
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +0 -12
|
@@ -1,1044 +1,1044 @@
|
|
|
1
|
-
//! Document extractor plugin trait.
|
|
2
|
-
//!
|
|
3
|
-
//! This module defines the trait for implementing custom document extractors.
|
|
4
|
-
|
|
5
|
-
use crate::Result;
|
|
6
|
-
use crate::core::config::ExtractionConfig;
|
|
7
|
-
use crate::plugins::Plugin;
|
|
8
|
-
use crate::types::ExtractionResult;
|
|
9
|
-
use async_trait::async_trait;
|
|
10
|
-
use std::path::Path;
|
|
11
|
-
use std::sync::Arc;
|
|
12
|
-
|
|
13
|
-
#[cfg(not(feature = "tokio-runtime"))]
|
|
14
|
-
use crate::KreuzbergError;
|
|
15
|
-
|
|
16
|
-
/// Trait for document extractor plugins.
|
|
17
|
-
///
|
|
18
|
-
/// Implement this trait to add support for new document formats or to override
|
|
19
|
-
/// built-in extraction behavior with custom logic.
|
|
20
|
-
///
|
|
21
|
-
/// # Priority System
|
|
22
|
-
///
|
|
23
|
-
/// When multiple extractors support the same MIME type, the registry selects
|
|
24
|
-
/// the extractor with the highest priority value. Use this to:
|
|
25
|
-
/// - Override built-in extractors (priority > 50)
|
|
26
|
-
/// - Provide fallback extractors (priority < 50)
|
|
27
|
-
/// - Implement specialized extractors for specific use cases
|
|
28
|
-
///
|
|
29
|
-
/// Default priority is 50.
|
|
30
|
-
///
|
|
31
|
-
/// # Thread Safety
|
|
32
|
-
///
|
|
33
|
-
/// Extractors must be thread-safe (`Send + Sync`) to support concurrent extraction.
|
|
34
|
-
///
|
|
35
|
-
/// # Example
|
|
36
|
-
///
|
|
37
|
-
/// ```rust
|
|
38
|
-
/// use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
39
|
-
/// use kreuzberg::{Result, ExtractionConfig};
|
|
40
|
-
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
41
|
-
/// use async_trait::async_trait;
|
|
42
|
-
/// use std::path::Path;
|
|
43
|
-
///
|
|
44
|
-
/// /// Custom PDF extractor with premium features
|
|
45
|
-
/// struct PremiumPdfExtractor;
|
|
46
|
-
///
|
|
47
|
-
/// impl Plugin for PremiumPdfExtractor {
|
|
48
|
-
/// fn name(&self) -> &str { "premium-pdf" }
|
|
49
|
-
/// fn version(&self) -> String { "2.0.0".to_string() }
|
|
50
|
-
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
51
|
-
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
52
|
-
/// }
|
|
53
|
-
///
|
|
54
|
-
/// #[async_trait]
|
|
55
|
-
/// impl DocumentExtractor for PremiumPdfExtractor {
|
|
56
|
-
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
|
|
57
|
-
/// -> Result<ExtractionResult> {
|
|
58
|
-
/// // Premium extraction logic with better accuracy
|
|
59
|
-
/// Ok(ExtractionResult {
|
|
60
|
-
/// content: "Premium extracted content".to_string(),
|
|
61
|
-
/// mime_type: mime_type.to_string(),
|
|
62
|
-
/// metadata: Metadata::default(),
|
|
63
|
-
/// tables: vec![],
|
|
64
|
-
/// detected_languages: None,
|
|
65
|
-
/// chunks: None,
|
|
66
|
-
/// images: None,
|
|
67
|
-
/// pages: None,
|
|
68
|
-
/// })
|
|
69
|
-
/// }
|
|
70
|
-
///
|
|
71
|
-
/// async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig)
|
|
72
|
-
/// -> Result<ExtractionResult> {
|
|
73
|
-
/// let bytes = std::fs::read(path)?;
|
|
74
|
-
/// self.extract_bytes(&bytes, mime_type, config).await
|
|
75
|
-
/// }
|
|
76
|
-
///
|
|
77
|
-
/// fn supported_mime_types(&self) -> &[&str] {
|
|
78
|
-
/// &["application/pdf"]
|
|
79
|
-
/// }
|
|
80
|
-
///
|
|
81
|
-
/// fn priority(&self) -> i32 {
|
|
82
|
-
/// 100 // Higher than default (50) - will be preferred
|
|
83
|
-
/// }
|
|
84
|
-
/// }
|
|
85
|
-
/// ```
|
|
86
|
-
#[async_trait]
|
|
87
|
-
pub trait DocumentExtractor: Plugin {
|
|
88
|
-
/// Extract content from a byte array.
|
|
89
|
-
///
|
|
90
|
-
/// This is the core extraction method that processes in-memory document data.
|
|
91
|
-
///
|
|
92
|
-
/// # Arguments
|
|
93
|
-
///
|
|
94
|
-
/// * `content` - Raw document bytes
|
|
95
|
-
/// * `mime_type` - MIME type of the document (already validated)
|
|
96
|
-
/// * `config` - Extraction configuration
|
|
97
|
-
///
|
|
98
|
-
/// # Returns
|
|
99
|
-
///
|
|
100
|
-
/// An `ExtractionResult` containing the extracted content, metadata, and tables.
|
|
101
|
-
///
|
|
102
|
-
/// # Errors
|
|
103
|
-
///
|
|
104
|
-
/// - `KreuzbergError::Parsing` - Document parsing failed
|
|
105
|
-
/// - `KreuzbergError::Validation` - Invalid document structure
|
|
106
|
-
/// - `KreuzbergError::Io` - I/O errors (these always bubble up)
|
|
107
|
-
/// - `KreuzbergError::MissingDependency` - Required dependency not available
|
|
108
|
-
///
|
|
109
|
-
/// # Example
|
|
110
|
-
///
|
|
111
|
-
/// ```rust,no_run
|
|
112
|
-
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
113
|
-
/// # use kreuzberg::{Result, ExtractionConfig};
|
|
114
|
-
/// # use kreuzberg::types::{ExtractionResult, Metadata};
|
|
115
|
-
/// # use async_trait::async_trait;
|
|
116
|
-
/// # use std::path::Path;
|
|
117
|
-
/// # struct MyExtractor;
|
|
118
|
-
/// # impl Plugin for MyExtractor {
|
|
119
|
-
/// # fn name(&self) -> &str { "my-extractor" }
|
|
120
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
121
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
122
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
123
|
-
/// # }
|
|
124
|
-
/// # #[async_trait]
|
|
125
|
-
/// # impl DocumentExtractor for MyExtractor {
|
|
126
|
-
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
127
|
-
/// # fn priority(&self) -> i32 { 50 }
|
|
128
|
-
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
129
|
-
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
|
|
130
|
-
/// -> Result<ExtractionResult> {
|
|
131
|
-
/// // Parse document
|
|
132
|
-
/// let text = String::from_utf8_lossy(content).to_string();
|
|
133
|
-
///
|
|
134
|
-
/// // Extract metadata
|
|
135
|
-
/// let mut metadata = Metadata::default();
|
|
136
|
-
/// metadata.additional.insert("byte_count".to_string(), serde_json::json!(content.len()));
|
|
137
|
-
///
|
|
138
|
-
/// Ok(ExtractionResult {
|
|
139
|
-
/// content: text,
|
|
140
|
-
/// mime_type: mime_type.to_string(),
|
|
141
|
-
/// metadata,
|
|
142
|
-
/// tables: vec![],
|
|
143
|
-
/// detected_languages: None,
|
|
144
|
-
/// chunks: None,
|
|
145
|
-
/// images: None,
|
|
146
|
-
/// pages: None,
|
|
147
|
-
/// })
|
|
148
|
-
/// }
|
|
149
|
-
/// # }
|
|
150
|
-
/// ```
|
|
151
|
-
async fn extract_bytes(
|
|
152
|
-
&self,
|
|
153
|
-
content: &[u8],
|
|
154
|
-
mime_type: &str,
|
|
155
|
-
config: &ExtractionConfig,
|
|
156
|
-
) -> Result<ExtractionResult>;
|
|
157
|
-
|
|
158
|
-
/// Extract content from a file.
|
|
159
|
-
///
|
|
160
|
-
/// Default implementation reads the file and calls `extract_bytes`.
|
|
161
|
-
/// Override for custom file handling, streaming, or memory optimizations.
|
|
162
|
-
///
|
|
163
|
-
/// # Arguments
|
|
164
|
-
///
|
|
165
|
-
/// * `path` - Path to the document file
|
|
166
|
-
/// * `mime_type` - MIME type of the document (already validated)
|
|
167
|
-
/// * `config` - Extraction configuration
|
|
168
|
-
///
|
|
169
|
-
/// # Errors
|
|
170
|
-
///
|
|
171
|
-
/// Same as `extract_bytes`, plus file I/O errors.
|
|
172
|
-
///
|
|
173
|
-
/// # Example - Custom File Handling
|
|
174
|
-
///
|
|
175
|
-
/// ```rust,no_run
|
|
176
|
-
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
177
|
-
/// # use kreuzberg::{Result, ExtractionResult, ExtractionConfig};
|
|
178
|
-
/// # use kreuzberg::types::Metadata;
|
|
179
|
-
/// # use async_trait::async_trait;
|
|
180
|
-
/// # use std::path::Path;
|
|
181
|
-
/// # struct StreamingExtractor;
|
|
182
|
-
/// # impl Plugin for StreamingExtractor {
|
|
183
|
-
/// # fn name(&self) -> &str { "streaming" }
|
|
184
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
185
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
186
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
187
|
-
/// # }
|
|
188
|
-
/// # #[async_trait]
|
|
189
|
-
/// # impl DocumentExtractor for StreamingExtractor {
|
|
190
|
-
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
191
|
-
/// # fn priority(&self) -> i32 { 50 }
|
|
192
|
-
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
193
|
-
/// /// Override for memory-efficient streaming extraction
|
|
194
|
-
/// async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig)
|
|
195
|
-
/// -> Result<ExtractionResult> {
|
|
196
|
-
/// // Stream large files instead of loading entirely into memory
|
|
197
|
-
/// let mut content = String::new();
|
|
198
|
-
///
|
|
199
|
-
/// // Use buffered reader for streaming
|
|
200
|
-
/// use std::io::{BufRead, BufReader};
|
|
201
|
-
/// let file = std::fs::File::open(path)?;
|
|
202
|
-
/// let reader = BufReader::new(file);
|
|
203
|
-
///
|
|
204
|
-
/// for line in reader.lines() {
|
|
205
|
-
/// content.push_str(&line?);
|
|
206
|
-
/// content.push('\n');
|
|
207
|
-
/// }
|
|
208
|
-
///
|
|
209
|
-
/// Ok(ExtractionResult {
|
|
210
|
-
/// content,
|
|
211
|
-
/// mime_type: mime_type.to_string(),
|
|
212
|
-
/// metadata: Metadata::default(),
|
|
213
|
-
/// tables: vec![],
|
|
214
|
-
/// detected_languages: None,
|
|
215
|
-
/// chunks: None,
|
|
216
|
-
/// images: None,
|
|
217
|
-
/// pages: None,
|
|
218
|
-
/// })
|
|
219
|
-
/// }
|
|
220
|
-
/// # }
|
|
221
|
-
/// ```
|
|
222
|
-
async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
223
|
-
#[cfg(feature = "tokio-runtime")]
|
|
224
|
-
{
|
|
225
|
-
use crate::core::io;
|
|
226
|
-
let bytes = io::read_file_async(path).await?;
|
|
227
|
-
self.extract_bytes(&bytes, mime_type, config).await
|
|
228
|
-
}
|
|
229
|
-
#[cfg(not(feature = "tokio-runtime"))]
|
|
230
|
-
{
|
|
231
|
-
let _ = (path, mime_type, config);
|
|
232
|
-
// For WASM and non-tokio environments, file extraction is not supported
|
|
233
|
-
// through the default implementation. Implementations must provide their own.
|
|
234
|
-
Err(KreuzbergError::Other(
|
|
235
|
-
"File-based extraction requires the tokio-runtime feature".to_string(),
|
|
236
|
-
))
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
/// Get the list of MIME types supported by this extractor.
|
|
241
|
-
///
|
|
242
|
-
/// Can include exact MIME types and prefix patterns:
|
|
243
|
-
/// - Exact: `"application/pdf"`, `"text/plain"`
|
|
244
|
-
/// - Prefix: `"image/*"` (matches any image type)
|
|
245
|
-
///
|
|
246
|
-
/// # Returns
|
|
247
|
-
///
|
|
248
|
-
/// A slice of MIME type strings.
|
|
249
|
-
///
|
|
250
|
-
/// # Example
|
|
251
|
-
///
|
|
252
|
-
/// ```rust
|
|
253
|
-
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
254
|
-
/// # use kreuzberg::Result;
|
|
255
|
-
/// # use async_trait::async_trait;
|
|
256
|
-
/// # use std::path::Path;
|
|
257
|
-
/// # struct MultiFormatExtractor;
|
|
258
|
-
/// # impl Plugin for MultiFormatExtractor {
|
|
259
|
-
/// # fn name(&self) -> &str { "multi-format" }
|
|
260
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
261
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
262
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
263
|
-
/// # }
|
|
264
|
-
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
265
|
-
/// # #[async_trait]
|
|
266
|
-
/// # impl DocumentExtractor for MultiFormatExtractor {
|
|
267
|
-
/// # fn priority(&self) -> i32 { 50 }
|
|
268
|
-
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
269
|
-
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
270
|
-
/// fn supported_mime_types(&self) -> &[&str] {
|
|
271
|
-
/// &[
|
|
272
|
-
/// "text/plain",
|
|
273
|
-
/// "text/markdown",
|
|
274
|
-
/// "application/json",
|
|
275
|
-
/// "application/xml",
|
|
276
|
-
/// "text/html",
|
|
277
|
-
/// ]
|
|
278
|
-
/// }
|
|
279
|
-
/// # }
|
|
280
|
-
/// ```
|
|
281
|
-
fn supported_mime_types(&self) -> &[&str];
|
|
282
|
-
|
|
283
|
-
/// Get the priority of this extractor.
|
|
284
|
-
///
|
|
285
|
-
/// Higher priority extractors are preferred when multiple extractors
|
|
286
|
-
/// support the same MIME type.
|
|
287
|
-
///
|
|
288
|
-
/// # Priority Guidelines
|
|
289
|
-
///
|
|
290
|
-
/// - **0-25**: Fallback/low-quality extractors
|
|
291
|
-
/// - **26-49**: Alternative extractors
|
|
292
|
-
/// - **50**: Default priority (built-in extractors)
|
|
293
|
-
/// - **51-75**: Premium/enhanced extractors
|
|
294
|
-
/// - **76-100**: Specialized/high-priority extractors
|
|
295
|
-
///
|
|
296
|
-
/// # Returns
|
|
297
|
-
///
|
|
298
|
-
/// Priority value (default: 50)
|
|
299
|
-
///
|
|
300
|
-
/// # Example
|
|
301
|
-
///
|
|
302
|
-
/// ```rust
|
|
303
|
-
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
304
|
-
/// # use kreuzberg::Result;
|
|
305
|
-
/// # use async_trait::async_trait;
|
|
306
|
-
/// # use std::path::Path;
|
|
307
|
-
/// # struct FallbackExtractor;
|
|
308
|
-
/// # impl Plugin for FallbackExtractor {
|
|
309
|
-
/// # fn name(&self) -> &str { "fallback" }
|
|
310
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
311
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
312
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
313
|
-
/// # }
|
|
314
|
-
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
315
|
-
/// # #[async_trait]
|
|
316
|
-
/// # impl DocumentExtractor for FallbackExtractor {
|
|
317
|
-
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
318
|
-
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
319
|
-
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
320
|
-
/// fn priority(&self) -> i32 {
|
|
321
|
-
/// 10 // Low priority - only used as fallback
|
|
322
|
-
/// }
|
|
323
|
-
/// # }
|
|
324
|
-
/// ```
|
|
325
|
-
fn priority(&self) -> i32 {
|
|
326
|
-
50
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
/// Optional: Check if this extractor can handle a specific file.
|
|
330
|
-
///
|
|
331
|
-
/// Allows for more sophisticated detection beyond MIME types.
|
|
332
|
-
/// Defaults to `true` (rely on MIME type matching).
|
|
333
|
-
///
|
|
334
|
-
/// # Arguments
|
|
335
|
-
///
|
|
336
|
-
/// * `path` - Path to the file to check
|
|
337
|
-
/// * `mime_type` - Detected MIME type
|
|
338
|
-
///
|
|
339
|
-
/// # Returns
|
|
340
|
-
///
|
|
341
|
-
/// `true` if the extractor can handle this file, `false` otherwise.
|
|
342
|
-
///
|
|
343
|
-
/// # Example
|
|
344
|
-
///
|
|
345
|
-
/// ```rust,no_run
|
|
346
|
-
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
347
|
-
/// # use kreuzberg::Result;
|
|
348
|
-
/// # use async_trait::async_trait;
|
|
349
|
-
/// # use std::path::Path;
|
|
350
|
-
/// # struct SmartExtractor;
|
|
351
|
-
/// # impl Plugin for SmartExtractor {
|
|
352
|
-
/// # fn name(&self) -> &str { "smart" }
|
|
353
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
354
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
355
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
356
|
-
/// # }
|
|
357
|
-
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
358
|
-
/// # #[async_trait]
|
|
359
|
-
/// # impl DocumentExtractor for SmartExtractor {
|
|
360
|
-
/// # fn supported_mime_types(&self) -> &[&str] { &["application/pdf"] }
|
|
361
|
-
/// # fn priority(&self) -> i32 { 50 }
|
|
362
|
-
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
363
|
-
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
364
|
-
/// /// Only handle PDFs that are searchable (have text layer)
|
|
365
|
-
/// fn can_handle(&self, path: &Path, mime_type: &str) -> bool {
|
|
366
|
-
/// if mime_type != "application/pdf" {
|
|
367
|
-
/// return false;
|
|
368
|
-
/// }
|
|
369
|
-
///
|
|
370
|
-
/// // Check if PDF has text layer (simplified example)
|
|
371
|
-
/// // In real implementation, analyze PDF structure here
|
|
372
|
-
/// let _ = path; // Use path for PDF analysis
|
|
373
|
-
/// true // Simplified - always accept
|
|
374
|
-
/// }
|
|
375
|
-
/// # }
|
|
376
|
-
/// ```
|
|
377
|
-
fn can_handle(&self, _path: &Path, _mime_type: &str) -> bool {
|
|
378
|
-
true
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/// Attempt to get a reference to this extractor as a SyncExtractor.
|
|
382
|
-
///
|
|
383
|
-
/// Returns None if the extractor doesn't support synchronous extraction.
|
|
384
|
-
/// This is used for WASM and other sync-only environments.
|
|
385
|
-
fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
|
|
386
|
-
None
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
/// Register a document extractor with the global registry.
|
|
391
|
-
///
|
|
392
|
-
/// The extractor will be registered for all MIME types it supports and will be
|
|
393
|
-
/// available for document extraction. The extractor's `name()` method is used as
|
|
394
|
-
/// the registration name.
|
|
395
|
-
///
|
|
396
|
-
/// # Arguments
|
|
397
|
-
///
|
|
398
|
-
/// * `extractor` - The extractor implementation wrapped in Arc
|
|
399
|
-
///
|
|
400
|
-
/// # Returns
|
|
401
|
-
///
|
|
402
|
-
/// - `Ok(())` if registration succeeded
|
|
403
|
-
/// - `Err(...)` if validation failed or initialization failed
|
|
404
|
-
///
|
|
405
|
-
/// # Errors
|
|
406
|
-
///
|
|
407
|
-
/// - `KreuzbergError::Validation` - Invalid extractor name (empty or contains whitespace)
|
|
408
|
-
/// - Any error from the extractor's `initialize()` method
|
|
409
|
-
///
|
|
410
|
-
/// # Example
|
|
411
|
-
///
|
|
412
|
-
/// ```rust
|
|
413
|
-
/// use kreuzberg::plugins::{Plugin, DocumentExtractor, register_extractor};
|
|
414
|
-
/// use kreuzberg::{Result, ExtractionConfig};
|
|
415
|
-
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
416
|
-
/// use async_trait::async_trait;
|
|
417
|
-
/// use std::sync::Arc;
|
|
418
|
-
/// use std::path::Path;
|
|
419
|
-
///
|
|
420
|
-
/// struct CustomExtractor;
|
|
421
|
-
///
|
|
422
|
-
/// impl Plugin for CustomExtractor {
|
|
423
|
-
/// fn name(&self) -> &str { "custom-extractor" }
|
|
424
|
-
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
425
|
-
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
426
|
-
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
427
|
-
/// }
|
|
428
|
-
///
|
|
429
|
-
/// #[async_trait]
|
|
430
|
-
/// impl DocumentExtractor for CustomExtractor {
|
|
431
|
-
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, _: &ExtractionConfig)
|
|
432
|
-
/// -> Result<ExtractionResult> {
|
|
433
|
-
/// Ok(ExtractionResult {
|
|
434
|
-
/// content: String::from_utf8_lossy(content).to_string(),
|
|
435
|
-
/// mime_type: mime_type.to_string(),
|
|
436
|
-
/// metadata: Metadata::default(),
|
|
437
|
-
/// tables: vec![],
|
|
438
|
-
/// detected_languages: None,
|
|
439
|
-
/// chunks: None,
|
|
440
|
-
/// images: None,
|
|
441
|
-
/// pages: None,
|
|
442
|
-
/// })
|
|
443
|
-
/// }
|
|
444
|
-
///
|
|
445
|
-
/// fn supported_mime_types(&self) -> &[&str] {
|
|
446
|
-
/// &["text/custom"]
|
|
447
|
-
/// }
|
|
448
|
-
/// }
|
|
449
|
-
///
|
|
450
|
-
/// # tokio_test::block_on(async {
|
|
451
|
-
/// let extractor = Arc::new(CustomExtractor);
|
|
452
|
-
/// register_extractor(extractor)?;
|
|
453
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
454
|
-
/// # });
|
|
455
|
-
/// ```
|
|
456
|
-
pub fn register_extractor(extractor: Arc<dyn DocumentExtractor>) -> crate::Result<()> {
|
|
457
|
-
use crate::plugins::registry::get_document_extractor_registry;
|
|
458
|
-
|
|
459
|
-
let registry = get_document_extractor_registry();
|
|
460
|
-
let mut registry = registry
|
|
461
|
-
.write()
|
|
462
|
-
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
463
|
-
|
|
464
|
-
registry.register(extractor)
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
/// Unregister a document extractor by name.
|
|
468
|
-
///
|
|
469
|
-
/// Removes the extractor from the global registry and calls its `shutdown()` method.
|
|
470
|
-
///
|
|
471
|
-
/// # Arguments
|
|
472
|
-
///
|
|
473
|
-
/// * `name` - Name of the extractor to unregister
|
|
474
|
-
///
|
|
475
|
-
/// # Returns
|
|
476
|
-
///
|
|
477
|
-
/// - `Ok(())` if the extractor was unregistered or didn't exist
|
|
478
|
-
/// - `Err(...)` if the shutdown method failed
|
|
479
|
-
///
|
|
480
|
-
/// # Example
|
|
481
|
-
///
|
|
482
|
-
/// ```rust
|
|
483
|
-
/// use kreuzberg::plugins::unregister_extractor;
|
|
484
|
-
///
|
|
485
|
-
/// # tokio_test::block_on(async {
|
|
486
|
-
/// unregister_extractor("custom-extractor")?;
|
|
487
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
488
|
-
/// # });
|
|
489
|
-
/// ```
|
|
490
|
-
pub fn unregister_extractor(name: &str) -> crate::Result<()> {
|
|
491
|
-
use crate::plugins::registry::get_document_extractor_registry;
|
|
492
|
-
|
|
493
|
-
let registry = get_document_extractor_registry();
|
|
494
|
-
let mut registry = registry
|
|
495
|
-
.write()
|
|
496
|
-
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
497
|
-
|
|
498
|
-
registry.remove(name)
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
/// List all registered extractors.
|
|
502
|
-
///
|
|
503
|
-
/// Returns the names of all extractors currently registered in the global registry.
|
|
504
|
-
///
|
|
505
|
-
/// # Returns
|
|
506
|
-
///
|
|
507
|
-
/// A vector of extractor names.
|
|
508
|
-
///
|
|
509
|
-
/// # Example
|
|
510
|
-
///
|
|
511
|
-
/// ```rust
|
|
512
|
-
/// use kreuzberg::plugins::list_extractors;
|
|
513
|
-
///
|
|
514
|
-
/// # tokio_test::block_on(async {
|
|
515
|
-
/// let extractors = list_extractors()?;
|
|
516
|
-
/// for name in extractors {
|
|
517
|
-
/// println!("Registered extractor: {}", name);
|
|
518
|
-
/// }
|
|
519
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
520
|
-
/// # });
|
|
521
|
-
/// ```
|
|
522
|
-
pub fn list_extractors() -> crate::Result<Vec<String>> {
|
|
523
|
-
use crate::plugins::registry::get_document_extractor_registry;
|
|
524
|
-
|
|
525
|
-
let registry = get_document_extractor_registry();
|
|
526
|
-
let registry = registry
|
|
527
|
-
.read()
|
|
528
|
-
.expect("~keep Failed to acquire read lock on extractor registry"); // ~keep
|
|
529
|
-
|
|
530
|
-
Ok(registry.list())
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
/// Clear all extractors from the global registry.
|
|
534
|
-
///
|
|
535
|
-
/// Removes all extractors and calls their `shutdown()` methods.
|
|
536
|
-
///
|
|
537
|
-
/// # Returns
|
|
538
|
-
///
|
|
539
|
-
/// - `Ok(())` if all extractors were cleared successfully
|
|
540
|
-
/// - `Err(...)` if any shutdown method failed
|
|
541
|
-
///
|
|
542
|
-
/// # Example
|
|
543
|
-
///
|
|
544
|
-
/// ```rust
|
|
545
|
-
/// use kreuzberg::plugins::clear_extractors;
|
|
546
|
-
///
|
|
547
|
-
/// # tokio_test::block_on(async {
|
|
548
|
-
/// clear_extractors()?;
|
|
549
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
550
|
-
/// # });
|
|
551
|
-
/// ```
|
|
552
|
-
pub fn clear_extractors() -> crate::Result<()> {
|
|
553
|
-
use crate::plugins::registry::get_document_extractor_registry;
|
|
554
|
-
|
|
555
|
-
let registry = get_document_extractor_registry();
|
|
556
|
-
let mut registry = registry
|
|
557
|
-
.write()
|
|
558
|
-
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
559
|
-
|
|
560
|
-
registry.shutdown_all()
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
#[cfg(test)]
|
|
564
|
-
mod tests {
|
|
565
|
-
use super::*;
|
|
566
|
-
use serial_test::serial;
|
|
567
|
-
|
|
568
|
-
struct MockExtractor {
|
|
569
|
-
mime_types: Vec<&'static str>,
|
|
570
|
-
priority: i32,
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
impl Plugin for MockExtractor {
|
|
574
|
-
fn name(&self) -> &str {
|
|
575
|
-
"mock-extractor"
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
fn version(&self) -> String {
|
|
579
|
-
"1.0.0".to_string()
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
fn initialize(&self) -> Result<()> {
|
|
583
|
-
Ok(())
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
fn shutdown(&self) -> Result<()> {
|
|
587
|
-
Ok(())
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
#[async_trait]
|
|
592
|
-
impl DocumentExtractor for MockExtractor {
|
|
593
|
-
async fn extract_bytes(
|
|
594
|
-
&self,
|
|
595
|
-
content: &[u8],
|
|
596
|
-
mime_type: &str,
|
|
597
|
-
_config: &ExtractionConfig,
|
|
598
|
-
) -> Result<ExtractionResult> {
|
|
599
|
-
Ok(ExtractionResult {
|
|
600
|
-
content: String::from_utf8_lossy(content).to_string(),
|
|
601
|
-
mime_type: mime_type.to_string(),
|
|
602
|
-
metadata: crate::types::Metadata::default(),
|
|
603
|
-
tables: vec![],
|
|
604
|
-
detected_languages: None,
|
|
605
|
-
chunks: None,
|
|
606
|
-
images: None,
|
|
607
|
-
pages: None,
|
|
608
|
-
})
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
fn supported_mime_types(&self) -> &[&str] {
|
|
612
|
-
&self.mime_types
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
fn priority(&self) -> i32 {
|
|
616
|
-
self.priority
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
#[tokio::test]
|
|
621
|
-
async fn test_document_extractor_extract_bytes() {
|
|
622
|
-
let extractor = MockExtractor {
|
|
623
|
-
mime_types: vec!["text/plain"],
|
|
624
|
-
priority: 50,
|
|
625
|
-
};
|
|
626
|
-
|
|
627
|
-
let config = ExtractionConfig::default();
|
|
628
|
-
let result = extractor
|
|
629
|
-
.extract_bytes(b"test content", "text/plain", &config)
|
|
630
|
-
.await
|
|
631
|
-
.unwrap();
|
|
632
|
-
|
|
633
|
-
assert_eq!(result.content, "test content");
|
|
634
|
-
assert_eq!(result.mime_type, "text/plain");
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
#[test]
|
|
638
|
-
fn test_document_extractor_supported_mime_types() {
|
|
639
|
-
let extractor = MockExtractor {
|
|
640
|
-
mime_types: vec!["text/plain", "text/markdown"],
|
|
641
|
-
priority: 50,
|
|
642
|
-
};
|
|
643
|
-
|
|
644
|
-
let supported = extractor.supported_mime_types();
|
|
645
|
-
assert_eq!(supported.len(), 2);
|
|
646
|
-
assert!(supported.contains(&"text/plain"));
|
|
647
|
-
assert!(supported.contains(&"text/markdown"));
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
#[test]
|
|
651
|
-
fn test_document_extractor_priority() {
|
|
652
|
-
let low_priority = MockExtractor {
|
|
653
|
-
mime_types: vec!["text/plain"],
|
|
654
|
-
priority: 10,
|
|
655
|
-
};
|
|
656
|
-
|
|
657
|
-
let high_priority = MockExtractor {
|
|
658
|
-
mime_types: vec!["text/plain"],
|
|
659
|
-
priority: 100,
|
|
660
|
-
};
|
|
661
|
-
|
|
662
|
-
assert_eq!(low_priority.priority(), 10);
|
|
663
|
-
assert_eq!(high_priority.priority(), 100);
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
#[test]
|
|
667
|
-
fn test_document_extractor_can_handle_default() {
|
|
668
|
-
let extractor = MockExtractor {
|
|
669
|
-
mime_types: vec!["text/plain"],
|
|
670
|
-
priority: 50,
|
|
671
|
-
};
|
|
672
|
-
|
|
673
|
-
use std::path::PathBuf;
|
|
674
|
-
let path = PathBuf::from("test.txt");
|
|
675
|
-
|
|
676
|
-
assert!(extractor.can_handle(&path, "text/plain"));
|
|
677
|
-
assert!(extractor.can_handle(&path, "application/pdf"));
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
#[tokio::test]
|
|
681
|
-
async fn test_document_extractor_extract_file_default_impl() {
|
|
682
|
-
use std::io::Write;
|
|
683
|
-
use tempfile::NamedTempFile;
|
|
684
|
-
|
|
685
|
-
let extractor = MockExtractor {
|
|
686
|
-
mime_types: vec!["text/plain"],
|
|
687
|
-
priority: 50,
|
|
688
|
-
};
|
|
689
|
-
|
|
690
|
-
let mut temp_file = NamedTempFile::new().unwrap();
|
|
691
|
-
temp_file.write_all(b"file content").unwrap();
|
|
692
|
-
let path = temp_file.path();
|
|
693
|
-
|
|
694
|
-
let config = ExtractionConfig::default();
|
|
695
|
-
let result = extractor.extract_file(path, "text/plain", &config).await.unwrap();
|
|
696
|
-
|
|
697
|
-
assert_eq!(result.content, "file content");
|
|
698
|
-
assert_eq!(result.mime_type, "text/plain");
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
#[tokio::test]
|
|
702
|
-
async fn test_document_extractor_empty_content() {
|
|
703
|
-
let extractor = MockExtractor {
|
|
704
|
-
mime_types: vec!["text/plain"],
|
|
705
|
-
priority: 50,
|
|
706
|
-
};
|
|
707
|
-
|
|
708
|
-
let config = ExtractionConfig::default();
|
|
709
|
-
let result = extractor.extract_bytes(b"", "text/plain", &config).await.unwrap();
|
|
710
|
-
|
|
711
|
-
assert_eq!(result.content, "");
|
|
712
|
-
assert_eq!(result.mime_type, "text/plain");
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
#[tokio::test]
|
|
716
|
-
async fn test_document_extractor_non_utf8_content() {
|
|
717
|
-
let extractor = MockExtractor {
|
|
718
|
-
mime_types: vec!["text/plain"],
|
|
719
|
-
priority: 50,
|
|
720
|
-
};
|
|
721
|
-
|
|
722
|
-
let invalid_utf8 = vec![0xFF, 0xFE, 0xFD];
|
|
723
|
-
let config = ExtractionConfig::default();
|
|
724
|
-
let result = extractor
|
|
725
|
-
.extract_bytes(&invalid_utf8, "text/plain", &config)
|
|
726
|
-
.await
|
|
727
|
-
.unwrap();
|
|
728
|
-
|
|
729
|
-
assert!(!result.content.is_empty());
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
#[test]
|
|
733
|
-
fn test_document_extractor_plugin_interface() {
|
|
734
|
-
let extractor = MockExtractor {
|
|
735
|
-
mime_types: vec!["text/plain"],
|
|
736
|
-
priority: 50,
|
|
737
|
-
};
|
|
738
|
-
|
|
739
|
-
assert_eq!(extractor.name(), "mock-extractor");
|
|
740
|
-
assert_eq!(extractor.version(), "1.0.0");
|
|
741
|
-
assert!(extractor.initialize().is_ok());
|
|
742
|
-
assert!(extractor.shutdown().is_ok());
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
#[test]
|
|
746
|
-
fn test_document_extractor_default_priority() {
|
|
747
|
-
struct DefaultPriorityExtractor;
|
|
748
|
-
|
|
749
|
-
impl Plugin for DefaultPriorityExtractor {
|
|
750
|
-
fn name(&self) -> &str {
|
|
751
|
-
"default"
|
|
752
|
-
}
|
|
753
|
-
fn version(&self) -> String {
|
|
754
|
-
"1.0.0".to_string()
|
|
755
|
-
}
|
|
756
|
-
fn initialize(&self) -> Result<()> {
|
|
757
|
-
Ok(())
|
|
758
|
-
}
|
|
759
|
-
fn shutdown(&self) -> Result<()> {
|
|
760
|
-
Ok(())
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
#[async_trait]
|
|
765
|
-
impl DocumentExtractor for DefaultPriorityExtractor {
|
|
766
|
-
async fn extract_bytes(
|
|
767
|
-
&self,
|
|
768
|
-
_content: &[u8],
|
|
769
|
-
_mime_type: &str,
|
|
770
|
-
_config: &ExtractionConfig,
|
|
771
|
-
) -> Result<ExtractionResult> {
|
|
772
|
-
Ok(ExtractionResult {
|
|
773
|
-
content: String::new(),
|
|
774
|
-
mime_type: String::new(),
|
|
775
|
-
metadata: crate::types::Metadata::default(),
|
|
776
|
-
tables: vec![],
|
|
777
|
-
detected_languages: None,
|
|
778
|
-
chunks: None,
|
|
779
|
-
images: None,
|
|
780
|
-
pages: None,
|
|
781
|
-
})
|
|
782
|
-
}
|
|
783
|
-
|
|
784
|
-
fn supported_mime_types(&self) -> &[&str] {
|
|
785
|
-
&["text/plain"]
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
|
|
789
|
-
let extractor = DefaultPriorityExtractor;
|
|
790
|
-
assert_eq!(extractor.priority(), 50);
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
#[test]
|
|
794
|
-
fn test_document_extractor_empty_mime_types() {
|
|
795
|
-
let extractor = MockExtractor {
|
|
796
|
-
mime_types: vec![],
|
|
797
|
-
priority: 50,
|
|
798
|
-
};
|
|
799
|
-
|
|
800
|
-
assert_eq!(extractor.supported_mime_types().len(), 0);
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
#[test]
|
|
804
|
-
fn test_document_extractor_wildcard_mime_types() {
|
|
805
|
-
let extractor = MockExtractor {
|
|
806
|
-
mime_types: vec!["text/*", "image/*"],
|
|
807
|
-
priority: 50,
|
|
808
|
-
};
|
|
809
|
-
|
|
810
|
-
let supported = extractor.supported_mime_types();
|
|
811
|
-
assert_eq!(supported.len(), 2);
|
|
812
|
-
assert!(supported.contains(&"text/*"));
|
|
813
|
-
assert!(supported.contains(&"image/*"));
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
#[test]
|
|
817
|
-
fn test_document_extractor_priority_ranges() {
|
|
818
|
-
let fallback = MockExtractor {
|
|
819
|
-
mime_types: vec!["text/plain"],
|
|
820
|
-
priority: 10,
|
|
821
|
-
};
|
|
822
|
-
let alternative = MockExtractor {
|
|
823
|
-
mime_types: vec!["text/plain"],
|
|
824
|
-
priority: 40,
|
|
825
|
-
};
|
|
826
|
-
let default = MockExtractor {
|
|
827
|
-
mime_types: vec!["text/plain"],
|
|
828
|
-
priority: 50,
|
|
829
|
-
};
|
|
830
|
-
let premium = MockExtractor {
|
|
831
|
-
mime_types: vec!["text/plain"],
|
|
832
|
-
priority: 75,
|
|
833
|
-
};
|
|
834
|
-
let specialized = MockExtractor {
|
|
835
|
-
mime_types: vec!["text/plain"],
|
|
836
|
-
priority: 100,
|
|
837
|
-
};
|
|
838
|
-
|
|
839
|
-
assert!(fallback.priority() < alternative.priority());
|
|
840
|
-
assert!(alternative.priority() < default.priority());
|
|
841
|
-
assert!(default.priority() < premium.priority());
|
|
842
|
-
assert!(premium.priority() < specialized.priority());
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
#[tokio::test]
|
|
846
|
-
async fn test_document_extractor_preserves_mime_type() {
|
|
847
|
-
let extractor = MockExtractor {
|
|
848
|
-
mime_types: vec!["application/json"],
|
|
849
|
-
priority: 50,
|
|
850
|
-
};
|
|
851
|
-
|
|
852
|
-
let config = ExtractionConfig::default();
|
|
853
|
-
let result = extractor
|
|
854
|
-
.extract_bytes(b"{\"key\":\"value\"}", "application/json", &config)
|
|
855
|
-
.await
|
|
856
|
-
.unwrap();
|
|
857
|
-
|
|
858
|
-
assert_eq!(result.mime_type, "application/json");
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
#[test]
|
|
862
|
-
#[serial]
|
|
863
|
-
fn test_register_extractor() {
|
|
864
|
-
use std::sync::Arc;
|
|
865
|
-
|
|
866
|
-
let extractor = Arc::new(MockExtractor {
|
|
867
|
-
mime_types: vec!["text/test-register"],
|
|
868
|
-
priority: 50,
|
|
869
|
-
});
|
|
870
|
-
let result = super::register_extractor(extractor);
|
|
871
|
-
assert!(result.is_ok());
|
|
872
|
-
|
|
873
|
-
let _ = super::unregister_extractor("mock-extractor");
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
#[test]
|
|
877
|
-
#[serial]
|
|
878
|
-
fn test_unregister_extractor() {
|
|
879
|
-
use std::sync::Arc;
|
|
880
|
-
|
|
881
|
-
let extractor = Arc::new(MockExtractor {
|
|
882
|
-
mime_types: vec!["text/test-unregister"],
|
|
883
|
-
priority: 50,
|
|
884
|
-
});
|
|
885
|
-
super::register_extractor(extractor).unwrap();
|
|
886
|
-
|
|
887
|
-
let result = super::unregister_extractor("mock-extractor");
|
|
888
|
-
assert!(result.is_ok());
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
#[test]
|
|
892
|
-
#[serial]
|
|
893
|
-
fn test_unregister_nonexistent_extractor() {
|
|
894
|
-
let result = super::unregister_extractor("nonexistent-extractor-xyz");
|
|
895
|
-
assert!(result.is_ok());
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
#[test]
|
|
899
|
-
#[serial]
|
|
900
|
-
fn test_list_extractors() {
|
|
901
|
-
use std::sync::Arc;
|
|
902
|
-
|
|
903
|
-
super::clear_extractors().unwrap();
|
|
904
|
-
|
|
905
|
-
let extractor1 = Arc::new(MockExtractor {
|
|
906
|
-
mime_types: vec!["text/test-list-1"],
|
|
907
|
-
priority: 50,
|
|
908
|
-
});
|
|
909
|
-
let extractor2 = Arc::new(MockExtractor {
|
|
910
|
-
mime_types: vec!["text/test-list-2"],
|
|
911
|
-
priority: 51,
|
|
912
|
-
});
|
|
913
|
-
|
|
914
|
-
let list_before = super::list_extractors().unwrap();
|
|
915
|
-
assert_eq!(list_before.len(), 0);
|
|
916
|
-
|
|
917
|
-
super::register_extractor(extractor1).unwrap();
|
|
918
|
-
super::register_extractor(extractor2).unwrap();
|
|
919
|
-
|
|
920
|
-
let list = super::list_extractors().unwrap();
|
|
921
|
-
assert_eq!(list.len(), 1);
|
|
922
|
-
assert!(list.contains(&"mock-extractor".to_string()));
|
|
923
|
-
|
|
924
|
-
super::unregister_extractor("mock-extractor").unwrap();
|
|
925
|
-
}
|
|
926
|
-
|
|
927
|
-
#[test]
|
|
928
|
-
#[serial]
|
|
929
|
-
fn test_clear_extractors() {
|
|
930
|
-
use std::sync::Arc;
|
|
931
|
-
|
|
932
|
-
super::clear_extractors().unwrap();
|
|
933
|
-
|
|
934
|
-
let extractor1 = Arc::new(MockExtractor {
|
|
935
|
-
mime_types: vec!["text/test-clear-1"],
|
|
936
|
-
priority: 50,
|
|
937
|
-
});
|
|
938
|
-
let extractor2 = Arc::new(MockExtractor {
|
|
939
|
-
mime_types: vec!["text/test-clear-2"],
|
|
940
|
-
priority: 51,
|
|
941
|
-
});
|
|
942
|
-
|
|
943
|
-
super::register_extractor(extractor1).unwrap();
|
|
944
|
-
super::register_extractor(extractor2).unwrap();
|
|
945
|
-
|
|
946
|
-
let result = super::clear_extractors();
|
|
947
|
-
assert!(result.is_ok());
|
|
948
|
-
|
|
949
|
-
let list = super::list_extractors().unwrap();
|
|
950
|
-
assert_eq!(list.len(), 0);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
#[test]
|
|
954
|
-
#[serial]
|
|
955
|
-
fn test_register_extractor_with_invalid_name() {
|
|
956
|
-
use std::sync::Arc;
|
|
957
|
-
|
|
958
|
-
struct InvalidNameExtractor;
|
|
959
|
-
impl Plugin for InvalidNameExtractor {
|
|
960
|
-
fn name(&self) -> &str {
|
|
961
|
-
"invalid name with spaces"
|
|
962
|
-
}
|
|
963
|
-
fn version(&self) -> String {
|
|
964
|
-
"1.0.0".to_string()
|
|
965
|
-
}
|
|
966
|
-
fn initialize(&self) -> Result<()> {
|
|
967
|
-
Ok(())
|
|
968
|
-
}
|
|
969
|
-
fn shutdown(&self) -> Result<()> {
|
|
970
|
-
Ok(())
|
|
971
|
-
}
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
#[async_trait]
|
|
975
|
-
impl DocumentExtractor for InvalidNameExtractor {
|
|
976
|
-
async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
977
|
-
Ok(ExtractionResult {
|
|
978
|
-
content: String::new(),
|
|
979
|
-
mime_type: String::new(),
|
|
980
|
-
metadata: crate::types::Metadata::default(),
|
|
981
|
-
tables: vec![],
|
|
982
|
-
detected_languages: None,
|
|
983
|
-
chunks: None,
|
|
984
|
-
images: None,
|
|
985
|
-
pages: None,
|
|
986
|
-
})
|
|
987
|
-
}
|
|
988
|
-
|
|
989
|
-
fn supported_mime_types(&self) -> &[&str] {
|
|
990
|
-
&["text/plain"]
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
let extractor = Arc::new(InvalidNameExtractor);
|
|
995
|
-
let result = super::register_extractor(extractor);
|
|
996
|
-
assert!(matches!(result, Err(crate::KreuzbergError::Validation { .. })));
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
#[test]
|
|
1000
|
-
#[serial]
|
|
1001
|
-
fn test_register_extractor_with_empty_name() {
|
|
1002
|
-
use std::sync::Arc;
|
|
1003
|
-
|
|
1004
|
-
struct EmptyNameExtractor;
|
|
1005
|
-
impl Plugin for EmptyNameExtractor {
|
|
1006
|
-
fn name(&self) -> &str {
|
|
1007
|
-
""
|
|
1008
|
-
}
|
|
1009
|
-
fn version(&self) -> String {
|
|
1010
|
-
"1.0.0".to_string()
|
|
1011
|
-
}
|
|
1012
|
-
fn initialize(&self) -> Result<()> {
|
|
1013
|
-
Ok(())
|
|
1014
|
-
}
|
|
1015
|
-
fn shutdown(&self) -> Result<()> {
|
|
1016
|
-
Ok(())
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
#[async_trait]
|
|
1021
|
-
impl DocumentExtractor for EmptyNameExtractor {
|
|
1022
|
-
async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
1023
|
-
Ok(ExtractionResult {
|
|
1024
|
-
content: String::new(),
|
|
1025
|
-
mime_type: String::new(),
|
|
1026
|
-
metadata: crate::types::Metadata::default(),
|
|
1027
|
-
tables: vec![],
|
|
1028
|
-
detected_languages: None,
|
|
1029
|
-
chunks: None,
|
|
1030
|
-
images: None,
|
|
1031
|
-
pages: None,
|
|
1032
|
-
})
|
|
1033
|
-
}
|
|
1034
|
-
|
|
1035
|
-
fn supported_mime_types(&self) -> &[&str] {
|
|
1036
|
-
&["text/plain"]
|
|
1037
|
-
}
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
let extractor = Arc::new(EmptyNameExtractor);
|
|
1041
|
-
let result = super::register_extractor(extractor);
|
|
1042
|
-
assert!(matches!(result, Err(crate::KreuzbergError::Validation { .. })));
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1
|
+
//! Document extractor plugin trait.
|
|
2
|
+
//!
|
|
3
|
+
//! This module defines the trait for implementing custom document extractors.
|
|
4
|
+
|
|
5
|
+
use crate::Result;
|
|
6
|
+
use crate::core::config::ExtractionConfig;
|
|
7
|
+
use crate::plugins::Plugin;
|
|
8
|
+
use crate::types::ExtractionResult;
|
|
9
|
+
use async_trait::async_trait;
|
|
10
|
+
use std::path::Path;
|
|
11
|
+
use std::sync::Arc;
|
|
12
|
+
|
|
13
|
+
#[cfg(not(feature = "tokio-runtime"))]
|
|
14
|
+
use crate::KreuzbergError;
|
|
15
|
+
|
|
16
|
+
/// Trait for document extractor plugins.
|
|
17
|
+
///
|
|
18
|
+
/// Implement this trait to add support for new document formats or to override
|
|
19
|
+
/// built-in extraction behavior with custom logic.
|
|
20
|
+
///
|
|
21
|
+
/// # Priority System
|
|
22
|
+
///
|
|
23
|
+
/// When multiple extractors support the same MIME type, the registry selects
|
|
24
|
+
/// the extractor with the highest priority value. Use this to:
|
|
25
|
+
/// - Override built-in extractors (priority > 50)
|
|
26
|
+
/// - Provide fallback extractors (priority < 50)
|
|
27
|
+
/// - Implement specialized extractors for specific use cases
|
|
28
|
+
///
|
|
29
|
+
/// Default priority is 50.
|
|
30
|
+
///
|
|
31
|
+
/// # Thread Safety
|
|
32
|
+
///
|
|
33
|
+
/// Extractors must be thread-safe (`Send + Sync`) to support concurrent extraction.
|
|
34
|
+
///
|
|
35
|
+
/// # Example
|
|
36
|
+
///
|
|
37
|
+
/// ```rust
|
|
38
|
+
/// use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
39
|
+
/// use kreuzberg::{Result, ExtractionConfig};
|
|
40
|
+
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
41
|
+
/// use async_trait::async_trait;
|
|
42
|
+
/// use std::path::Path;
|
|
43
|
+
///
|
|
44
|
+
/// /// Custom PDF extractor with premium features
|
|
45
|
+
/// struct PremiumPdfExtractor;
|
|
46
|
+
///
|
|
47
|
+
/// impl Plugin for PremiumPdfExtractor {
|
|
48
|
+
/// fn name(&self) -> &str { "premium-pdf" }
|
|
49
|
+
/// fn version(&self) -> String { "2.0.0".to_string() }
|
|
50
|
+
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
51
|
+
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
52
|
+
/// }
|
|
53
|
+
///
|
|
54
|
+
/// #[async_trait]
|
|
55
|
+
/// impl DocumentExtractor for PremiumPdfExtractor {
|
|
56
|
+
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
|
|
57
|
+
/// -> Result<ExtractionResult> {
|
|
58
|
+
/// // Premium extraction logic with better accuracy
|
|
59
|
+
/// Ok(ExtractionResult {
|
|
60
|
+
/// content: "Premium extracted content".to_string(),
|
|
61
|
+
/// mime_type: mime_type.to_string(),
|
|
62
|
+
/// metadata: Metadata::default(),
|
|
63
|
+
/// tables: vec![],
|
|
64
|
+
/// detected_languages: None,
|
|
65
|
+
/// chunks: None,
|
|
66
|
+
/// images: None,
|
|
67
|
+
/// pages: None,
|
|
68
|
+
/// })
|
|
69
|
+
/// }
|
|
70
|
+
///
|
|
71
|
+
/// async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig)
|
|
72
|
+
/// -> Result<ExtractionResult> {
|
|
73
|
+
/// let bytes = std::fs::read(path)?;
|
|
74
|
+
/// self.extract_bytes(&bytes, mime_type, config).await
|
|
75
|
+
/// }
|
|
76
|
+
///
|
|
77
|
+
/// fn supported_mime_types(&self) -> &[&str] {
|
|
78
|
+
/// &["application/pdf"]
|
|
79
|
+
/// }
|
|
80
|
+
///
|
|
81
|
+
/// fn priority(&self) -> i32 {
|
|
82
|
+
/// 100 // Higher than default (50) - will be preferred
|
|
83
|
+
/// }
|
|
84
|
+
/// }
|
|
85
|
+
/// ```
|
|
86
|
+
#[async_trait]
|
|
87
|
+
pub trait DocumentExtractor: Plugin {
|
|
88
|
+
/// Extract content from a byte array.
|
|
89
|
+
///
|
|
90
|
+
/// This is the core extraction method that processes in-memory document data.
|
|
91
|
+
///
|
|
92
|
+
/// # Arguments
|
|
93
|
+
///
|
|
94
|
+
/// * `content` - Raw document bytes
|
|
95
|
+
/// * `mime_type` - MIME type of the document (already validated)
|
|
96
|
+
/// * `config` - Extraction configuration
|
|
97
|
+
///
|
|
98
|
+
/// # Returns
|
|
99
|
+
///
|
|
100
|
+
/// An `ExtractionResult` containing the extracted content, metadata, and tables.
|
|
101
|
+
///
|
|
102
|
+
/// # Errors
|
|
103
|
+
///
|
|
104
|
+
/// - `KreuzbergError::Parsing` - Document parsing failed
|
|
105
|
+
/// - `KreuzbergError::Validation` - Invalid document structure
|
|
106
|
+
/// - `KreuzbergError::Io` - I/O errors (these always bubble up)
|
|
107
|
+
/// - `KreuzbergError::MissingDependency` - Required dependency not available
|
|
108
|
+
///
|
|
109
|
+
/// # Example
|
|
110
|
+
///
|
|
111
|
+
/// ```rust,no_run
|
|
112
|
+
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
113
|
+
/// # use kreuzberg::{Result, ExtractionConfig};
|
|
114
|
+
/// # use kreuzberg::types::{ExtractionResult, Metadata};
|
|
115
|
+
/// # use async_trait::async_trait;
|
|
116
|
+
/// # use std::path::Path;
|
|
117
|
+
/// # struct MyExtractor;
|
|
118
|
+
/// # impl Plugin for MyExtractor {
|
|
119
|
+
/// # fn name(&self) -> &str { "my-extractor" }
|
|
120
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
121
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
122
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
123
|
+
/// # }
|
|
124
|
+
/// # #[async_trait]
|
|
125
|
+
/// # impl DocumentExtractor for MyExtractor {
|
|
126
|
+
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
127
|
+
/// # fn priority(&self) -> i32 { 50 }
|
|
128
|
+
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
129
|
+
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
|
|
130
|
+
/// -> Result<ExtractionResult> {
|
|
131
|
+
/// // Parse document
|
|
132
|
+
/// let text = String::from_utf8_lossy(content).to_string();
|
|
133
|
+
///
|
|
134
|
+
/// // Extract metadata
|
|
135
|
+
/// let mut metadata = Metadata::default();
|
|
136
|
+
/// metadata.additional.insert("byte_count".to_string(), serde_json::json!(content.len()));
|
|
137
|
+
///
|
|
138
|
+
/// Ok(ExtractionResult {
|
|
139
|
+
/// content: text,
|
|
140
|
+
/// mime_type: mime_type.to_string(),
|
|
141
|
+
/// metadata,
|
|
142
|
+
/// tables: vec![],
|
|
143
|
+
/// detected_languages: None,
|
|
144
|
+
/// chunks: None,
|
|
145
|
+
/// images: None,
|
|
146
|
+
/// pages: None,
|
|
147
|
+
/// })
|
|
148
|
+
/// }
|
|
149
|
+
/// # }
|
|
150
|
+
/// ```
|
|
151
|
+
async fn extract_bytes(
|
|
152
|
+
&self,
|
|
153
|
+
content: &[u8],
|
|
154
|
+
mime_type: &str,
|
|
155
|
+
config: &ExtractionConfig,
|
|
156
|
+
) -> Result<ExtractionResult>;
|
|
157
|
+
|
|
158
|
+
/// Extract content from a file.
|
|
159
|
+
///
|
|
160
|
+
/// Default implementation reads the file and calls `extract_bytes`.
|
|
161
|
+
/// Override for custom file handling, streaming, or memory optimizations.
|
|
162
|
+
///
|
|
163
|
+
/// # Arguments
|
|
164
|
+
///
|
|
165
|
+
/// * `path` - Path to the document file
|
|
166
|
+
/// * `mime_type` - MIME type of the document (already validated)
|
|
167
|
+
/// * `config` - Extraction configuration
|
|
168
|
+
///
|
|
169
|
+
/// # Errors
|
|
170
|
+
///
|
|
171
|
+
/// Same as `extract_bytes`, plus file I/O errors.
|
|
172
|
+
///
|
|
173
|
+
/// # Example - Custom File Handling
|
|
174
|
+
///
|
|
175
|
+
/// ```rust,no_run
|
|
176
|
+
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
177
|
+
/// # use kreuzberg::{Result, ExtractionResult, ExtractionConfig};
|
|
178
|
+
/// # use kreuzberg::types::Metadata;
|
|
179
|
+
/// # use async_trait::async_trait;
|
|
180
|
+
/// # use std::path::Path;
|
|
181
|
+
/// # struct StreamingExtractor;
|
|
182
|
+
/// # impl Plugin for StreamingExtractor {
|
|
183
|
+
/// # fn name(&self) -> &str { "streaming" }
|
|
184
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
185
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
186
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
187
|
+
/// # }
|
|
188
|
+
/// # #[async_trait]
|
|
189
|
+
/// # impl DocumentExtractor for StreamingExtractor {
|
|
190
|
+
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
191
|
+
/// # fn priority(&self) -> i32 { 50 }
|
|
192
|
+
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
193
|
+
/// /// Override for memory-efficient streaming extraction
|
|
194
|
+
/// async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig)
|
|
195
|
+
/// -> Result<ExtractionResult> {
|
|
196
|
+
/// // Stream large files instead of loading entirely into memory
|
|
197
|
+
/// let mut content = String::new();
|
|
198
|
+
///
|
|
199
|
+
/// // Use buffered reader for streaming
|
|
200
|
+
/// use std::io::{BufRead, BufReader};
|
|
201
|
+
/// let file = std::fs::File::open(path)?;
|
|
202
|
+
/// let reader = BufReader::new(file);
|
|
203
|
+
///
|
|
204
|
+
/// for line in reader.lines() {
|
|
205
|
+
/// content.push_str(&line?);
|
|
206
|
+
/// content.push('\n');
|
|
207
|
+
/// }
|
|
208
|
+
///
|
|
209
|
+
/// Ok(ExtractionResult {
|
|
210
|
+
/// content,
|
|
211
|
+
/// mime_type: mime_type.to_string(),
|
|
212
|
+
/// metadata: Metadata::default(),
|
|
213
|
+
/// tables: vec![],
|
|
214
|
+
/// detected_languages: None,
|
|
215
|
+
/// chunks: None,
|
|
216
|
+
/// images: None,
|
|
217
|
+
/// pages: None,
|
|
218
|
+
/// })
|
|
219
|
+
/// }
|
|
220
|
+
/// # }
|
|
221
|
+
/// ```
|
|
222
|
+
async fn extract_file(&self, path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
223
|
+
#[cfg(feature = "tokio-runtime")]
|
|
224
|
+
{
|
|
225
|
+
use crate::core::io;
|
|
226
|
+
let bytes = io::read_file_async(path).await?;
|
|
227
|
+
self.extract_bytes(&bytes, mime_type, config).await
|
|
228
|
+
}
|
|
229
|
+
#[cfg(not(feature = "tokio-runtime"))]
|
|
230
|
+
{
|
|
231
|
+
let _ = (path, mime_type, config);
|
|
232
|
+
// For WASM and non-tokio environments, file extraction is not supported
|
|
233
|
+
// through the default implementation. Implementations must provide their own.
|
|
234
|
+
Err(KreuzbergError::Other(
|
|
235
|
+
"File-based extraction requires the tokio-runtime feature".to_string(),
|
|
236
|
+
))
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/// Get the list of MIME types supported by this extractor.
|
|
241
|
+
///
|
|
242
|
+
/// Can include exact MIME types and prefix patterns:
|
|
243
|
+
/// - Exact: `"application/pdf"`, `"text/plain"`
|
|
244
|
+
/// - Prefix: `"image/*"` (matches any image type)
|
|
245
|
+
///
|
|
246
|
+
/// # Returns
|
|
247
|
+
///
|
|
248
|
+
/// A slice of MIME type strings.
|
|
249
|
+
///
|
|
250
|
+
/// # Example
|
|
251
|
+
///
|
|
252
|
+
/// ```rust
|
|
253
|
+
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
254
|
+
/// # use kreuzberg::Result;
|
|
255
|
+
/// # use async_trait::async_trait;
|
|
256
|
+
/// # use std::path::Path;
|
|
257
|
+
/// # struct MultiFormatExtractor;
|
|
258
|
+
/// # impl Plugin for MultiFormatExtractor {
|
|
259
|
+
/// # fn name(&self) -> &str { "multi-format" }
|
|
260
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
261
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
262
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
263
|
+
/// # }
|
|
264
|
+
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
265
|
+
/// # #[async_trait]
|
|
266
|
+
/// # impl DocumentExtractor for MultiFormatExtractor {
|
|
267
|
+
/// # fn priority(&self) -> i32 { 50 }
|
|
268
|
+
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
269
|
+
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
270
|
+
/// fn supported_mime_types(&self) -> &[&str] {
|
|
271
|
+
/// &[
|
|
272
|
+
/// "text/plain",
|
|
273
|
+
/// "text/markdown",
|
|
274
|
+
/// "application/json",
|
|
275
|
+
/// "application/xml",
|
|
276
|
+
/// "text/html",
|
|
277
|
+
/// ]
|
|
278
|
+
/// }
|
|
279
|
+
/// # }
|
|
280
|
+
/// ```
|
|
281
|
+
fn supported_mime_types(&self) -> &[&str];
|
|
282
|
+
|
|
283
|
+
/// Get the priority of this extractor.
|
|
284
|
+
///
|
|
285
|
+
/// Higher priority extractors are preferred when multiple extractors
|
|
286
|
+
/// support the same MIME type.
|
|
287
|
+
///
|
|
288
|
+
/// # Priority Guidelines
|
|
289
|
+
///
|
|
290
|
+
/// - **0-25**: Fallback/low-quality extractors
|
|
291
|
+
/// - **26-49**: Alternative extractors
|
|
292
|
+
/// - **50**: Default priority (built-in extractors)
|
|
293
|
+
/// - **51-75**: Premium/enhanced extractors
|
|
294
|
+
/// - **76-100**: Specialized/high-priority extractors
|
|
295
|
+
///
|
|
296
|
+
/// # Returns
|
|
297
|
+
///
|
|
298
|
+
/// Priority value (default: 50)
|
|
299
|
+
///
|
|
300
|
+
/// # Example
|
|
301
|
+
///
|
|
302
|
+
/// ```rust
|
|
303
|
+
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
304
|
+
/// # use kreuzberg::Result;
|
|
305
|
+
/// # use async_trait::async_trait;
|
|
306
|
+
/// # use std::path::Path;
|
|
307
|
+
/// # struct FallbackExtractor;
|
|
308
|
+
/// # impl Plugin for FallbackExtractor {
|
|
309
|
+
/// # fn name(&self) -> &str { "fallback" }
|
|
310
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
311
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
312
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
313
|
+
/// # }
|
|
314
|
+
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
315
|
+
/// # #[async_trait]
|
|
316
|
+
/// # impl DocumentExtractor for FallbackExtractor {
|
|
317
|
+
/// # fn supported_mime_types(&self) -> &[&str] { &["text/plain"] }
|
|
318
|
+
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
319
|
+
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
320
|
+
/// fn priority(&self) -> i32 {
|
|
321
|
+
/// 10 // Low priority - only used as fallback
|
|
322
|
+
/// }
|
|
323
|
+
/// # }
|
|
324
|
+
/// ```
|
|
325
|
+
fn priority(&self) -> i32 {
|
|
326
|
+
50
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/// Optional: Check if this extractor can handle a specific file.
|
|
330
|
+
///
|
|
331
|
+
/// Allows for more sophisticated detection beyond MIME types.
|
|
332
|
+
/// Defaults to `true` (rely on MIME type matching).
|
|
333
|
+
///
|
|
334
|
+
/// # Arguments
|
|
335
|
+
///
|
|
336
|
+
/// * `path` - Path to the file to check
|
|
337
|
+
/// * `mime_type` - Detected MIME type
|
|
338
|
+
///
|
|
339
|
+
/// # Returns
|
|
340
|
+
///
|
|
341
|
+
/// `true` if the extractor can handle this file, `false` otherwise.
|
|
342
|
+
///
|
|
343
|
+
/// # Example
|
|
344
|
+
///
|
|
345
|
+
/// ```rust,no_run
|
|
346
|
+
/// # use kreuzberg::plugins::{Plugin, DocumentExtractor};
|
|
347
|
+
/// # use kreuzberg::Result;
|
|
348
|
+
/// # use async_trait::async_trait;
|
|
349
|
+
/// # use std::path::Path;
|
|
350
|
+
/// # struct SmartExtractor;
|
|
351
|
+
/// # impl Plugin for SmartExtractor {
|
|
352
|
+
/// # fn name(&self) -> &str { "smart" }
|
|
353
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
354
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
355
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
356
|
+
/// # }
|
|
357
|
+
/// # use kreuzberg::{ExtractionResult, ExtractionConfig};
|
|
358
|
+
/// # #[async_trait]
|
|
359
|
+
/// # impl DocumentExtractor for SmartExtractor {
|
|
360
|
+
/// # fn supported_mime_types(&self) -> &[&str] { &["application/pdf"] }
|
|
361
|
+
/// # fn priority(&self) -> i32 { 50 }
|
|
362
|
+
/// # async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
363
|
+
/// # async fn extract_file(&self, _: &Path, _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> { todo!() }
|
|
364
|
+
/// /// Only handle PDFs that are searchable (have text layer)
|
|
365
|
+
/// fn can_handle(&self, path: &Path, mime_type: &str) -> bool {
|
|
366
|
+
/// if mime_type != "application/pdf" {
|
|
367
|
+
/// return false;
|
|
368
|
+
/// }
|
|
369
|
+
///
|
|
370
|
+
/// // Check if PDF has text layer (simplified example)
|
|
371
|
+
/// // In real implementation, analyze PDF structure here
|
|
372
|
+
/// let _ = path; // Use path for PDF analysis
|
|
373
|
+
/// true // Simplified - always accept
|
|
374
|
+
/// }
|
|
375
|
+
/// # }
|
|
376
|
+
/// ```
|
|
377
|
+
fn can_handle(&self, _path: &Path, _mime_type: &str) -> bool {
|
|
378
|
+
true
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/// Attempt to get a reference to this extractor as a SyncExtractor.
|
|
382
|
+
///
|
|
383
|
+
/// Returns None if the extractor doesn't support synchronous extraction.
|
|
384
|
+
/// This is used for WASM and other sync-only environments.
|
|
385
|
+
fn as_sync_extractor(&self) -> Option<&dyn crate::extractors::SyncExtractor> {
|
|
386
|
+
None
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/// Register a document extractor with the global registry.
|
|
391
|
+
///
|
|
392
|
+
/// The extractor will be registered for all MIME types it supports and will be
|
|
393
|
+
/// available for document extraction. The extractor's `name()` method is used as
|
|
394
|
+
/// the registration name.
|
|
395
|
+
///
|
|
396
|
+
/// # Arguments
|
|
397
|
+
///
|
|
398
|
+
/// * `extractor` - The extractor implementation wrapped in Arc
|
|
399
|
+
///
|
|
400
|
+
/// # Returns
|
|
401
|
+
///
|
|
402
|
+
/// - `Ok(())` if registration succeeded
|
|
403
|
+
/// - `Err(...)` if validation failed or initialization failed
|
|
404
|
+
///
|
|
405
|
+
/// # Errors
|
|
406
|
+
///
|
|
407
|
+
/// - `KreuzbergError::Validation` - Invalid extractor name (empty or contains whitespace)
|
|
408
|
+
/// - Any error from the extractor's `initialize()` method
|
|
409
|
+
///
|
|
410
|
+
/// # Example
|
|
411
|
+
///
|
|
412
|
+
/// ```rust
|
|
413
|
+
/// use kreuzberg::plugins::{Plugin, DocumentExtractor, register_extractor};
|
|
414
|
+
/// use kreuzberg::{Result, ExtractionConfig};
|
|
415
|
+
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
416
|
+
/// use async_trait::async_trait;
|
|
417
|
+
/// use std::sync::Arc;
|
|
418
|
+
/// use std::path::Path;
|
|
419
|
+
///
|
|
420
|
+
/// struct CustomExtractor;
|
|
421
|
+
///
|
|
422
|
+
/// impl Plugin for CustomExtractor {
|
|
423
|
+
/// fn name(&self) -> &str { "custom-extractor" }
|
|
424
|
+
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
425
|
+
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
426
|
+
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
427
|
+
/// }
|
|
428
|
+
///
|
|
429
|
+
/// #[async_trait]
|
|
430
|
+
/// impl DocumentExtractor for CustomExtractor {
|
|
431
|
+
/// async fn extract_bytes(&self, content: &[u8], mime_type: &str, _: &ExtractionConfig)
|
|
432
|
+
/// -> Result<ExtractionResult> {
|
|
433
|
+
/// Ok(ExtractionResult {
|
|
434
|
+
/// content: String::from_utf8_lossy(content).to_string(),
|
|
435
|
+
/// mime_type: mime_type.to_string(),
|
|
436
|
+
/// metadata: Metadata::default(),
|
|
437
|
+
/// tables: vec![],
|
|
438
|
+
/// detected_languages: None,
|
|
439
|
+
/// chunks: None,
|
|
440
|
+
/// images: None,
|
|
441
|
+
/// pages: None,
|
|
442
|
+
/// })
|
|
443
|
+
/// }
|
|
444
|
+
///
|
|
445
|
+
/// fn supported_mime_types(&self) -> &[&str] {
|
|
446
|
+
/// &["text/custom"]
|
|
447
|
+
/// }
|
|
448
|
+
/// }
|
|
449
|
+
///
|
|
450
|
+
/// # tokio_test::block_on(async {
|
|
451
|
+
/// let extractor = Arc::new(CustomExtractor);
|
|
452
|
+
/// register_extractor(extractor)?;
|
|
453
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
454
|
+
/// # });
|
|
455
|
+
/// ```
|
|
456
|
+
pub fn register_extractor(extractor: Arc<dyn DocumentExtractor>) -> crate::Result<()> {
|
|
457
|
+
use crate::plugins::registry::get_document_extractor_registry;
|
|
458
|
+
|
|
459
|
+
let registry = get_document_extractor_registry();
|
|
460
|
+
let mut registry = registry
|
|
461
|
+
.write()
|
|
462
|
+
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
463
|
+
|
|
464
|
+
registry.register(extractor)
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/// Unregister a document extractor by name.
|
|
468
|
+
///
|
|
469
|
+
/// Removes the extractor from the global registry and calls its `shutdown()` method.
|
|
470
|
+
///
|
|
471
|
+
/// # Arguments
|
|
472
|
+
///
|
|
473
|
+
/// * `name` - Name of the extractor to unregister
|
|
474
|
+
///
|
|
475
|
+
/// # Returns
|
|
476
|
+
///
|
|
477
|
+
/// - `Ok(())` if the extractor was unregistered or didn't exist
|
|
478
|
+
/// - `Err(...)` if the shutdown method failed
|
|
479
|
+
///
|
|
480
|
+
/// # Example
|
|
481
|
+
///
|
|
482
|
+
/// ```rust
|
|
483
|
+
/// use kreuzberg::plugins::unregister_extractor;
|
|
484
|
+
///
|
|
485
|
+
/// # tokio_test::block_on(async {
|
|
486
|
+
/// unregister_extractor("custom-extractor")?;
|
|
487
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
488
|
+
/// # });
|
|
489
|
+
/// ```
|
|
490
|
+
pub fn unregister_extractor(name: &str) -> crate::Result<()> {
|
|
491
|
+
use crate::plugins::registry::get_document_extractor_registry;
|
|
492
|
+
|
|
493
|
+
let registry = get_document_extractor_registry();
|
|
494
|
+
let mut registry = registry
|
|
495
|
+
.write()
|
|
496
|
+
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
497
|
+
|
|
498
|
+
registry.remove(name)
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/// List all registered extractors.
|
|
502
|
+
///
|
|
503
|
+
/// Returns the names of all extractors currently registered in the global registry.
|
|
504
|
+
///
|
|
505
|
+
/// # Returns
|
|
506
|
+
///
|
|
507
|
+
/// A vector of extractor names.
|
|
508
|
+
///
|
|
509
|
+
/// # Example
|
|
510
|
+
///
|
|
511
|
+
/// ```rust
|
|
512
|
+
/// use kreuzberg::plugins::list_extractors;
|
|
513
|
+
///
|
|
514
|
+
/// # tokio_test::block_on(async {
|
|
515
|
+
/// let extractors = list_extractors()?;
|
|
516
|
+
/// for name in extractors {
|
|
517
|
+
/// println!("Registered extractor: {}", name);
|
|
518
|
+
/// }
|
|
519
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
520
|
+
/// # });
|
|
521
|
+
/// ```
|
|
522
|
+
pub fn list_extractors() -> crate::Result<Vec<String>> {
|
|
523
|
+
use crate::plugins::registry::get_document_extractor_registry;
|
|
524
|
+
|
|
525
|
+
let registry = get_document_extractor_registry();
|
|
526
|
+
let registry = registry
|
|
527
|
+
.read()
|
|
528
|
+
.expect("~keep Failed to acquire read lock on extractor registry"); // ~keep
|
|
529
|
+
|
|
530
|
+
Ok(registry.list())
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/// Clear all extractors from the global registry.
|
|
534
|
+
///
|
|
535
|
+
/// Removes all extractors and calls their `shutdown()` methods.
|
|
536
|
+
///
|
|
537
|
+
/// # Returns
|
|
538
|
+
///
|
|
539
|
+
/// - `Ok(())` if all extractors were cleared successfully
|
|
540
|
+
/// - `Err(...)` if any shutdown method failed
|
|
541
|
+
///
|
|
542
|
+
/// # Example
|
|
543
|
+
///
|
|
544
|
+
/// ```rust
|
|
545
|
+
/// use kreuzberg::plugins::clear_extractors;
|
|
546
|
+
///
|
|
547
|
+
/// # tokio_test::block_on(async {
|
|
548
|
+
/// clear_extractors()?;
|
|
549
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
550
|
+
/// # });
|
|
551
|
+
/// ```
|
|
552
|
+
pub fn clear_extractors() -> crate::Result<()> {
|
|
553
|
+
use crate::plugins::registry::get_document_extractor_registry;
|
|
554
|
+
|
|
555
|
+
let registry = get_document_extractor_registry();
|
|
556
|
+
let mut registry = registry
|
|
557
|
+
.write()
|
|
558
|
+
.expect("~keep Failed to acquire write lock on extractor registry"); // ~keep
|
|
559
|
+
|
|
560
|
+
registry.shutdown_all()
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
#[cfg(test)]
|
|
564
|
+
mod tests {
|
|
565
|
+
use super::*;
|
|
566
|
+
use serial_test::serial;
|
|
567
|
+
|
|
568
|
+
struct MockExtractor {
|
|
569
|
+
mime_types: Vec<&'static str>,
|
|
570
|
+
priority: i32,
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
impl Plugin for MockExtractor {
|
|
574
|
+
fn name(&self) -> &str {
|
|
575
|
+
"mock-extractor"
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
fn version(&self) -> String {
|
|
579
|
+
"1.0.0".to_string()
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
fn initialize(&self) -> Result<()> {
|
|
583
|
+
Ok(())
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
fn shutdown(&self) -> Result<()> {
|
|
587
|
+
Ok(())
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
#[async_trait]
|
|
592
|
+
impl DocumentExtractor for MockExtractor {
|
|
593
|
+
async fn extract_bytes(
|
|
594
|
+
&self,
|
|
595
|
+
content: &[u8],
|
|
596
|
+
mime_type: &str,
|
|
597
|
+
_config: &ExtractionConfig,
|
|
598
|
+
) -> Result<ExtractionResult> {
|
|
599
|
+
Ok(ExtractionResult {
|
|
600
|
+
content: String::from_utf8_lossy(content).to_string(),
|
|
601
|
+
mime_type: mime_type.to_string(),
|
|
602
|
+
metadata: crate::types::Metadata::default(),
|
|
603
|
+
tables: vec![],
|
|
604
|
+
detected_languages: None,
|
|
605
|
+
chunks: None,
|
|
606
|
+
images: None,
|
|
607
|
+
pages: None,
|
|
608
|
+
})
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
fn supported_mime_types(&self) -> &[&str] {
|
|
612
|
+
&self.mime_types
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
fn priority(&self) -> i32 {
|
|
616
|
+
self.priority
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
#[tokio::test]
|
|
621
|
+
async fn test_document_extractor_extract_bytes() {
|
|
622
|
+
let extractor = MockExtractor {
|
|
623
|
+
mime_types: vec!["text/plain"],
|
|
624
|
+
priority: 50,
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
let config = ExtractionConfig::default();
|
|
628
|
+
let result = extractor
|
|
629
|
+
.extract_bytes(b"test content", "text/plain", &config)
|
|
630
|
+
.await
|
|
631
|
+
.unwrap();
|
|
632
|
+
|
|
633
|
+
assert_eq!(result.content, "test content");
|
|
634
|
+
assert_eq!(result.mime_type, "text/plain");
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
#[test]
|
|
638
|
+
fn test_document_extractor_supported_mime_types() {
|
|
639
|
+
let extractor = MockExtractor {
|
|
640
|
+
mime_types: vec!["text/plain", "text/markdown"],
|
|
641
|
+
priority: 50,
|
|
642
|
+
};
|
|
643
|
+
|
|
644
|
+
let supported = extractor.supported_mime_types();
|
|
645
|
+
assert_eq!(supported.len(), 2);
|
|
646
|
+
assert!(supported.contains(&"text/plain"));
|
|
647
|
+
assert!(supported.contains(&"text/markdown"));
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
#[test]
|
|
651
|
+
fn test_document_extractor_priority() {
|
|
652
|
+
let low_priority = MockExtractor {
|
|
653
|
+
mime_types: vec!["text/plain"],
|
|
654
|
+
priority: 10,
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
let high_priority = MockExtractor {
|
|
658
|
+
mime_types: vec!["text/plain"],
|
|
659
|
+
priority: 100,
|
|
660
|
+
};
|
|
661
|
+
|
|
662
|
+
assert_eq!(low_priority.priority(), 10);
|
|
663
|
+
assert_eq!(high_priority.priority(), 100);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
#[test]
|
|
667
|
+
fn test_document_extractor_can_handle_default() {
|
|
668
|
+
let extractor = MockExtractor {
|
|
669
|
+
mime_types: vec!["text/plain"],
|
|
670
|
+
priority: 50,
|
|
671
|
+
};
|
|
672
|
+
|
|
673
|
+
use std::path::PathBuf;
|
|
674
|
+
let path = PathBuf::from("test.txt");
|
|
675
|
+
|
|
676
|
+
assert!(extractor.can_handle(&path, "text/plain"));
|
|
677
|
+
assert!(extractor.can_handle(&path, "application/pdf"));
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
#[tokio::test]
|
|
681
|
+
async fn test_document_extractor_extract_file_default_impl() {
|
|
682
|
+
use std::io::Write;
|
|
683
|
+
use tempfile::NamedTempFile;
|
|
684
|
+
|
|
685
|
+
let extractor = MockExtractor {
|
|
686
|
+
mime_types: vec!["text/plain"],
|
|
687
|
+
priority: 50,
|
|
688
|
+
};
|
|
689
|
+
|
|
690
|
+
let mut temp_file = NamedTempFile::new().unwrap();
|
|
691
|
+
temp_file.write_all(b"file content").unwrap();
|
|
692
|
+
let path = temp_file.path();
|
|
693
|
+
|
|
694
|
+
let config = ExtractionConfig::default();
|
|
695
|
+
let result = extractor.extract_file(path, "text/plain", &config).await.unwrap();
|
|
696
|
+
|
|
697
|
+
assert_eq!(result.content, "file content");
|
|
698
|
+
assert_eq!(result.mime_type, "text/plain");
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
#[tokio::test]
|
|
702
|
+
async fn test_document_extractor_empty_content() {
|
|
703
|
+
let extractor = MockExtractor {
|
|
704
|
+
mime_types: vec!["text/plain"],
|
|
705
|
+
priority: 50,
|
|
706
|
+
};
|
|
707
|
+
|
|
708
|
+
let config = ExtractionConfig::default();
|
|
709
|
+
let result = extractor.extract_bytes(b"", "text/plain", &config).await.unwrap();
|
|
710
|
+
|
|
711
|
+
assert_eq!(result.content, "");
|
|
712
|
+
assert_eq!(result.mime_type, "text/plain");
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
#[tokio::test]
|
|
716
|
+
async fn test_document_extractor_non_utf8_content() {
|
|
717
|
+
let extractor = MockExtractor {
|
|
718
|
+
mime_types: vec!["text/plain"],
|
|
719
|
+
priority: 50,
|
|
720
|
+
};
|
|
721
|
+
|
|
722
|
+
let invalid_utf8 = vec![0xFF, 0xFE, 0xFD];
|
|
723
|
+
let config = ExtractionConfig::default();
|
|
724
|
+
let result = extractor
|
|
725
|
+
.extract_bytes(&invalid_utf8, "text/plain", &config)
|
|
726
|
+
.await
|
|
727
|
+
.unwrap();
|
|
728
|
+
|
|
729
|
+
assert!(!result.content.is_empty());
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
#[test]
|
|
733
|
+
fn test_document_extractor_plugin_interface() {
|
|
734
|
+
let extractor = MockExtractor {
|
|
735
|
+
mime_types: vec!["text/plain"],
|
|
736
|
+
priority: 50,
|
|
737
|
+
};
|
|
738
|
+
|
|
739
|
+
assert_eq!(extractor.name(), "mock-extractor");
|
|
740
|
+
assert_eq!(extractor.version(), "1.0.0");
|
|
741
|
+
assert!(extractor.initialize().is_ok());
|
|
742
|
+
assert!(extractor.shutdown().is_ok());
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
#[test]
|
|
746
|
+
fn test_document_extractor_default_priority() {
|
|
747
|
+
struct DefaultPriorityExtractor;
|
|
748
|
+
|
|
749
|
+
impl Plugin for DefaultPriorityExtractor {
|
|
750
|
+
fn name(&self) -> &str {
|
|
751
|
+
"default"
|
|
752
|
+
}
|
|
753
|
+
fn version(&self) -> String {
|
|
754
|
+
"1.0.0".to_string()
|
|
755
|
+
}
|
|
756
|
+
fn initialize(&self) -> Result<()> {
|
|
757
|
+
Ok(())
|
|
758
|
+
}
|
|
759
|
+
fn shutdown(&self) -> Result<()> {
|
|
760
|
+
Ok(())
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
#[async_trait]
|
|
765
|
+
impl DocumentExtractor for DefaultPriorityExtractor {
|
|
766
|
+
async fn extract_bytes(
|
|
767
|
+
&self,
|
|
768
|
+
_content: &[u8],
|
|
769
|
+
_mime_type: &str,
|
|
770
|
+
_config: &ExtractionConfig,
|
|
771
|
+
) -> Result<ExtractionResult> {
|
|
772
|
+
Ok(ExtractionResult {
|
|
773
|
+
content: String::new(),
|
|
774
|
+
mime_type: String::new(),
|
|
775
|
+
metadata: crate::types::Metadata::default(),
|
|
776
|
+
tables: vec![],
|
|
777
|
+
detected_languages: None,
|
|
778
|
+
chunks: None,
|
|
779
|
+
images: None,
|
|
780
|
+
pages: None,
|
|
781
|
+
})
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
fn supported_mime_types(&self) -> &[&str] {
|
|
785
|
+
&["text/plain"]
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
let extractor = DefaultPriorityExtractor;
|
|
790
|
+
assert_eq!(extractor.priority(), 50);
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
#[test]
|
|
794
|
+
fn test_document_extractor_empty_mime_types() {
|
|
795
|
+
let extractor = MockExtractor {
|
|
796
|
+
mime_types: vec![],
|
|
797
|
+
priority: 50,
|
|
798
|
+
};
|
|
799
|
+
|
|
800
|
+
assert_eq!(extractor.supported_mime_types().len(), 0);
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
#[test]
|
|
804
|
+
fn test_document_extractor_wildcard_mime_types() {
|
|
805
|
+
let extractor = MockExtractor {
|
|
806
|
+
mime_types: vec!["text/*", "image/*"],
|
|
807
|
+
priority: 50,
|
|
808
|
+
};
|
|
809
|
+
|
|
810
|
+
let supported = extractor.supported_mime_types();
|
|
811
|
+
assert_eq!(supported.len(), 2);
|
|
812
|
+
assert!(supported.contains(&"text/*"));
|
|
813
|
+
assert!(supported.contains(&"image/*"));
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
#[test]
|
|
817
|
+
fn test_document_extractor_priority_ranges() {
|
|
818
|
+
let fallback = MockExtractor {
|
|
819
|
+
mime_types: vec!["text/plain"],
|
|
820
|
+
priority: 10,
|
|
821
|
+
};
|
|
822
|
+
let alternative = MockExtractor {
|
|
823
|
+
mime_types: vec!["text/plain"],
|
|
824
|
+
priority: 40,
|
|
825
|
+
};
|
|
826
|
+
let default = MockExtractor {
|
|
827
|
+
mime_types: vec!["text/plain"],
|
|
828
|
+
priority: 50,
|
|
829
|
+
};
|
|
830
|
+
let premium = MockExtractor {
|
|
831
|
+
mime_types: vec!["text/plain"],
|
|
832
|
+
priority: 75,
|
|
833
|
+
};
|
|
834
|
+
let specialized = MockExtractor {
|
|
835
|
+
mime_types: vec!["text/plain"],
|
|
836
|
+
priority: 100,
|
|
837
|
+
};
|
|
838
|
+
|
|
839
|
+
assert!(fallback.priority() < alternative.priority());
|
|
840
|
+
assert!(alternative.priority() < default.priority());
|
|
841
|
+
assert!(default.priority() < premium.priority());
|
|
842
|
+
assert!(premium.priority() < specialized.priority());
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
#[tokio::test]
|
|
846
|
+
async fn test_document_extractor_preserves_mime_type() {
|
|
847
|
+
let extractor = MockExtractor {
|
|
848
|
+
mime_types: vec!["application/json"],
|
|
849
|
+
priority: 50,
|
|
850
|
+
};
|
|
851
|
+
|
|
852
|
+
let config = ExtractionConfig::default();
|
|
853
|
+
let result = extractor
|
|
854
|
+
.extract_bytes(b"{\"key\":\"value\"}", "application/json", &config)
|
|
855
|
+
.await
|
|
856
|
+
.unwrap();
|
|
857
|
+
|
|
858
|
+
assert_eq!(result.mime_type, "application/json");
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
#[test]
|
|
862
|
+
#[serial]
|
|
863
|
+
fn test_register_extractor() {
|
|
864
|
+
use std::sync::Arc;
|
|
865
|
+
|
|
866
|
+
let extractor = Arc::new(MockExtractor {
|
|
867
|
+
mime_types: vec!["text/test-register"],
|
|
868
|
+
priority: 50,
|
|
869
|
+
});
|
|
870
|
+
let result = super::register_extractor(extractor);
|
|
871
|
+
assert!(result.is_ok());
|
|
872
|
+
|
|
873
|
+
let _ = super::unregister_extractor("mock-extractor");
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
#[test]
|
|
877
|
+
#[serial]
|
|
878
|
+
fn test_unregister_extractor() {
|
|
879
|
+
use std::sync::Arc;
|
|
880
|
+
|
|
881
|
+
let extractor = Arc::new(MockExtractor {
|
|
882
|
+
mime_types: vec!["text/test-unregister"],
|
|
883
|
+
priority: 50,
|
|
884
|
+
});
|
|
885
|
+
super::register_extractor(extractor).unwrap();
|
|
886
|
+
|
|
887
|
+
let result = super::unregister_extractor("mock-extractor");
|
|
888
|
+
assert!(result.is_ok());
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
#[test]
|
|
892
|
+
#[serial]
|
|
893
|
+
fn test_unregister_nonexistent_extractor() {
|
|
894
|
+
let result = super::unregister_extractor("nonexistent-extractor-xyz");
|
|
895
|
+
assert!(result.is_ok());
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
#[test]
|
|
899
|
+
#[serial]
|
|
900
|
+
fn test_list_extractors() {
|
|
901
|
+
use std::sync::Arc;
|
|
902
|
+
|
|
903
|
+
super::clear_extractors().unwrap();
|
|
904
|
+
|
|
905
|
+
let extractor1 = Arc::new(MockExtractor {
|
|
906
|
+
mime_types: vec!["text/test-list-1"],
|
|
907
|
+
priority: 50,
|
|
908
|
+
});
|
|
909
|
+
let extractor2 = Arc::new(MockExtractor {
|
|
910
|
+
mime_types: vec!["text/test-list-2"],
|
|
911
|
+
priority: 51,
|
|
912
|
+
});
|
|
913
|
+
|
|
914
|
+
let list_before = super::list_extractors().unwrap();
|
|
915
|
+
assert_eq!(list_before.len(), 0);
|
|
916
|
+
|
|
917
|
+
super::register_extractor(extractor1).unwrap();
|
|
918
|
+
super::register_extractor(extractor2).unwrap();
|
|
919
|
+
|
|
920
|
+
let list = super::list_extractors().unwrap();
|
|
921
|
+
assert_eq!(list.len(), 1);
|
|
922
|
+
assert!(list.contains(&"mock-extractor".to_string()));
|
|
923
|
+
|
|
924
|
+
super::unregister_extractor("mock-extractor").unwrap();
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
#[test]
|
|
928
|
+
#[serial]
|
|
929
|
+
fn test_clear_extractors() {
|
|
930
|
+
use std::sync::Arc;
|
|
931
|
+
|
|
932
|
+
super::clear_extractors().unwrap();
|
|
933
|
+
|
|
934
|
+
let extractor1 = Arc::new(MockExtractor {
|
|
935
|
+
mime_types: vec!["text/test-clear-1"],
|
|
936
|
+
priority: 50,
|
|
937
|
+
});
|
|
938
|
+
let extractor2 = Arc::new(MockExtractor {
|
|
939
|
+
mime_types: vec!["text/test-clear-2"],
|
|
940
|
+
priority: 51,
|
|
941
|
+
});
|
|
942
|
+
|
|
943
|
+
super::register_extractor(extractor1).unwrap();
|
|
944
|
+
super::register_extractor(extractor2).unwrap();
|
|
945
|
+
|
|
946
|
+
let result = super::clear_extractors();
|
|
947
|
+
assert!(result.is_ok());
|
|
948
|
+
|
|
949
|
+
let list = super::list_extractors().unwrap();
|
|
950
|
+
assert_eq!(list.len(), 0);
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
#[test]
|
|
954
|
+
#[serial]
|
|
955
|
+
fn test_register_extractor_with_invalid_name() {
|
|
956
|
+
use std::sync::Arc;
|
|
957
|
+
|
|
958
|
+
struct InvalidNameExtractor;
|
|
959
|
+
impl Plugin for InvalidNameExtractor {
|
|
960
|
+
fn name(&self) -> &str {
|
|
961
|
+
"invalid name with spaces"
|
|
962
|
+
}
|
|
963
|
+
fn version(&self) -> String {
|
|
964
|
+
"1.0.0".to_string()
|
|
965
|
+
}
|
|
966
|
+
fn initialize(&self) -> Result<()> {
|
|
967
|
+
Ok(())
|
|
968
|
+
}
|
|
969
|
+
fn shutdown(&self) -> Result<()> {
|
|
970
|
+
Ok(())
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
#[async_trait]
|
|
975
|
+
impl DocumentExtractor for InvalidNameExtractor {
|
|
976
|
+
async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
977
|
+
Ok(ExtractionResult {
|
|
978
|
+
content: String::new(),
|
|
979
|
+
mime_type: String::new(),
|
|
980
|
+
metadata: crate::types::Metadata::default(),
|
|
981
|
+
tables: vec![],
|
|
982
|
+
detected_languages: None,
|
|
983
|
+
chunks: None,
|
|
984
|
+
images: None,
|
|
985
|
+
pages: None,
|
|
986
|
+
})
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
fn supported_mime_types(&self) -> &[&str] {
|
|
990
|
+
&["text/plain"]
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
let extractor = Arc::new(InvalidNameExtractor);
|
|
995
|
+
let result = super::register_extractor(extractor);
|
|
996
|
+
assert!(matches!(result, Err(crate::KreuzbergError::Validation { .. })));
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
#[test]
|
|
1000
|
+
#[serial]
|
|
1001
|
+
fn test_register_extractor_with_empty_name() {
|
|
1002
|
+
use std::sync::Arc;
|
|
1003
|
+
|
|
1004
|
+
struct EmptyNameExtractor;
|
|
1005
|
+
impl Plugin for EmptyNameExtractor {
|
|
1006
|
+
fn name(&self) -> &str {
|
|
1007
|
+
""
|
|
1008
|
+
}
|
|
1009
|
+
fn version(&self) -> String {
|
|
1010
|
+
"1.0.0".to_string()
|
|
1011
|
+
}
|
|
1012
|
+
fn initialize(&self) -> Result<()> {
|
|
1013
|
+
Ok(())
|
|
1014
|
+
}
|
|
1015
|
+
fn shutdown(&self) -> Result<()> {
|
|
1016
|
+
Ok(())
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
#[async_trait]
|
|
1021
|
+
impl DocumentExtractor for EmptyNameExtractor {
|
|
1022
|
+
async fn extract_bytes(&self, _: &[u8], _: &str, _: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
1023
|
+
Ok(ExtractionResult {
|
|
1024
|
+
content: String::new(),
|
|
1025
|
+
mime_type: String::new(),
|
|
1026
|
+
metadata: crate::types::Metadata::default(),
|
|
1027
|
+
tables: vec![],
|
|
1028
|
+
detected_languages: None,
|
|
1029
|
+
chunks: None,
|
|
1030
|
+
images: None,
|
|
1031
|
+
pages: None,
|
|
1032
|
+
})
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
fn supported_mime_types(&self) -> &[&str] {
|
|
1036
|
+
&["text/plain"]
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
let extractor = Arc::new(EmptyNameExtractor);
|
|
1041
|
+
let result = super::register_extractor(extractor);
|
|
1042
|
+
assert!(matches!(result, Err(crate::KreuzbergError::Validation { .. })));
|
|
1043
|
+
}
|
|
1044
|
+
}
|