kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +105 -2
- data/README.md +454 -454
- data/Rakefile +33 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6940 -6941
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3158
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -214
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -81
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -80
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -340
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -109
- data/lib/{pdfium.dll → libpdfium.dylib} +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -546
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +1 -1
- data/vendor/kreuzberg/Cargo.toml +5 -5
- data/vendor/kreuzberg/README.md +230 -230
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -843
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1184
- data/vendor/kreuzberg/src/embeddings.rs +500 -500
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -601
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -574
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -749
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -44
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -346
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -68
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -420
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -164
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +1 -1
- data/vendor/kreuzberg-ffi/README.md +851 -851
- data/vendor/kreuzberg-ffi/build.rs +176 -176
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -27
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -1087
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -3616
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -247
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -470
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
- data/vendor/kreuzberg-tesseract/LICENSE +22 -22
- data/vendor/kreuzberg-tesseract/README.md +399 -399
- data/vendor/kreuzberg-tesseract/build.rs +1354 -1354
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -211
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +73 -4
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +0 -12
|
@@ -1,429 +1,429 @@
|
|
|
1
|
-
//! Built-in document extractors.
|
|
2
|
-
//!
|
|
3
|
-
//! This module contains the default extractors that ship with Kreuzberg.
|
|
4
|
-
//! All extractors implement the `DocumentExtractor` plugin trait.
|
|
5
|
-
|
|
6
|
-
use crate::Result;
|
|
7
|
-
use crate::core::config::ExtractionConfig;
|
|
8
|
-
use crate::plugins::registry::get_document_extractor_registry;
|
|
9
|
-
use crate::types::ExtractionResult;
|
|
10
|
-
use once_cell::sync::Lazy;
|
|
11
|
-
use std::sync::Arc;
|
|
12
|
-
|
|
13
|
-
/// Trait for extractors that can work synchronously (WASM-compatible).
|
|
14
|
-
///
|
|
15
|
-
/// This trait defines the synchronous extraction interface for WASM targets and other
|
|
16
|
-
/// environments where async/tokio runtimes are not available or desirable.
|
|
17
|
-
///
|
|
18
|
-
/// # Implementation
|
|
19
|
-
///
|
|
20
|
-
/// Extractors that need to support WASM should implement this trait in addition to
|
|
21
|
-
/// the async `DocumentExtractor` trait. This allows the same extractor to work in both
|
|
22
|
-
/// environments by delegating to the sync implementation.
|
|
23
|
-
///
|
|
24
|
-
/// # MIME Type Validation
|
|
25
|
-
///
|
|
26
|
-
/// The `mime_type` parameter is guaranteed to be already validated.
|
|
27
|
-
///
|
|
28
|
-
/// # Example
|
|
29
|
-
///
|
|
30
|
-
/// ```rust,ignore
|
|
31
|
-
/// impl SyncExtractor for PlainTextExtractor {
|
|
32
|
-
/// fn extract_sync(&self, content: &[u8], config: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
33
|
-
/// let text = String::from_utf8_lossy(content).to_string();
|
|
34
|
-
/// Ok(ExtractionResult {
|
|
35
|
-
/// content: text,
|
|
36
|
-
/// mime_type: "text/plain".to_string(),
|
|
37
|
-
/// metadata: Metadata::default(),
|
|
38
|
-
/// tables: vec![],
|
|
39
|
-
/// detected_languages: None,
|
|
40
|
-
/// chunks: None,
|
|
41
|
-
/// images: None,
|
|
42
|
-
/// })
|
|
43
|
-
/// }
|
|
44
|
-
/// }
|
|
45
|
-
/// ```
|
|
46
|
-
pub trait SyncExtractor {
|
|
47
|
-
/// Extract content from a byte array synchronously.
|
|
48
|
-
///
|
|
49
|
-
/// This method performs extraction without requiring an async runtime.
|
|
50
|
-
/// It is called by `extract_bytes_sync()` when the `tokio-runtime` feature is disabled.
|
|
51
|
-
///
|
|
52
|
-
/// # Arguments
|
|
53
|
-
///
|
|
54
|
-
/// * `content` - Raw document bytes
|
|
55
|
-
/// * `mime_type` - MIME type of the document (already validated)
|
|
56
|
-
/// * `config` - Extraction configuration
|
|
57
|
-
///
|
|
58
|
-
/// # Returns
|
|
59
|
-
///
|
|
60
|
-
/// An `ExtractionResult` containing the extracted content and metadata.
|
|
61
|
-
fn extract_sync(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult>;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
pub mod structured;
|
|
65
|
-
pub mod text;
|
|
66
|
-
|
|
67
|
-
#[cfg(feature = "archives")]
|
|
68
|
-
pub mod security;
|
|
69
|
-
|
|
70
|
-
#[cfg(feature = "ocr")]
|
|
71
|
-
pub mod image;
|
|
72
|
-
|
|
73
|
-
#[cfg(feature = "archives")]
|
|
74
|
-
pub mod archive;
|
|
75
|
-
|
|
76
|
-
#[cfg(feature = "email")]
|
|
77
|
-
pub mod email;
|
|
78
|
-
|
|
79
|
-
#[cfg(feature = "excel")]
|
|
80
|
-
pub mod excel;
|
|
81
|
-
|
|
82
|
-
#[cfg(feature = "html")]
|
|
83
|
-
pub mod html;
|
|
84
|
-
|
|
85
|
-
#[cfg(feature = "office")]
|
|
86
|
-
pub mod bibtex;
|
|
87
|
-
|
|
88
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
89
|
-
pub mod docx;
|
|
90
|
-
|
|
91
|
-
#[cfg(feature = "office")]
|
|
92
|
-
pub mod epub;
|
|
93
|
-
|
|
94
|
-
#[cfg(feature = "office")]
|
|
95
|
-
pub mod fictionbook;
|
|
96
|
-
|
|
97
|
-
#[cfg(feature = "office")]
|
|
98
|
-
pub mod markdown;
|
|
99
|
-
|
|
100
|
-
#[cfg(feature = "office")]
|
|
101
|
-
pub mod rst;
|
|
102
|
-
|
|
103
|
-
#[cfg(feature = "office")]
|
|
104
|
-
pub mod latex;
|
|
105
|
-
|
|
106
|
-
#[cfg(feature = "office")]
|
|
107
|
-
pub mod jupyter;
|
|
108
|
-
|
|
109
|
-
#[cfg(feature = "office")]
|
|
110
|
-
pub mod orgmode;
|
|
111
|
-
|
|
112
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
113
|
-
pub mod odt;
|
|
114
|
-
|
|
115
|
-
#[cfg(feature = "office")]
|
|
116
|
-
pub mod opml;
|
|
117
|
-
|
|
118
|
-
#[cfg(feature = "office")]
|
|
119
|
-
pub mod typst;
|
|
120
|
-
|
|
121
|
-
#[cfg(feature = "xml")]
|
|
122
|
-
pub mod jats;
|
|
123
|
-
|
|
124
|
-
#[cfg(feature = "pdf")]
|
|
125
|
-
pub mod pdf;
|
|
126
|
-
|
|
127
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
128
|
-
pub mod pptx;
|
|
129
|
-
|
|
130
|
-
#[cfg(feature = "office")]
|
|
131
|
-
pub mod rtf;
|
|
132
|
-
|
|
133
|
-
#[cfg(feature = "xml")]
|
|
134
|
-
pub mod xml;
|
|
135
|
-
|
|
136
|
-
#[cfg(feature = "xml")]
|
|
137
|
-
pub mod docbook;
|
|
138
|
-
|
|
139
|
-
pub use structured::StructuredExtractor;
|
|
140
|
-
pub use text::{MarkdownExtractor, PlainTextExtractor};
|
|
141
|
-
|
|
142
|
-
#[cfg(feature = "ocr")]
|
|
143
|
-
pub use image::ImageExtractor;
|
|
144
|
-
|
|
145
|
-
#[cfg(feature = "archives")]
|
|
146
|
-
pub use archive::{SevenZExtractor, TarExtractor, ZipExtractor};
|
|
147
|
-
|
|
148
|
-
#[cfg(feature = "email")]
|
|
149
|
-
pub use email::EmailExtractor;
|
|
150
|
-
|
|
151
|
-
#[cfg(feature = "excel")]
|
|
152
|
-
pub use excel::ExcelExtractor;
|
|
153
|
-
|
|
154
|
-
#[cfg(feature = "html")]
|
|
155
|
-
pub use html::HtmlExtractor;
|
|
156
|
-
|
|
157
|
-
#[cfg(feature = "office")]
|
|
158
|
-
pub use bibtex::BibtexExtractor;
|
|
159
|
-
|
|
160
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
161
|
-
pub use docx::DocxExtractor;
|
|
162
|
-
|
|
163
|
-
#[cfg(feature = "office")]
|
|
164
|
-
pub use epub::EpubExtractor;
|
|
165
|
-
|
|
166
|
-
#[cfg(feature = "office")]
|
|
167
|
-
pub use fictionbook::FictionBookExtractor;
|
|
168
|
-
|
|
169
|
-
#[cfg(feature = "office")]
|
|
170
|
-
pub use markdown::MarkdownExtractor as EnhancedMarkdownExtractor;
|
|
171
|
-
|
|
172
|
-
#[cfg(feature = "office")]
|
|
173
|
-
pub use rst::RstExtractor;
|
|
174
|
-
|
|
175
|
-
#[cfg(feature = "office")]
|
|
176
|
-
pub use latex::LatexExtractor;
|
|
177
|
-
|
|
178
|
-
#[cfg(feature = "office")]
|
|
179
|
-
pub use jupyter::JupyterExtractor;
|
|
180
|
-
|
|
181
|
-
#[cfg(feature = "office")]
|
|
182
|
-
pub use orgmode::OrgModeExtractor;
|
|
183
|
-
|
|
184
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
185
|
-
pub use odt::OdtExtractor;
|
|
186
|
-
|
|
187
|
-
#[cfg(feature = "xml")]
|
|
188
|
-
pub use jats::JatsExtractor;
|
|
189
|
-
|
|
190
|
-
#[cfg(feature = "office")]
|
|
191
|
-
pub use opml::OpmlExtractor;
|
|
192
|
-
|
|
193
|
-
#[cfg(feature = "office")]
|
|
194
|
-
pub use typst::TypstExtractor;
|
|
195
|
-
|
|
196
|
-
#[cfg(feature = "pdf")]
|
|
197
|
-
pub use pdf::PdfExtractor;
|
|
198
|
-
|
|
199
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
200
|
-
pub use pptx::PptxExtractor;
|
|
201
|
-
|
|
202
|
-
#[cfg(feature = "office")]
|
|
203
|
-
pub use rtf::RtfExtractor;
|
|
204
|
-
|
|
205
|
-
#[cfg(feature = "xml")]
|
|
206
|
-
pub use xml::XmlExtractor;
|
|
207
|
-
|
|
208
|
-
#[cfg(feature = "xml")]
|
|
209
|
-
pub use docbook::DocbookExtractor;
|
|
210
|
-
|
|
211
|
-
/// Lazy-initialized flag that ensures extractors are registered exactly once.
|
|
212
|
-
///
|
|
213
|
-
/// This static is accessed on first extraction operation to automatically
|
|
214
|
-
/// register all built-in extractors with the plugin registry.
|
|
215
|
-
static EXTRACTORS_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_default_extractors);
|
|
216
|
-
|
|
217
|
-
/// Ensure built-in extractors are registered.
|
|
218
|
-
///
|
|
219
|
-
/// This function is called automatically on first extraction operation.
|
|
220
|
-
/// It's safe to call multiple times - registration only happens once,
|
|
221
|
-
/// unless the registry was cleared, in which case extractors are re-registered.
|
|
222
|
-
pub fn ensure_initialized() -> Result<()> {
|
|
223
|
-
EXTRACTORS_INITIALIZED
|
|
224
|
-
.as_ref()
|
|
225
|
-
.map(|_| ())
|
|
226
|
-
.map_err(|e| crate::KreuzbergError::Plugin {
|
|
227
|
-
message: format!("Failed to register default extractors: {}", e),
|
|
228
|
-
plugin_name: "built-in-extractors".to_string(),
|
|
229
|
-
})?;
|
|
230
|
-
|
|
231
|
-
let registry = get_document_extractor_registry();
|
|
232
|
-
let registry_guard = registry
|
|
233
|
-
.read()
|
|
234
|
-
.map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
|
|
235
|
-
|
|
236
|
-
if registry_guard.list().is_empty() {
|
|
237
|
-
drop(registry_guard);
|
|
238
|
-
register_default_extractors()?;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
Ok(())
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
/// Register all built-in extractors with the global registry.
|
|
245
|
-
///
|
|
246
|
-
/// This function should be called once at application startup to register
|
|
247
|
-
/// the default extractors (PlainText, Markdown, XML, etc.).
|
|
248
|
-
///
|
|
249
|
-
/// **Note:** This is called automatically on first extraction operation.
|
|
250
|
-
/// Explicit calling is optional.
|
|
251
|
-
///
|
|
252
|
-
/// # Example
|
|
253
|
-
///
|
|
254
|
-
/// ```rust
|
|
255
|
-
/// use kreuzberg::extractors::register_default_extractors;
|
|
256
|
-
///
|
|
257
|
-
/// # fn main() -> kreuzberg::Result<()> {
|
|
258
|
-
/// register_default_extractors()?;
|
|
259
|
-
/// # Ok(())
|
|
260
|
-
/// # }
|
|
261
|
-
/// ```
|
|
262
|
-
pub fn register_default_extractors() -> Result<()> {
|
|
263
|
-
let registry = get_document_extractor_registry();
|
|
264
|
-
let mut registry = registry
|
|
265
|
-
.write()
|
|
266
|
-
.map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
|
|
267
|
-
|
|
268
|
-
registry.register(Arc::new(PlainTextExtractor::new()))?;
|
|
269
|
-
registry.register(Arc::new(MarkdownExtractor::new()))?;
|
|
270
|
-
registry.register(Arc::new(StructuredExtractor::new()))?;
|
|
271
|
-
|
|
272
|
-
#[cfg(feature = "ocr")]
|
|
273
|
-
registry.register(Arc::new(ImageExtractor::new()))?;
|
|
274
|
-
|
|
275
|
-
#[cfg(feature = "xml")]
|
|
276
|
-
registry.register(Arc::new(XmlExtractor::new()))?;
|
|
277
|
-
|
|
278
|
-
#[cfg(feature = "pdf")]
|
|
279
|
-
registry.register(Arc::new(PdfExtractor::new()))?;
|
|
280
|
-
|
|
281
|
-
#[cfg(feature = "excel")]
|
|
282
|
-
registry.register(Arc::new(ExcelExtractor::new()))?;
|
|
283
|
-
|
|
284
|
-
#[cfg(feature = "office")]
|
|
285
|
-
{
|
|
286
|
-
registry.register(Arc::new(EnhancedMarkdownExtractor::new()))?;
|
|
287
|
-
registry.register(Arc::new(BibtexExtractor::new()))?;
|
|
288
|
-
registry.register(Arc::new(EpubExtractor::new()))?;
|
|
289
|
-
registry.register(Arc::new(FictionBookExtractor::new()))?;
|
|
290
|
-
registry.register(Arc::new(RtfExtractor::new()))?;
|
|
291
|
-
registry.register(Arc::new(RstExtractor::new()))?;
|
|
292
|
-
registry.register(Arc::new(LatexExtractor::new()))?;
|
|
293
|
-
registry.register(Arc::new(JupyterExtractor::new()))?;
|
|
294
|
-
registry.register(Arc::new(OrgModeExtractor::new()))?;
|
|
295
|
-
registry.register(Arc::new(OpmlExtractor::new()))?;
|
|
296
|
-
registry.register(Arc::new(TypstExtractor::new()))?;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
300
|
-
{
|
|
301
|
-
registry.register(Arc::new(DocxExtractor::new()))?;
|
|
302
|
-
registry.register(Arc::new(PptxExtractor::new()))?;
|
|
303
|
-
registry.register(Arc::new(OdtExtractor::new()))?;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
#[cfg(feature = "email")]
|
|
307
|
-
registry.register(Arc::new(EmailExtractor::new()))?;
|
|
308
|
-
|
|
309
|
-
#[cfg(feature = "html")]
|
|
310
|
-
registry.register(Arc::new(HtmlExtractor::new()))?;
|
|
311
|
-
|
|
312
|
-
#[cfg(feature = "archives")]
|
|
313
|
-
{
|
|
314
|
-
registry.register(Arc::new(ZipExtractor::new()))?;
|
|
315
|
-
registry.register(Arc::new(TarExtractor::new()))?;
|
|
316
|
-
registry.register(Arc::new(SevenZExtractor::new()))?;
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
Ok(())
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
#[cfg(test)]
|
|
323
|
-
mod tests {
|
|
324
|
-
use super::*;
|
|
325
|
-
|
|
326
|
-
#[test]
|
|
327
|
-
fn test_register_default_extractors() {
|
|
328
|
-
let registry = get_document_extractor_registry();
|
|
329
|
-
{
|
|
330
|
-
let mut reg = registry
|
|
331
|
-
.write()
|
|
332
|
-
.expect("Failed to acquire write lock on registry in test");
|
|
333
|
-
*reg = crate::plugins::registry::DocumentExtractorRegistry::new();
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
register_default_extractors().expect("Failed to register extractors");
|
|
337
|
-
|
|
338
|
-
let reg = registry
|
|
339
|
-
.read()
|
|
340
|
-
.expect("Failed to acquire read lock on registry in test");
|
|
341
|
-
let extractor_names = reg.list();
|
|
342
|
-
|
|
343
|
-
#[allow(unused_mut)]
|
|
344
|
-
let mut expected_count = 3;
|
|
345
|
-
assert!(extractor_names.contains(&"plain-text-extractor".to_string()));
|
|
346
|
-
assert!(extractor_names.contains(&"markdown-extractor".to_string()));
|
|
347
|
-
assert!(extractor_names.contains(&"structured-extractor".to_string()));
|
|
348
|
-
|
|
349
|
-
#[cfg(feature = "ocr")]
|
|
350
|
-
{
|
|
351
|
-
expected_count += 1;
|
|
352
|
-
assert!(extractor_names.contains(&"image-extractor".to_string()));
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
#[cfg(feature = "xml")]
|
|
356
|
-
{
|
|
357
|
-
expected_count += 1;
|
|
358
|
-
assert!(extractor_names.contains(&"xml-extractor".to_string()));
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
#[cfg(feature = "pdf")]
|
|
362
|
-
{
|
|
363
|
-
expected_count += 1;
|
|
364
|
-
assert!(extractor_names.contains(&"pdf-extractor".to_string()));
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
#[cfg(feature = "excel")]
|
|
368
|
-
{
|
|
369
|
-
expected_count += 1;
|
|
370
|
-
assert!(extractor_names.contains(&"excel-extractor".to_string()));
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
#[cfg(feature = "office")]
|
|
374
|
-
{
|
|
375
|
-
expected_count += 10;
|
|
376
|
-
assert!(extractor_names.contains(&"markdown-extractor".to_string()));
|
|
377
|
-
assert!(extractor_names.contains(&"bibtex-extractor".to_string()));
|
|
378
|
-
assert!(extractor_names.contains(&"epub-extractor".to_string()));
|
|
379
|
-
assert!(extractor_names.contains(&"fictionbook-extractor".to_string()));
|
|
380
|
-
assert!(extractor_names.contains(&"rtf-extractor".to_string()));
|
|
381
|
-
assert!(extractor_names.contains(&"rst-extractor".to_string()));
|
|
382
|
-
assert!(extractor_names.contains(&"latex-extractor".to_string()));
|
|
383
|
-
assert!(extractor_names.contains(&"jupyter-extractor".to_string()));
|
|
384
|
-
assert!(extractor_names.contains(&"orgmode-extractor".to_string()));
|
|
385
|
-
assert!(extractor_names.contains(&"opml-extractor".to_string()));
|
|
386
|
-
assert!(extractor_names.contains(&"typst-extractor".to_string()));
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
390
|
-
{
|
|
391
|
-
expected_count += 3;
|
|
392
|
-
assert!(extractor_names.contains(&"docx-extractor".to_string()));
|
|
393
|
-
assert!(extractor_names.contains(&"pptx-extractor".to_string()));
|
|
394
|
-
assert!(extractor_names.contains(&"odt-extractor".to_string()));
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
#[cfg(feature = "email")]
|
|
398
|
-
{
|
|
399
|
-
expected_count += 1;
|
|
400
|
-
assert!(extractor_names.contains(&"email-extractor".to_string()));
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
#[cfg(feature = "html")]
|
|
404
|
-
{
|
|
405
|
-
expected_count += 1;
|
|
406
|
-
assert!(extractor_names.contains(&"html-extractor".to_string()));
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
#[cfg(feature = "archives")]
|
|
410
|
-
{
|
|
411
|
-
expected_count += 3;
|
|
412
|
-
assert!(extractor_names.contains(&"zip-extractor".to_string()));
|
|
413
|
-
assert!(extractor_names.contains(&"tar-extractor".to_string()));
|
|
414
|
-
assert!(extractor_names.contains(&"7z-extractor".to_string()));
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
assert_eq!(
|
|
418
|
-
extractor_names.len(),
|
|
419
|
-
expected_count,
|
|
420
|
-
"Expected {} extractors based on enabled features",
|
|
421
|
-
expected_count
|
|
422
|
-
);
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
#[test]
|
|
426
|
-
fn test_ensure_initialized() {
|
|
427
|
-
ensure_initialized().expect("Failed to ensure extractors initialized");
|
|
428
|
-
}
|
|
429
|
-
}
|
|
1
|
+
//! Built-in document extractors.
|
|
2
|
+
//!
|
|
3
|
+
//! This module contains the default extractors that ship with Kreuzberg.
|
|
4
|
+
//! All extractors implement the `DocumentExtractor` plugin trait.
|
|
5
|
+
|
|
6
|
+
use crate::Result;
|
|
7
|
+
use crate::core::config::ExtractionConfig;
|
|
8
|
+
use crate::plugins::registry::get_document_extractor_registry;
|
|
9
|
+
use crate::types::ExtractionResult;
|
|
10
|
+
use once_cell::sync::Lazy;
|
|
11
|
+
use std::sync::Arc;
|
|
12
|
+
|
|
13
|
+
/// Trait for extractors that can work synchronously (WASM-compatible).
|
|
14
|
+
///
|
|
15
|
+
/// This trait defines the synchronous extraction interface for WASM targets and other
|
|
16
|
+
/// environments where async/tokio runtimes are not available or desirable.
|
|
17
|
+
///
|
|
18
|
+
/// # Implementation
|
|
19
|
+
///
|
|
20
|
+
/// Extractors that need to support WASM should implement this trait in addition to
|
|
21
|
+
/// the async `DocumentExtractor` trait. This allows the same extractor to work in both
|
|
22
|
+
/// environments by delegating to the sync implementation.
|
|
23
|
+
///
|
|
24
|
+
/// # MIME Type Validation
|
|
25
|
+
///
|
|
26
|
+
/// The `mime_type` parameter is guaranteed to be already validated.
|
|
27
|
+
///
|
|
28
|
+
/// # Example
|
|
29
|
+
///
|
|
30
|
+
/// ```rust,ignore
|
|
31
|
+
/// impl SyncExtractor for PlainTextExtractor {
|
|
32
|
+
/// fn extract_sync(&self, content: &[u8], config: &ExtractionConfig) -> Result<ExtractionResult> {
|
|
33
|
+
/// let text = String::from_utf8_lossy(content).to_string();
|
|
34
|
+
/// Ok(ExtractionResult {
|
|
35
|
+
/// content: text,
|
|
36
|
+
/// mime_type: "text/plain".to_string(),
|
|
37
|
+
/// metadata: Metadata::default(),
|
|
38
|
+
/// tables: vec![],
|
|
39
|
+
/// detected_languages: None,
|
|
40
|
+
/// chunks: None,
|
|
41
|
+
/// images: None,
|
|
42
|
+
/// })
|
|
43
|
+
/// }
|
|
44
|
+
/// }
|
|
45
|
+
/// ```
|
|
46
|
+
pub trait SyncExtractor {
|
|
47
|
+
/// Extract content from a byte array synchronously.
|
|
48
|
+
///
|
|
49
|
+
/// This method performs extraction without requiring an async runtime.
|
|
50
|
+
/// It is called by `extract_bytes_sync()` when the `tokio-runtime` feature is disabled.
|
|
51
|
+
///
|
|
52
|
+
/// # Arguments
|
|
53
|
+
///
|
|
54
|
+
/// * `content` - Raw document bytes
|
|
55
|
+
/// * `mime_type` - MIME type of the document (already validated)
|
|
56
|
+
/// * `config` - Extraction configuration
|
|
57
|
+
///
|
|
58
|
+
/// # Returns
|
|
59
|
+
///
|
|
60
|
+
/// An `ExtractionResult` containing the extracted content and metadata.
|
|
61
|
+
fn extract_sync(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult>;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
pub mod structured;
|
|
65
|
+
pub mod text;
|
|
66
|
+
|
|
67
|
+
#[cfg(feature = "archives")]
|
|
68
|
+
pub mod security;
|
|
69
|
+
|
|
70
|
+
#[cfg(feature = "ocr")]
|
|
71
|
+
pub mod image;
|
|
72
|
+
|
|
73
|
+
#[cfg(feature = "archives")]
|
|
74
|
+
pub mod archive;
|
|
75
|
+
|
|
76
|
+
#[cfg(feature = "email")]
|
|
77
|
+
pub mod email;
|
|
78
|
+
|
|
79
|
+
#[cfg(feature = "excel")]
|
|
80
|
+
pub mod excel;
|
|
81
|
+
|
|
82
|
+
#[cfg(feature = "html")]
|
|
83
|
+
pub mod html;
|
|
84
|
+
|
|
85
|
+
#[cfg(feature = "office")]
|
|
86
|
+
pub mod bibtex;
|
|
87
|
+
|
|
88
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
89
|
+
pub mod docx;
|
|
90
|
+
|
|
91
|
+
#[cfg(feature = "office")]
|
|
92
|
+
pub mod epub;
|
|
93
|
+
|
|
94
|
+
#[cfg(feature = "office")]
|
|
95
|
+
pub mod fictionbook;
|
|
96
|
+
|
|
97
|
+
#[cfg(feature = "office")]
|
|
98
|
+
pub mod markdown;
|
|
99
|
+
|
|
100
|
+
#[cfg(feature = "office")]
|
|
101
|
+
pub mod rst;
|
|
102
|
+
|
|
103
|
+
#[cfg(feature = "office")]
|
|
104
|
+
pub mod latex;
|
|
105
|
+
|
|
106
|
+
#[cfg(feature = "office")]
|
|
107
|
+
pub mod jupyter;
|
|
108
|
+
|
|
109
|
+
#[cfg(feature = "office")]
|
|
110
|
+
pub mod orgmode;
|
|
111
|
+
|
|
112
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
113
|
+
pub mod odt;
|
|
114
|
+
|
|
115
|
+
#[cfg(feature = "office")]
|
|
116
|
+
pub mod opml;
|
|
117
|
+
|
|
118
|
+
#[cfg(feature = "office")]
|
|
119
|
+
pub mod typst;
|
|
120
|
+
|
|
121
|
+
#[cfg(feature = "xml")]
|
|
122
|
+
pub mod jats;
|
|
123
|
+
|
|
124
|
+
#[cfg(feature = "pdf")]
|
|
125
|
+
pub mod pdf;
|
|
126
|
+
|
|
127
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
128
|
+
pub mod pptx;
|
|
129
|
+
|
|
130
|
+
#[cfg(feature = "office")]
|
|
131
|
+
pub mod rtf;
|
|
132
|
+
|
|
133
|
+
#[cfg(feature = "xml")]
|
|
134
|
+
pub mod xml;
|
|
135
|
+
|
|
136
|
+
#[cfg(feature = "xml")]
|
|
137
|
+
pub mod docbook;
|
|
138
|
+
|
|
139
|
+
pub use structured::StructuredExtractor;
|
|
140
|
+
pub use text::{MarkdownExtractor, PlainTextExtractor};
|
|
141
|
+
|
|
142
|
+
#[cfg(feature = "ocr")]
|
|
143
|
+
pub use image::ImageExtractor;
|
|
144
|
+
|
|
145
|
+
#[cfg(feature = "archives")]
|
|
146
|
+
pub use archive::{SevenZExtractor, TarExtractor, ZipExtractor};
|
|
147
|
+
|
|
148
|
+
#[cfg(feature = "email")]
|
|
149
|
+
pub use email::EmailExtractor;
|
|
150
|
+
|
|
151
|
+
#[cfg(feature = "excel")]
|
|
152
|
+
pub use excel::ExcelExtractor;
|
|
153
|
+
|
|
154
|
+
#[cfg(feature = "html")]
|
|
155
|
+
pub use html::HtmlExtractor;
|
|
156
|
+
|
|
157
|
+
#[cfg(feature = "office")]
|
|
158
|
+
pub use bibtex::BibtexExtractor;
|
|
159
|
+
|
|
160
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
161
|
+
pub use docx::DocxExtractor;
|
|
162
|
+
|
|
163
|
+
#[cfg(feature = "office")]
|
|
164
|
+
pub use epub::EpubExtractor;
|
|
165
|
+
|
|
166
|
+
#[cfg(feature = "office")]
|
|
167
|
+
pub use fictionbook::FictionBookExtractor;
|
|
168
|
+
|
|
169
|
+
#[cfg(feature = "office")]
|
|
170
|
+
pub use markdown::MarkdownExtractor as EnhancedMarkdownExtractor;
|
|
171
|
+
|
|
172
|
+
#[cfg(feature = "office")]
|
|
173
|
+
pub use rst::RstExtractor;
|
|
174
|
+
|
|
175
|
+
#[cfg(feature = "office")]
|
|
176
|
+
pub use latex::LatexExtractor;
|
|
177
|
+
|
|
178
|
+
#[cfg(feature = "office")]
|
|
179
|
+
pub use jupyter::JupyterExtractor;
|
|
180
|
+
|
|
181
|
+
#[cfg(feature = "office")]
|
|
182
|
+
pub use orgmode::OrgModeExtractor;
|
|
183
|
+
|
|
184
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
185
|
+
pub use odt::OdtExtractor;
|
|
186
|
+
|
|
187
|
+
#[cfg(feature = "xml")]
|
|
188
|
+
pub use jats::JatsExtractor;
|
|
189
|
+
|
|
190
|
+
#[cfg(feature = "office")]
|
|
191
|
+
pub use opml::OpmlExtractor;
|
|
192
|
+
|
|
193
|
+
#[cfg(feature = "office")]
|
|
194
|
+
pub use typst::TypstExtractor;
|
|
195
|
+
|
|
196
|
+
#[cfg(feature = "pdf")]
|
|
197
|
+
pub use pdf::PdfExtractor;
|
|
198
|
+
|
|
199
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
200
|
+
pub use pptx::PptxExtractor;
|
|
201
|
+
|
|
202
|
+
#[cfg(feature = "office")]
|
|
203
|
+
pub use rtf::RtfExtractor;
|
|
204
|
+
|
|
205
|
+
#[cfg(feature = "xml")]
|
|
206
|
+
pub use xml::XmlExtractor;
|
|
207
|
+
|
|
208
|
+
#[cfg(feature = "xml")]
|
|
209
|
+
pub use docbook::DocbookExtractor;
|
|
210
|
+
|
|
211
|
+
/// Lazy-initialized flag that ensures extractors are registered exactly once.
|
|
212
|
+
///
|
|
213
|
+
/// This static is accessed on first extraction operation to automatically
|
|
214
|
+
/// register all built-in extractors with the plugin registry.
|
|
215
|
+
static EXTRACTORS_INITIALIZED: Lazy<Result<()>> = Lazy::new(register_default_extractors);
|
|
216
|
+
|
|
217
|
+
/// Ensure built-in extractors are registered.
|
|
218
|
+
///
|
|
219
|
+
/// This function is called automatically on first extraction operation.
|
|
220
|
+
/// It's safe to call multiple times - registration only happens once,
|
|
221
|
+
/// unless the registry was cleared, in which case extractors are re-registered.
|
|
222
|
+
pub fn ensure_initialized() -> Result<()> {
|
|
223
|
+
EXTRACTORS_INITIALIZED
|
|
224
|
+
.as_ref()
|
|
225
|
+
.map(|_| ())
|
|
226
|
+
.map_err(|e| crate::KreuzbergError::Plugin {
|
|
227
|
+
message: format!("Failed to register default extractors: {}", e),
|
|
228
|
+
plugin_name: "built-in-extractors".to_string(),
|
|
229
|
+
})?;
|
|
230
|
+
|
|
231
|
+
let registry = get_document_extractor_registry();
|
|
232
|
+
let registry_guard = registry
|
|
233
|
+
.read()
|
|
234
|
+
.map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
|
|
235
|
+
|
|
236
|
+
if registry_guard.list().is_empty() {
|
|
237
|
+
drop(registry_guard);
|
|
238
|
+
register_default_extractors()?;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
Ok(())
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/// Register all built-in extractors with the global registry.
|
|
245
|
+
///
|
|
246
|
+
/// This function should be called once at application startup to register
|
|
247
|
+
/// the default extractors (PlainText, Markdown, XML, etc.).
|
|
248
|
+
///
|
|
249
|
+
/// **Note:** This is called automatically on first extraction operation.
|
|
250
|
+
/// Explicit calling is optional.
|
|
251
|
+
///
|
|
252
|
+
/// # Example
|
|
253
|
+
///
|
|
254
|
+
/// ```rust
|
|
255
|
+
/// use kreuzberg::extractors::register_default_extractors;
|
|
256
|
+
///
|
|
257
|
+
/// # fn main() -> kreuzberg::Result<()> {
|
|
258
|
+
/// register_default_extractors()?;
|
|
259
|
+
/// # Ok(())
|
|
260
|
+
/// # }
|
|
261
|
+
/// ```
|
|
262
|
+
pub fn register_default_extractors() -> Result<()> {
|
|
263
|
+
let registry = get_document_extractor_registry();
|
|
264
|
+
let mut registry = registry
|
|
265
|
+
.write()
|
|
266
|
+
.map_err(|e| crate::KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
|
|
267
|
+
|
|
268
|
+
registry.register(Arc::new(PlainTextExtractor::new()))?;
|
|
269
|
+
registry.register(Arc::new(MarkdownExtractor::new()))?;
|
|
270
|
+
registry.register(Arc::new(StructuredExtractor::new()))?;
|
|
271
|
+
|
|
272
|
+
#[cfg(feature = "ocr")]
|
|
273
|
+
registry.register(Arc::new(ImageExtractor::new()))?;
|
|
274
|
+
|
|
275
|
+
#[cfg(feature = "xml")]
|
|
276
|
+
registry.register(Arc::new(XmlExtractor::new()))?;
|
|
277
|
+
|
|
278
|
+
#[cfg(feature = "pdf")]
|
|
279
|
+
registry.register(Arc::new(PdfExtractor::new()))?;
|
|
280
|
+
|
|
281
|
+
#[cfg(feature = "excel")]
|
|
282
|
+
registry.register(Arc::new(ExcelExtractor::new()))?;
|
|
283
|
+
|
|
284
|
+
#[cfg(feature = "office")]
|
|
285
|
+
{
|
|
286
|
+
registry.register(Arc::new(EnhancedMarkdownExtractor::new()))?;
|
|
287
|
+
registry.register(Arc::new(BibtexExtractor::new()))?;
|
|
288
|
+
registry.register(Arc::new(EpubExtractor::new()))?;
|
|
289
|
+
registry.register(Arc::new(FictionBookExtractor::new()))?;
|
|
290
|
+
registry.register(Arc::new(RtfExtractor::new()))?;
|
|
291
|
+
registry.register(Arc::new(RstExtractor::new()))?;
|
|
292
|
+
registry.register(Arc::new(LatexExtractor::new()))?;
|
|
293
|
+
registry.register(Arc::new(JupyterExtractor::new()))?;
|
|
294
|
+
registry.register(Arc::new(OrgModeExtractor::new()))?;
|
|
295
|
+
registry.register(Arc::new(OpmlExtractor::new()))?;
|
|
296
|
+
registry.register(Arc::new(TypstExtractor::new()))?;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
300
|
+
{
|
|
301
|
+
registry.register(Arc::new(DocxExtractor::new()))?;
|
|
302
|
+
registry.register(Arc::new(PptxExtractor::new()))?;
|
|
303
|
+
registry.register(Arc::new(OdtExtractor::new()))?;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
#[cfg(feature = "email")]
|
|
307
|
+
registry.register(Arc::new(EmailExtractor::new()))?;
|
|
308
|
+
|
|
309
|
+
#[cfg(feature = "html")]
|
|
310
|
+
registry.register(Arc::new(HtmlExtractor::new()))?;
|
|
311
|
+
|
|
312
|
+
#[cfg(feature = "archives")]
|
|
313
|
+
{
|
|
314
|
+
registry.register(Arc::new(ZipExtractor::new()))?;
|
|
315
|
+
registry.register(Arc::new(TarExtractor::new()))?;
|
|
316
|
+
registry.register(Arc::new(SevenZExtractor::new()))?;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
Ok(())
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
#[cfg(test)]
|
|
323
|
+
mod tests {
|
|
324
|
+
use super::*;
|
|
325
|
+
|
|
326
|
+
#[test]
|
|
327
|
+
fn test_register_default_extractors() {
|
|
328
|
+
let registry = get_document_extractor_registry();
|
|
329
|
+
{
|
|
330
|
+
let mut reg = registry
|
|
331
|
+
.write()
|
|
332
|
+
.expect("Failed to acquire write lock on registry in test");
|
|
333
|
+
*reg = crate::plugins::registry::DocumentExtractorRegistry::new();
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
register_default_extractors().expect("Failed to register extractors");
|
|
337
|
+
|
|
338
|
+
let reg = registry
|
|
339
|
+
.read()
|
|
340
|
+
.expect("Failed to acquire read lock on registry in test");
|
|
341
|
+
let extractor_names = reg.list();
|
|
342
|
+
|
|
343
|
+
#[allow(unused_mut)]
|
|
344
|
+
let mut expected_count = 3;
|
|
345
|
+
assert!(extractor_names.contains(&"plain-text-extractor".to_string()));
|
|
346
|
+
assert!(extractor_names.contains(&"markdown-extractor".to_string()));
|
|
347
|
+
assert!(extractor_names.contains(&"structured-extractor".to_string()));
|
|
348
|
+
|
|
349
|
+
#[cfg(feature = "ocr")]
|
|
350
|
+
{
|
|
351
|
+
expected_count += 1;
|
|
352
|
+
assert!(extractor_names.contains(&"image-extractor".to_string()));
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
#[cfg(feature = "xml")]
|
|
356
|
+
{
|
|
357
|
+
expected_count += 1;
|
|
358
|
+
assert!(extractor_names.contains(&"xml-extractor".to_string()));
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
#[cfg(feature = "pdf")]
|
|
362
|
+
{
|
|
363
|
+
expected_count += 1;
|
|
364
|
+
assert!(extractor_names.contains(&"pdf-extractor".to_string()));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
#[cfg(feature = "excel")]
|
|
368
|
+
{
|
|
369
|
+
expected_count += 1;
|
|
370
|
+
assert!(extractor_names.contains(&"excel-extractor".to_string()));
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
#[cfg(feature = "office")]
|
|
374
|
+
{
|
|
375
|
+
expected_count += 10;
|
|
376
|
+
assert!(extractor_names.contains(&"markdown-extractor".to_string()));
|
|
377
|
+
assert!(extractor_names.contains(&"bibtex-extractor".to_string()));
|
|
378
|
+
assert!(extractor_names.contains(&"epub-extractor".to_string()));
|
|
379
|
+
assert!(extractor_names.contains(&"fictionbook-extractor".to_string()));
|
|
380
|
+
assert!(extractor_names.contains(&"rtf-extractor".to_string()));
|
|
381
|
+
assert!(extractor_names.contains(&"rst-extractor".to_string()));
|
|
382
|
+
assert!(extractor_names.contains(&"latex-extractor".to_string()));
|
|
383
|
+
assert!(extractor_names.contains(&"jupyter-extractor".to_string()));
|
|
384
|
+
assert!(extractor_names.contains(&"orgmode-extractor".to_string()));
|
|
385
|
+
assert!(extractor_names.contains(&"opml-extractor".to_string()));
|
|
386
|
+
assert!(extractor_names.contains(&"typst-extractor".to_string()));
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
#[cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
390
|
+
{
|
|
391
|
+
expected_count += 3;
|
|
392
|
+
assert!(extractor_names.contains(&"docx-extractor".to_string()));
|
|
393
|
+
assert!(extractor_names.contains(&"pptx-extractor".to_string()));
|
|
394
|
+
assert!(extractor_names.contains(&"odt-extractor".to_string()));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
#[cfg(feature = "email")]
|
|
398
|
+
{
|
|
399
|
+
expected_count += 1;
|
|
400
|
+
assert!(extractor_names.contains(&"email-extractor".to_string()));
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
#[cfg(feature = "html")]
|
|
404
|
+
{
|
|
405
|
+
expected_count += 1;
|
|
406
|
+
assert!(extractor_names.contains(&"html-extractor".to_string()));
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
#[cfg(feature = "archives")]
|
|
410
|
+
{
|
|
411
|
+
expected_count += 3;
|
|
412
|
+
assert!(extractor_names.contains(&"zip-extractor".to_string()));
|
|
413
|
+
assert!(extractor_names.contains(&"tar-extractor".to_string()));
|
|
414
|
+
assert!(extractor_names.contains(&"7z-extractor".to_string()));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
assert_eq!(
|
|
418
|
+
extractor_names.len(),
|
|
419
|
+
expected_count,
|
|
420
|
+
"Expected {} extractors based on enabled features",
|
|
421
|
+
expected_count
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
#[test]
|
|
426
|
+
fn test_ensure_initialized() {
|
|
427
|
+
ensure_initialized().expect("Failed to ensure extractors initialized");
|
|
428
|
+
}
|
|
429
|
+
}
|