kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +104 -2
- data/README.md +454 -454
- data/Rakefile +33 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6750 -6941
- data/ext/kreuzberg_rb/native/Cargo.toml +53 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +52 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3158
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -214
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -81
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -80
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -340
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -109
- data/lib/{pdfium.dll → libpdfium.so} +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -546
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +2 -2
- data/vendor/kreuzberg/Cargo.toml +5 -5
- data/vendor/kreuzberg/README.md +230 -230
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +887 -843
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +87 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1184
- data/vendor/kreuzberg/src/embeddings.rs +500 -500
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +634 -601
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -574
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -749
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -44
- data/vendor/kreuzberg/src/pdf/bundled.rs +452 -346
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -68
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -420
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +165 -164
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
- data/vendor/kreuzberg-tesseract/LICENSE +22 -22
- data/vendor/kreuzberg-tesseract/README.md +399 -399
- data/vendor/kreuzberg-tesseract/build.rs +1354 -1354
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -211
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +81 -22
- data/vendor/kreuzberg-ffi/Cargo.toml +0 -63
- data/vendor/kreuzberg-ffi/README.md +0 -851
- data/vendor/kreuzberg-ffi/build.rs +0 -176
- data/vendor/kreuzberg-ffi/cbindgen.toml +0 -27
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +0 -12
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +0 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +0 -1087
- data/vendor/kreuzberg-ffi/src/lib.rs +0 -3616
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +0 -247
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +0 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +0 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +0 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +0 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +0 -470
|
@@ -1,639 +1,639 @@
|
|
|
1
|
-
//! OCR backend plugin trait.
|
|
2
|
-
//!
|
|
3
|
-
//! This module defines the trait for implementing custom OCR backends.
|
|
4
|
-
|
|
5
|
-
use crate::Result;
|
|
6
|
-
use crate::core::config::OcrConfig;
|
|
7
|
-
use crate::plugins::Plugin;
|
|
8
|
-
use crate::types::ExtractionResult;
|
|
9
|
-
use async_trait::async_trait;
|
|
10
|
-
use std::path::Path;
|
|
11
|
-
use std::sync::Arc;
|
|
12
|
-
|
|
13
|
-
#[cfg(not(feature = "tokio-runtime"))]
|
|
14
|
-
use crate::KreuzbergError;
|
|
15
|
-
|
|
16
|
-
/// OCR backend types.
|
|
17
|
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
18
|
-
pub enum OcrBackendType {
|
|
19
|
-
/// Tesseract OCR (native Rust binding)
|
|
20
|
-
Tesseract,
|
|
21
|
-
/// EasyOCR (Python-based, via FFI)
|
|
22
|
-
EasyOCR,
|
|
23
|
-
/// PaddleOCR (Python-based, via FFI)
|
|
24
|
-
PaddleOCR,
|
|
25
|
-
/// Custom/third-party OCR backend
|
|
26
|
-
Custom,
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/// Trait for OCR backend plugins.
|
|
30
|
-
///
|
|
31
|
-
/// Implement this trait to add custom OCR capabilities. OCR backends can be:
|
|
32
|
-
/// - Native Rust implementations (like Tesseract)
|
|
33
|
-
/// - FFI bridges to Python libraries (like EasyOCR, PaddleOCR)
|
|
34
|
-
/// - Cloud-based OCR services (Google Vision, AWS Textract, etc.)
|
|
35
|
-
///
|
|
36
|
-
/// # Thread Safety
|
|
37
|
-
///
|
|
38
|
-
/// OCR backends must be thread-safe (`Send + Sync`) to support concurrent processing.
|
|
39
|
-
///
|
|
40
|
-
/// # Example
|
|
41
|
-
///
|
|
42
|
-
/// ```rust
|
|
43
|
-
/// use kreuzberg::plugins::{Plugin, OcrBackend, OcrBackendType};
|
|
44
|
-
/// use kreuzberg::{Result, OcrConfig};
|
|
45
|
-
/// use async_trait::async_trait;
|
|
46
|
-
/// use std::path::Path;
|
|
47
|
-
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
48
|
-
///
|
|
49
|
-
/// struct CustomOcrBackend;
|
|
50
|
-
///
|
|
51
|
-
/// impl Plugin for CustomOcrBackend {
|
|
52
|
-
/// fn name(&self) -> &str { "custom-ocr" }
|
|
53
|
-
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
54
|
-
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
55
|
-
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
56
|
-
/// }
|
|
57
|
-
///
|
|
58
|
-
/// #[async_trait]
|
|
59
|
-
/// impl OcrBackend for CustomOcrBackend {
|
|
60
|
-
/// async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult> {
|
|
61
|
-
/// // Implement OCR logic here
|
|
62
|
-
/// Ok(ExtractionResult {
|
|
63
|
-
/// content: "Extracted text".to_string(),
|
|
64
|
-
/// mime_type: "text/plain".to_string(),
|
|
65
|
-
/// metadata: Metadata::default(),
|
|
66
|
-
/// tables: vec![],
|
|
67
|
-
/// detected_languages: None,
|
|
68
|
-
/// chunks: None,
|
|
69
|
-
/// images: None,
|
|
70
|
-
/// pages: None,
|
|
71
|
-
/// })
|
|
72
|
-
/// }
|
|
73
|
-
///
|
|
74
|
-
/// async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
|
|
75
|
-
/// let bytes = std::fs::read(path)?;
|
|
76
|
-
/// self.process_image(&bytes, config).await
|
|
77
|
-
/// }
|
|
78
|
-
///
|
|
79
|
-
/// fn supports_language(&self, lang: &str) -> bool {
|
|
80
|
-
/// matches!(lang, "eng" | "deu" | "fra")
|
|
81
|
-
/// }
|
|
82
|
-
///
|
|
83
|
-
/// fn backend_type(&self) -> OcrBackendType {
|
|
84
|
-
/// OcrBackendType::Custom
|
|
85
|
-
/// }
|
|
86
|
-
/// }
|
|
87
|
-
/// ```
|
|
88
|
-
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
|
|
89
|
-
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
|
|
90
|
-
pub trait OcrBackend: Plugin {
|
|
91
|
-
/// Process an image and extract text via OCR.
|
|
92
|
-
///
|
|
93
|
-
/// # Arguments
|
|
94
|
-
///
|
|
95
|
-
/// * `image_bytes` - Raw image data (JPEG, PNG, TIFF, etc.)
|
|
96
|
-
/// * `config` - OCR configuration (language, PSM mode, etc.)
|
|
97
|
-
///
|
|
98
|
-
/// # Returns
|
|
99
|
-
///
|
|
100
|
-
/// An `ExtractionResult` containing the extracted text and metadata.
|
|
101
|
-
///
|
|
102
|
-
/// # Errors
|
|
103
|
-
///
|
|
104
|
-
/// - `KreuzbergError::Ocr` - OCR processing failed
|
|
105
|
-
/// - `KreuzbergError::Validation` - Invalid image format or configuration
|
|
106
|
-
/// - `KreuzbergError::Io` - I/O errors (these always bubble up)
|
|
107
|
-
///
|
|
108
|
-
/// # Example
|
|
109
|
-
///
|
|
110
|
-
/// ```rust
|
|
111
|
-
/// # use kreuzberg::plugins::{Plugin, OcrBackend};
|
|
112
|
-
/// # use kreuzberg::{Result, OcrConfig};
|
|
113
|
-
/// # use async_trait::async_trait;
|
|
114
|
-
/// # use std::path::Path;
|
|
115
|
-
/// # use kreuzberg::types::{ExtractionResult, Metadata};
|
|
116
|
-
/// # struct MyOcr;
|
|
117
|
-
/// # impl Plugin for MyOcr {
|
|
118
|
-
/// # fn name(&self) -> &str { "my-ocr" }
|
|
119
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
120
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
121
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
122
|
-
/// # }
|
|
123
|
-
/// # use kreuzberg::plugins::OcrBackendType;
|
|
124
|
-
/// # #[async_trait]
|
|
125
|
-
/// # impl OcrBackend for MyOcr {
|
|
126
|
-
/// # fn supports_language(&self, _: &str) -> bool { true }
|
|
127
|
-
/// # fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
128
|
-
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
129
|
-
/// async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult> {
|
|
130
|
-
/// // Validate image format
|
|
131
|
-
/// if image_bytes.is_empty() {
|
|
132
|
-
/// return Err(kreuzberg::KreuzbergError::Validation {
|
|
133
|
-
/// message: "Empty image data".to_string(),
|
|
134
|
-
/// source: None,
|
|
135
|
-
/// });
|
|
136
|
-
/// }
|
|
137
|
-
///
|
|
138
|
-
/// // Perform OCR processing
|
|
139
|
-
/// let text = format!("Extracted text in language: {}", config.language);
|
|
140
|
-
///
|
|
141
|
-
/// Ok(ExtractionResult {
|
|
142
|
-
/// content: text,
|
|
143
|
-
/// mime_type: "text/plain".to_string(),
|
|
144
|
-
/// metadata: Metadata::default(),
|
|
145
|
-
/// tables: vec![],
|
|
146
|
-
/// detected_languages: None,
|
|
147
|
-
/// chunks: None,
|
|
148
|
-
/// images: None,
|
|
149
|
-
/// pages: None,
|
|
150
|
-
/// })
|
|
151
|
-
/// }
|
|
152
|
-
/// # }
|
|
153
|
-
/// ```
|
|
154
|
-
async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult>;
|
|
155
|
-
|
|
156
|
-
/// Process a file and extract text via OCR.
|
|
157
|
-
///
|
|
158
|
-
/// Default implementation reads the file and calls `process_image`.
|
|
159
|
-
/// Override for custom file handling or optimizations.
|
|
160
|
-
///
|
|
161
|
-
/// # Arguments
|
|
162
|
-
///
|
|
163
|
-
/// * `path` - Path to the image file
|
|
164
|
-
/// * `config` - OCR configuration
|
|
165
|
-
///
|
|
166
|
-
/// # Errors
|
|
167
|
-
///
|
|
168
|
-
/// Same as `process_image`, plus file I/O errors.
|
|
169
|
-
async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
|
|
170
|
-
#[cfg(feature = "tokio-runtime")]
|
|
171
|
-
{
|
|
172
|
-
use crate::core::io;
|
|
173
|
-
let bytes = io::read_file_async(path).await?;
|
|
174
|
-
self.process_image(&bytes, config).await
|
|
175
|
-
}
|
|
176
|
-
#[cfg(not(feature = "tokio-runtime"))]
|
|
177
|
-
{
|
|
178
|
-
let _ = (path, config);
|
|
179
|
-
// For WASM and non-tokio environments, file-based OCR is not supported
|
|
180
|
-
// through the default implementation. Implementations must provide their own.
|
|
181
|
-
Err(KreuzbergError::Other(
|
|
182
|
-
"File-based OCR processing requires the tokio-runtime feature".to_string(),
|
|
183
|
-
))
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/// Check if this backend supports a given language code.
|
|
188
|
-
///
|
|
189
|
-
/// # Arguments
|
|
190
|
-
///
|
|
191
|
-
/// * `lang` - ISO 639-2/3 language code (e.g., "eng", "deu", "fra")
|
|
192
|
-
///
|
|
193
|
-
/// # Returns
|
|
194
|
-
///
|
|
195
|
-
/// `true` if the language is supported, `false` otherwise.
|
|
196
|
-
///
|
|
197
|
-
/// # Example
|
|
198
|
-
///
|
|
199
|
-
/// ```rust
|
|
200
|
-
/// # use kreuzberg::plugins::{Plugin, OcrBackend};
|
|
201
|
-
/// # use kreuzberg::Result;
|
|
202
|
-
/// # use async_trait::async_trait;
|
|
203
|
-
/// # use std::path::Path;
|
|
204
|
-
/// # struct MyOcr { languages: Vec<String> }
|
|
205
|
-
/// # impl Plugin for MyOcr {
|
|
206
|
-
/// # fn name(&self) -> &str { "my-ocr" }
|
|
207
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
208
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
209
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
210
|
-
/// # }
|
|
211
|
-
/// # use kreuzberg::plugins::OcrBackendType;
|
|
212
|
-
/// # use kreuzberg::{ExtractionResult, OcrConfig};
|
|
213
|
-
/// # #[async_trait]
|
|
214
|
-
/// # impl OcrBackend for MyOcr {
|
|
215
|
-
/// # fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
216
|
-
/// # async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
217
|
-
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
218
|
-
/// fn supports_language(&self, lang: &str) -> bool {
|
|
219
|
-
/// self.languages.contains(&lang.to_string())
|
|
220
|
-
/// }
|
|
221
|
-
/// # }
|
|
222
|
-
/// ```
|
|
223
|
-
fn supports_language(&self, lang: &str) -> bool;
|
|
224
|
-
|
|
225
|
-
/// Get the backend type identifier.
|
|
226
|
-
///
|
|
227
|
-
/// # Returns
|
|
228
|
-
///
|
|
229
|
-
/// The backend type enum value.
|
|
230
|
-
///
|
|
231
|
-
/// # Example
|
|
232
|
-
///
|
|
233
|
-
/// ```rust
|
|
234
|
-
/// # use kreuzberg::plugins::{Plugin, OcrBackend, OcrBackendType};
|
|
235
|
-
/// # use kreuzberg::Result;
|
|
236
|
-
/// # use async_trait::async_trait;
|
|
237
|
-
/// # use std::path::Path;
|
|
238
|
-
/// # struct TesseractBackend;
|
|
239
|
-
/// # impl Plugin for TesseractBackend {
|
|
240
|
-
/// # fn name(&self) -> &str { "tesseract" }
|
|
241
|
-
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
242
|
-
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
243
|
-
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
244
|
-
/// # }
|
|
245
|
-
/// # use kreuzberg::{ExtractionResult, OcrConfig};
|
|
246
|
-
/// # #[async_trait]
|
|
247
|
-
/// # impl OcrBackend for TesseractBackend {
|
|
248
|
-
/// # fn supports_language(&self, _: &str) -> bool { true }
|
|
249
|
-
/// # async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
250
|
-
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
251
|
-
/// fn backend_type(&self) -> OcrBackendType {
|
|
252
|
-
/// OcrBackendType::Tesseract
|
|
253
|
-
/// }
|
|
254
|
-
/// # }
|
|
255
|
-
/// ```
|
|
256
|
-
fn backend_type(&self) -> OcrBackendType;
|
|
257
|
-
|
|
258
|
-
/// Optional: Get a list of all supported languages.
|
|
259
|
-
///
|
|
260
|
-
/// Defaults to empty list. Override to provide comprehensive language support info.
|
|
261
|
-
fn supported_languages(&self) -> Vec<String> {
|
|
262
|
-
vec![]
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
/// Optional: Check if the backend supports table detection.
|
|
266
|
-
///
|
|
267
|
-
/// Defaults to `false`. Override if your backend can detect and extract tables.
|
|
268
|
-
fn supports_table_detection(&self) -> bool {
|
|
269
|
-
false
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
/// Register an OCR backend with the global registry.
|
|
274
|
-
///
|
|
275
|
-
/// The OCR backend will be registered with its name from the `name()` method
|
|
276
|
-
/// and can be used for OCR processing via the extraction pipeline.
|
|
277
|
-
///
|
|
278
|
-
/// # Arguments
|
|
279
|
-
///
|
|
280
|
-
/// * `backend` - The OCR backend implementation wrapped in Arc
|
|
281
|
-
///
|
|
282
|
-
/// # Returns
|
|
283
|
-
///
|
|
284
|
-
/// - `Ok(())` if registration succeeded
|
|
285
|
-
/// - `Err(...)` if validation failed or initialization failed
|
|
286
|
-
///
|
|
287
|
-
/// # Errors
|
|
288
|
-
///
|
|
289
|
-
/// - `KreuzbergError::Validation` - Invalid backend name (empty or contains whitespace)
|
|
290
|
-
/// - Any error from the backend's `initialize()` method
|
|
291
|
-
///
|
|
292
|
-
/// # Example
|
|
293
|
-
///
|
|
294
|
-
/// ```rust
|
|
295
|
-
/// use kreuzberg::plugins::{Plugin, OcrBackend, register_ocr_backend, OcrBackendType};
|
|
296
|
-
/// use kreuzberg::{Result, OcrConfig};
|
|
297
|
-
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
298
|
-
/// use async_trait::async_trait;
|
|
299
|
-
/// use std::sync::Arc;
|
|
300
|
-
/// use std::path::Path;
|
|
301
|
-
///
|
|
302
|
-
/// struct CustomOcr;
|
|
303
|
-
///
|
|
304
|
-
/// impl Plugin for CustomOcr {
|
|
305
|
-
/// fn name(&self) -> &str { "custom-ocr" }
|
|
306
|
-
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
307
|
-
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
308
|
-
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
309
|
-
/// }
|
|
310
|
-
///
|
|
311
|
-
/// #[async_trait]
|
|
312
|
-
/// impl OcrBackend for CustomOcr {
|
|
313
|
-
/// async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> {
|
|
314
|
-
/// Ok(ExtractionResult {
|
|
315
|
-
/// content: "text".to_string(),
|
|
316
|
-
/// mime_type: "text/plain".to_string(),
|
|
317
|
-
/// metadata: Metadata::default(),
|
|
318
|
-
/// tables: vec![],
|
|
319
|
-
/// detected_languages: None,
|
|
320
|
-
/// chunks: None,
|
|
321
|
-
/// images: None,
|
|
322
|
-
/// pages: None,
|
|
323
|
-
/// })
|
|
324
|
-
/// }
|
|
325
|
-
/// fn supports_language(&self, _: &str) -> bool { true }
|
|
326
|
-
/// fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
327
|
-
/// }
|
|
328
|
-
///
|
|
329
|
-
/// # tokio_test::block_on(async {
|
|
330
|
-
/// let backend = Arc::new(CustomOcr);
|
|
331
|
-
/// register_ocr_backend(backend)?;
|
|
332
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
333
|
-
/// # });
|
|
334
|
-
/// ```
|
|
335
|
-
pub fn register_ocr_backend(backend: Arc<dyn OcrBackend>) -> crate::Result<()> {
|
|
336
|
-
use crate::plugins::registry::get_ocr_backend_registry;
|
|
337
|
-
|
|
338
|
-
let registry = get_ocr_backend_registry();
|
|
339
|
-
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
340
|
-
let mut registry = registry
|
|
341
|
-
.write()
|
|
342
|
-
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
343
|
-
|
|
344
|
-
registry.register(backend)
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
/// Unregister an OCR backend by name.
|
|
348
|
-
///
|
|
349
|
-
/// Removes the OCR backend from the global registry and calls its `shutdown()` method.
|
|
350
|
-
///
|
|
351
|
-
/// # Arguments
|
|
352
|
-
///
|
|
353
|
-
/// * `name` - Name of the OCR backend to unregister
|
|
354
|
-
///
|
|
355
|
-
/// # Returns
|
|
356
|
-
///
|
|
357
|
-
/// - `Ok(())` if the backend was unregistered or didn't exist
|
|
358
|
-
/// - `Err(...)` if the shutdown method failed
|
|
359
|
-
///
|
|
360
|
-
/// # Example
|
|
361
|
-
///
|
|
362
|
-
/// ```rust
|
|
363
|
-
/// use kreuzberg::plugins::unregister_ocr_backend;
|
|
364
|
-
///
|
|
365
|
-
/// # tokio_test::block_on(async {
|
|
366
|
-
/// unregister_ocr_backend("custom-ocr")?;
|
|
367
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
368
|
-
/// # });
|
|
369
|
-
/// ```
|
|
370
|
-
pub fn unregister_ocr_backend(name: &str) -> crate::Result<()> {
|
|
371
|
-
use crate::plugins::registry::get_ocr_backend_registry;
|
|
372
|
-
|
|
373
|
-
let registry = get_ocr_backend_registry();
|
|
374
|
-
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
375
|
-
let mut registry = registry
|
|
376
|
-
.write()
|
|
377
|
-
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
378
|
-
|
|
379
|
-
registry.remove(name)
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
/// List all registered OCR backends.
|
|
383
|
-
///
|
|
384
|
-
/// Returns the names of all OCR backends currently registered in the global registry.
|
|
385
|
-
///
|
|
386
|
-
/// # Returns
|
|
387
|
-
///
|
|
388
|
-
/// A vector of OCR backend names.
|
|
389
|
-
///
|
|
390
|
-
/// # Example
|
|
391
|
-
///
|
|
392
|
-
/// ```rust
|
|
393
|
-
/// use kreuzberg::plugins::list_ocr_backends;
|
|
394
|
-
///
|
|
395
|
-
/// # tokio_test::block_on(async {
|
|
396
|
-
/// let backends = list_ocr_backends()?;
|
|
397
|
-
/// for name in backends {
|
|
398
|
-
/// println!("Registered OCR backend: {}", name);
|
|
399
|
-
/// }
|
|
400
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
401
|
-
/// # });
|
|
402
|
-
/// ```
|
|
403
|
-
pub fn list_ocr_backends() -> crate::Result<Vec<String>> {
|
|
404
|
-
use crate::plugins::registry::get_ocr_backend_registry;
|
|
405
|
-
|
|
406
|
-
let registry = get_ocr_backend_registry();
|
|
407
|
-
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
408
|
-
let registry = registry
|
|
409
|
-
.read()
|
|
410
|
-
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
411
|
-
|
|
412
|
-
Ok(registry.list())
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
/// Clear all OCR backends from the global registry.
|
|
416
|
-
///
|
|
417
|
-
/// Removes all OCR backends and calls their `shutdown()` methods.
|
|
418
|
-
///
|
|
419
|
-
/// # Returns
|
|
420
|
-
///
|
|
421
|
-
/// - `Ok(())` if all backends were cleared successfully
|
|
422
|
-
/// - `Err(...)` if any shutdown method failed
|
|
423
|
-
///
|
|
424
|
-
/// # Example
|
|
425
|
-
///
|
|
426
|
-
/// ```rust
|
|
427
|
-
/// use kreuzberg::plugins::clear_ocr_backends;
|
|
428
|
-
///
|
|
429
|
-
/// # tokio_test::block_on(async {
|
|
430
|
-
/// clear_ocr_backends()?;
|
|
431
|
-
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
432
|
-
/// # });
|
|
433
|
-
/// ```
|
|
434
|
-
pub fn clear_ocr_backends() -> crate::Result<()> {
|
|
435
|
-
use crate::plugins::registry::get_ocr_backend_registry;
|
|
436
|
-
|
|
437
|
-
let registry = get_ocr_backend_registry();
|
|
438
|
-
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
439
|
-
let mut registry = registry
|
|
440
|
-
.write()
|
|
441
|
-
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
442
|
-
|
|
443
|
-
registry.shutdown_all()
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
#[cfg(test)]
|
|
447
|
-
mod tests {
|
|
448
|
-
use super::*;
|
|
449
|
-
|
|
450
|
-
struct MockOcrBackend {
|
|
451
|
-
languages: Vec<String>,
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
impl Plugin for MockOcrBackend {
|
|
455
|
-
fn name(&self) -> &str {
|
|
456
|
-
"mock-ocr"
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
fn version(&self) -> String {
|
|
460
|
-
"1.0.0".to_string()
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
fn initialize(&self) -> Result<()> {
|
|
464
|
-
Ok(())
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
fn shutdown(&self) -> Result<()> {
|
|
468
|
-
Ok(())
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
#[async_trait]
|
|
473
|
-
impl OcrBackend for MockOcrBackend {
|
|
474
|
-
async fn process_image(&self, _image_bytes: &[u8], _config: &OcrConfig) -> Result<ExtractionResult> {
|
|
475
|
-
Ok(ExtractionResult {
|
|
476
|
-
content: "Mocked OCR text".to_string(),
|
|
477
|
-
mime_type: "text/plain".to_string(),
|
|
478
|
-
metadata: crate::types::Metadata::default(),
|
|
479
|
-
tables: vec![],
|
|
480
|
-
detected_languages: None,
|
|
481
|
-
chunks: None,
|
|
482
|
-
images: None,
|
|
483
|
-
pages: None,
|
|
484
|
-
})
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
fn supports_language(&self, lang: &str) -> bool {
|
|
488
|
-
self.languages.iter().any(|l| l == lang)
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
fn backend_type(&self) -> OcrBackendType {
|
|
492
|
-
OcrBackendType::Custom
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
fn supported_languages(&self) -> Vec<String> {
|
|
496
|
-
self.languages.clone()
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
#[tokio::test]
|
|
501
|
-
async fn test_ocr_backend_process_image() {
|
|
502
|
-
let backend = MockOcrBackend {
|
|
503
|
-
languages: vec!["eng".to_string(), "deu".to_string()],
|
|
504
|
-
};
|
|
505
|
-
|
|
506
|
-
let config = OcrConfig {
|
|
507
|
-
backend: "mock".to_string(),
|
|
508
|
-
language: "eng".to_string(),
|
|
509
|
-
tesseract_config: None,
|
|
510
|
-
};
|
|
511
|
-
|
|
512
|
-
let result = backend.process_image(b"fake image data", &config).await.unwrap();
|
|
513
|
-
assert_eq!(result.content, "Mocked OCR text");
|
|
514
|
-
assert_eq!(result.mime_type, "text/plain");
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
#[test]
|
|
518
|
-
fn test_ocr_backend_supports_language() {
|
|
519
|
-
let backend = MockOcrBackend {
|
|
520
|
-
languages: vec!["eng".to_string(), "deu".to_string()],
|
|
521
|
-
};
|
|
522
|
-
|
|
523
|
-
assert!(backend.supports_language("eng"));
|
|
524
|
-
assert!(backend.supports_language("deu"));
|
|
525
|
-
assert!(!backend.supports_language("fra"));
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
#[test]
|
|
529
|
-
fn test_ocr_backend_type() {
|
|
530
|
-
let backend = MockOcrBackend {
|
|
531
|
-
languages: vec!["eng".to_string()],
|
|
532
|
-
};
|
|
533
|
-
|
|
534
|
-
assert_eq!(backend.backend_type(), OcrBackendType::Custom);
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
#[test]
|
|
538
|
-
fn test_ocr_backend_supported_languages() {
|
|
539
|
-
let backend = MockOcrBackend {
|
|
540
|
-
languages: vec!["eng".to_string(), "deu".to_string(), "fra".to_string()],
|
|
541
|
-
};
|
|
542
|
-
|
|
543
|
-
let supported = backend.supported_languages();
|
|
544
|
-
assert_eq!(supported.len(), 3);
|
|
545
|
-
assert!(supported.contains(&"eng".to_string()));
|
|
546
|
-
assert!(supported.contains(&"deu".to_string()));
|
|
547
|
-
assert!(supported.contains(&"fra".to_string()));
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
#[test]
|
|
551
|
-
fn test_ocr_backend_type_variants() {
|
|
552
|
-
assert_eq!(OcrBackendType::Tesseract, OcrBackendType::Tesseract);
|
|
553
|
-
assert_ne!(OcrBackendType::Tesseract, OcrBackendType::EasyOCR);
|
|
554
|
-
assert_ne!(OcrBackendType::EasyOCR, OcrBackendType::PaddleOCR);
|
|
555
|
-
assert_ne!(OcrBackendType::PaddleOCR, OcrBackendType::Custom);
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
#[test]
|
|
559
|
-
fn test_ocr_backend_type_debug() {
|
|
560
|
-
let backend_type = OcrBackendType::Tesseract;
|
|
561
|
-
let debug_str = format!("{:?}", backend_type);
|
|
562
|
-
assert!(debug_str.contains("Tesseract"));
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
#[test]
|
|
566
|
-
fn test_ocr_backend_type_clone() {
|
|
567
|
-
let backend_type = OcrBackendType::EasyOCR;
|
|
568
|
-
let cloned = backend_type;
|
|
569
|
-
assert_eq!(backend_type, cloned);
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
#[test]
|
|
573
|
-
fn test_ocr_backend_default_table_detection() {
|
|
574
|
-
let backend = MockOcrBackend {
|
|
575
|
-
languages: vec!["eng".to_string()],
|
|
576
|
-
};
|
|
577
|
-
assert!(!backend.supports_table_detection());
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
#[tokio::test]
|
|
581
|
-
async fn test_ocr_backend_process_file_default_impl() {
|
|
582
|
-
use std::io::Write;
|
|
583
|
-
use tempfile::NamedTempFile;
|
|
584
|
-
|
|
585
|
-
let backend = MockOcrBackend {
|
|
586
|
-
languages: vec!["eng".to_string()],
|
|
587
|
-
};
|
|
588
|
-
|
|
589
|
-
let mut temp_file = NamedTempFile::new().unwrap();
|
|
590
|
-
temp_file.write_all(b"fake image data").unwrap();
|
|
591
|
-
let path = temp_file.path();
|
|
592
|
-
|
|
593
|
-
let config = OcrConfig {
|
|
594
|
-
backend: "mock".to_string(),
|
|
595
|
-
language: "eng".to_string(),
|
|
596
|
-
tesseract_config: None,
|
|
597
|
-
};
|
|
598
|
-
|
|
599
|
-
let result = backend.process_file(path, &config).await.unwrap();
|
|
600
|
-
assert_eq!(result.content, "Mocked OCR text");
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
#[test]
|
|
604
|
-
fn test_ocr_backend_plugin_interface() {
|
|
605
|
-
let backend = MockOcrBackend {
|
|
606
|
-
languages: vec!["eng".to_string()],
|
|
607
|
-
};
|
|
608
|
-
|
|
609
|
-
assert_eq!(backend.name(), "mock-ocr");
|
|
610
|
-
assert_eq!(backend.version(), "1.0.0");
|
|
611
|
-
assert!(backend.initialize().is_ok());
|
|
612
|
-
assert!(backend.shutdown().is_ok());
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
#[test]
|
|
616
|
-
fn test_ocr_backend_empty_languages() {
|
|
617
|
-
let backend = MockOcrBackend { languages: vec![] };
|
|
618
|
-
|
|
619
|
-
let supported = backend.supported_languages();
|
|
620
|
-
assert_eq!(supported.len(), 0);
|
|
621
|
-
assert!(!backend.supports_language("eng"));
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
#[tokio::test]
|
|
625
|
-
async fn test_ocr_backend_with_empty_image() {
|
|
626
|
-
let backend = MockOcrBackend {
|
|
627
|
-
languages: vec!["eng".to_string()],
|
|
628
|
-
};
|
|
629
|
-
|
|
630
|
-
let config = OcrConfig {
|
|
631
|
-
backend: "mock".to_string(),
|
|
632
|
-
language: "eng".to_string(),
|
|
633
|
-
tesseract_config: None,
|
|
634
|
-
};
|
|
635
|
-
|
|
636
|
-
let result = backend.process_image(b"", &config).await;
|
|
637
|
-
assert!(result.is_ok());
|
|
638
|
-
}
|
|
639
|
-
}
|
|
1
|
+
//! OCR backend plugin trait.
|
|
2
|
+
//!
|
|
3
|
+
//! This module defines the trait for implementing custom OCR backends.
|
|
4
|
+
|
|
5
|
+
use crate::Result;
|
|
6
|
+
use crate::core::config::OcrConfig;
|
|
7
|
+
use crate::plugins::Plugin;
|
|
8
|
+
use crate::types::ExtractionResult;
|
|
9
|
+
use async_trait::async_trait;
|
|
10
|
+
use std::path::Path;
|
|
11
|
+
use std::sync::Arc;
|
|
12
|
+
|
|
13
|
+
#[cfg(not(feature = "tokio-runtime"))]
|
|
14
|
+
use crate::KreuzbergError;
|
|
15
|
+
|
|
16
|
+
/// OCR backend types.
|
|
17
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
18
|
+
pub enum OcrBackendType {
|
|
19
|
+
/// Tesseract OCR (native Rust binding)
|
|
20
|
+
Tesseract,
|
|
21
|
+
/// EasyOCR (Python-based, via FFI)
|
|
22
|
+
EasyOCR,
|
|
23
|
+
/// PaddleOCR (Python-based, via FFI)
|
|
24
|
+
PaddleOCR,
|
|
25
|
+
/// Custom/third-party OCR backend
|
|
26
|
+
Custom,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/// Trait for OCR backend plugins.
|
|
30
|
+
///
|
|
31
|
+
/// Implement this trait to add custom OCR capabilities. OCR backends can be:
|
|
32
|
+
/// - Native Rust implementations (like Tesseract)
|
|
33
|
+
/// - FFI bridges to Python libraries (like EasyOCR, PaddleOCR)
|
|
34
|
+
/// - Cloud-based OCR services (Google Vision, AWS Textract, etc.)
|
|
35
|
+
///
|
|
36
|
+
/// # Thread Safety
|
|
37
|
+
///
|
|
38
|
+
/// OCR backends must be thread-safe (`Send + Sync`) to support concurrent processing.
|
|
39
|
+
///
|
|
40
|
+
/// # Example
|
|
41
|
+
///
|
|
42
|
+
/// ```rust
|
|
43
|
+
/// use kreuzberg::plugins::{Plugin, OcrBackend, OcrBackendType};
|
|
44
|
+
/// use kreuzberg::{Result, OcrConfig};
|
|
45
|
+
/// use async_trait::async_trait;
|
|
46
|
+
/// use std::path::Path;
|
|
47
|
+
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
48
|
+
///
|
|
49
|
+
/// struct CustomOcrBackend;
|
|
50
|
+
///
|
|
51
|
+
/// impl Plugin for CustomOcrBackend {
|
|
52
|
+
/// fn name(&self) -> &str { "custom-ocr" }
|
|
53
|
+
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
54
|
+
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
55
|
+
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
56
|
+
/// }
|
|
57
|
+
///
|
|
58
|
+
/// #[async_trait]
|
|
59
|
+
/// impl OcrBackend for CustomOcrBackend {
|
|
60
|
+
/// async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult> {
|
|
61
|
+
/// // Implement OCR logic here
|
|
62
|
+
/// Ok(ExtractionResult {
|
|
63
|
+
/// content: "Extracted text".to_string(),
|
|
64
|
+
/// mime_type: "text/plain".to_string(),
|
|
65
|
+
/// metadata: Metadata::default(),
|
|
66
|
+
/// tables: vec![],
|
|
67
|
+
/// detected_languages: None,
|
|
68
|
+
/// chunks: None,
|
|
69
|
+
/// images: None,
|
|
70
|
+
/// pages: None,
|
|
71
|
+
/// })
|
|
72
|
+
/// }
|
|
73
|
+
///
|
|
74
|
+
/// async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
|
|
75
|
+
/// let bytes = std::fs::read(path)?;
|
|
76
|
+
/// self.process_image(&bytes, config).await
|
|
77
|
+
/// }
|
|
78
|
+
///
|
|
79
|
+
/// fn supports_language(&self, lang: &str) -> bool {
|
|
80
|
+
/// matches!(lang, "eng" | "deu" | "fra")
|
|
81
|
+
/// }
|
|
82
|
+
///
|
|
83
|
+
/// fn backend_type(&self) -> OcrBackendType {
|
|
84
|
+
/// OcrBackendType::Custom
|
|
85
|
+
/// }
|
|
86
|
+
/// }
|
|
87
|
+
/// ```
|
|
88
|
+
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
|
|
89
|
+
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
|
|
90
|
+
pub trait OcrBackend: Plugin {
|
|
91
|
+
/// Process an image and extract text via OCR.
|
|
92
|
+
///
|
|
93
|
+
/// # Arguments
|
|
94
|
+
///
|
|
95
|
+
/// * `image_bytes` - Raw image data (JPEG, PNG, TIFF, etc.)
|
|
96
|
+
/// * `config` - OCR configuration (language, PSM mode, etc.)
|
|
97
|
+
///
|
|
98
|
+
/// # Returns
|
|
99
|
+
///
|
|
100
|
+
/// An `ExtractionResult` containing the extracted text and metadata.
|
|
101
|
+
///
|
|
102
|
+
/// # Errors
|
|
103
|
+
///
|
|
104
|
+
/// - `KreuzbergError::Ocr` - OCR processing failed
|
|
105
|
+
/// - `KreuzbergError::Validation` - Invalid image format or configuration
|
|
106
|
+
/// - `KreuzbergError::Io` - I/O errors (these always bubble up)
|
|
107
|
+
///
|
|
108
|
+
/// # Example
|
|
109
|
+
///
|
|
110
|
+
/// ```rust
|
|
111
|
+
/// # use kreuzberg::plugins::{Plugin, OcrBackend};
|
|
112
|
+
/// # use kreuzberg::{Result, OcrConfig};
|
|
113
|
+
/// # use async_trait::async_trait;
|
|
114
|
+
/// # use std::path::Path;
|
|
115
|
+
/// # use kreuzberg::types::{ExtractionResult, Metadata};
|
|
116
|
+
/// # struct MyOcr;
|
|
117
|
+
/// # impl Plugin for MyOcr {
|
|
118
|
+
/// # fn name(&self) -> &str { "my-ocr" }
|
|
119
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
120
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
121
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
122
|
+
/// # }
|
|
123
|
+
/// # use kreuzberg::plugins::OcrBackendType;
|
|
124
|
+
/// # #[async_trait]
|
|
125
|
+
/// # impl OcrBackend for MyOcr {
|
|
126
|
+
/// # fn supports_language(&self, _: &str) -> bool { true }
|
|
127
|
+
/// # fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
128
|
+
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
129
|
+
/// async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult> {
|
|
130
|
+
/// // Validate image format
|
|
131
|
+
/// if image_bytes.is_empty() {
|
|
132
|
+
/// return Err(kreuzberg::KreuzbergError::Validation {
|
|
133
|
+
/// message: "Empty image data".to_string(),
|
|
134
|
+
/// source: None,
|
|
135
|
+
/// });
|
|
136
|
+
/// }
|
|
137
|
+
///
|
|
138
|
+
/// // Perform OCR processing
|
|
139
|
+
/// let text = format!("Extracted text in language: {}", config.language);
|
|
140
|
+
///
|
|
141
|
+
/// Ok(ExtractionResult {
|
|
142
|
+
/// content: text,
|
|
143
|
+
/// mime_type: "text/plain".to_string(),
|
|
144
|
+
/// metadata: Metadata::default(),
|
|
145
|
+
/// tables: vec![],
|
|
146
|
+
/// detected_languages: None,
|
|
147
|
+
/// chunks: None,
|
|
148
|
+
/// images: None,
|
|
149
|
+
/// pages: None,
|
|
150
|
+
/// })
|
|
151
|
+
/// }
|
|
152
|
+
/// # }
|
|
153
|
+
/// ```
|
|
154
|
+
async fn process_image(&self, image_bytes: &[u8], config: &OcrConfig) -> Result<ExtractionResult>;
|
|
155
|
+
|
|
156
|
+
/// Process a file and extract text via OCR.
|
|
157
|
+
///
|
|
158
|
+
/// Default implementation reads the file and calls `process_image`.
|
|
159
|
+
/// Override for custom file handling or optimizations.
|
|
160
|
+
///
|
|
161
|
+
/// # Arguments
|
|
162
|
+
///
|
|
163
|
+
/// * `path` - Path to the image file
|
|
164
|
+
/// * `config` - OCR configuration
|
|
165
|
+
///
|
|
166
|
+
/// # Errors
|
|
167
|
+
///
|
|
168
|
+
/// Same as `process_image`, plus file I/O errors.
|
|
169
|
+
async fn process_file(&self, path: &Path, config: &OcrConfig) -> Result<ExtractionResult> {
|
|
170
|
+
#[cfg(feature = "tokio-runtime")]
|
|
171
|
+
{
|
|
172
|
+
use crate::core::io;
|
|
173
|
+
let bytes = io::read_file_async(path).await?;
|
|
174
|
+
self.process_image(&bytes, config).await
|
|
175
|
+
}
|
|
176
|
+
#[cfg(not(feature = "tokio-runtime"))]
|
|
177
|
+
{
|
|
178
|
+
let _ = (path, config);
|
|
179
|
+
// For WASM and non-tokio environments, file-based OCR is not supported
|
|
180
|
+
// through the default implementation. Implementations must provide their own.
|
|
181
|
+
Err(KreuzbergError::Other(
|
|
182
|
+
"File-based OCR processing requires the tokio-runtime feature".to_string(),
|
|
183
|
+
))
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/// Check if this backend supports a given language code.
|
|
188
|
+
///
|
|
189
|
+
/// # Arguments
|
|
190
|
+
///
|
|
191
|
+
/// * `lang` - ISO 639-2/3 language code (e.g., "eng", "deu", "fra")
|
|
192
|
+
///
|
|
193
|
+
/// # Returns
|
|
194
|
+
///
|
|
195
|
+
/// `true` if the language is supported, `false` otherwise.
|
|
196
|
+
///
|
|
197
|
+
/// # Example
|
|
198
|
+
///
|
|
199
|
+
/// ```rust
|
|
200
|
+
/// # use kreuzberg::plugins::{Plugin, OcrBackend};
|
|
201
|
+
/// # use kreuzberg::Result;
|
|
202
|
+
/// # use async_trait::async_trait;
|
|
203
|
+
/// # use std::path::Path;
|
|
204
|
+
/// # struct MyOcr { languages: Vec<String> }
|
|
205
|
+
/// # impl Plugin for MyOcr {
|
|
206
|
+
/// # fn name(&self) -> &str { "my-ocr" }
|
|
207
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
208
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
209
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
210
|
+
/// # }
|
|
211
|
+
/// # use kreuzberg::plugins::OcrBackendType;
|
|
212
|
+
/// # use kreuzberg::{ExtractionResult, OcrConfig};
|
|
213
|
+
/// # #[async_trait]
|
|
214
|
+
/// # impl OcrBackend for MyOcr {
|
|
215
|
+
/// # fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
216
|
+
/// # async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
217
|
+
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
218
|
+
/// fn supports_language(&self, lang: &str) -> bool {
|
|
219
|
+
/// self.languages.contains(&lang.to_string())
|
|
220
|
+
/// }
|
|
221
|
+
/// # }
|
|
222
|
+
/// ```
|
|
223
|
+
fn supports_language(&self, lang: &str) -> bool;
|
|
224
|
+
|
|
225
|
+
/// Get the backend type identifier.
|
|
226
|
+
///
|
|
227
|
+
/// # Returns
|
|
228
|
+
///
|
|
229
|
+
/// The backend type enum value.
|
|
230
|
+
///
|
|
231
|
+
/// # Example
|
|
232
|
+
///
|
|
233
|
+
/// ```rust
|
|
234
|
+
/// # use kreuzberg::plugins::{Plugin, OcrBackend, OcrBackendType};
|
|
235
|
+
/// # use kreuzberg::Result;
|
|
236
|
+
/// # use async_trait::async_trait;
|
|
237
|
+
/// # use std::path::Path;
|
|
238
|
+
/// # struct TesseractBackend;
|
|
239
|
+
/// # impl Plugin for TesseractBackend {
|
|
240
|
+
/// # fn name(&self) -> &str { "tesseract" }
|
|
241
|
+
/// # fn version(&self) -> String { "1.0.0".to_string() }
|
|
242
|
+
/// # fn initialize(&self) -> Result<()> { Ok(()) }
|
|
243
|
+
/// # fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
244
|
+
/// # }
|
|
245
|
+
/// # use kreuzberg::{ExtractionResult, OcrConfig};
|
|
246
|
+
/// # #[async_trait]
|
|
247
|
+
/// # impl OcrBackend for TesseractBackend {
|
|
248
|
+
/// # fn supports_language(&self, _: &str) -> bool { true }
|
|
249
|
+
/// # async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
250
|
+
/// # async fn process_file(&self, _: &Path, _: &OcrConfig) -> Result<ExtractionResult> { todo!() }
|
|
251
|
+
/// fn backend_type(&self) -> OcrBackendType {
|
|
252
|
+
/// OcrBackendType::Tesseract
|
|
253
|
+
/// }
|
|
254
|
+
/// # }
|
|
255
|
+
/// ```
|
|
256
|
+
fn backend_type(&self) -> OcrBackendType;
|
|
257
|
+
|
|
258
|
+
/// Optional: Get a list of all supported languages.
|
|
259
|
+
///
|
|
260
|
+
/// Defaults to empty list. Override to provide comprehensive language support info.
|
|
261
|
+
fn supported_languages(&self) -> Vec<String> {
|
|
262
|
+
vec![]
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/// Optional: Check if the backend supports table detection.
|
|
266
|
+
///
|
|
267
|
+
/// Defaults to `false`. Override if your backend can detect and extract tables.
|
|
268
|
+
fn supports_table_detection(&self) -> bool {
|
|
269
|
+
false
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/// Register an OCR backend with the global registry.
|
|
274
|
+
///
|
|
275
|
+
/// The OCR backend will be registered with its name from the `name()` method
|
|
276
|
+
/// and can be used for OCR processing via the extraction pipeline.
|
|
277
|
+
///
|
|
278
|
+
/// # Arguments
|
|
279
|
+
///
|
|
280
|
+
/// * `backend` - The OCR backend implementation wrapped in Arc
|
|
281
|
+
///
|
|
282
|
+
/// # Returns
|
|
283
|
+
///
|
|
284
|
+
/// - `Ok(())` if registration succeeded
|
|
285
|
+
/// - `Err(...)` if validation failed or initialization failed
|
|
286
|
+
///
|
|
287
|
+
/// # Errors
|
|
288
|
+
///
|
|
289
|
+
/// - `KreuzbergError::Validation` - Invalid backend name (empty or contains whitespace)
|
|
290
|
+
/// - Any error from the backend's `initialize()` method
|
|
291
|
+
///
|
|
292
|
+
/// # Example
|
|
293
|
+
///
|
|
294
|
+
/// ```rust
|
|
295
|
+
/// use kreuzberg::plugins::{Plugin, OcrBackend, register_ocr_backend, OcrBackendType};
|
|
296
|
+
/// use kreuzberg::{Result, OcrConfig};
|
|
297
|
+
/// use kreuzberg::types::{ExtractionResult, Metadata};
|
|
298
|
+
/// use async_trait::async_trait;
|
|
299
|
+
/// use std::sync::Arc;
|
|
300
|
+
/// use std::path::Path;
|
|
301
|
+
///
|
|
302
|
+
/// struct CustomOcr;
|
|
303
|
+
///
|
|
304
|
+
/// impl Plugin for CustomOcr {
|
|
305
|
+
/// fn name(&self) -> &str { "custom-ocr" }
|
|
306
|
+
/// fn version(&self) -> String { "1.0.0".to_string() }
|
|
307
|
+
/// fn initialize(&self) -> Result<()> { Ok(()) }
|
|
308
|
+
/// fn shutdown(&self) -> Result<()> { Ok(()) }
|
|
309
|
+
/// }
|
|
310
|
+
///
|
|
311
|
+
/// #[async_trait]
|
|
312
|
+
/// impl OcrBackend for CustomOcr {
|
|
313
|
+
/// async fn process_image(&self, _: &[u8], _: &OcrConfig) -> Result<ExtractionResult> {
|
|
314
|
+
/// Ok(ExtractionResult {
|
|
315
|
+
/// content: "text".to_string(),
|
|
316
|
+
/// mime_type: "text/plain".to_string(),
|
|
317
|
+
/// metadata: Metadata::default(),
|
|
318
|
+
/// tables: vec![],
|
|
319
|
+
/// detected_languages: None,
|
|
320
|
+
/// chunks: None,
|
|
321
|
+
/// images: None,
|
|
322
|
+
/// pages: None,
|
|
323
|
+
/// })
|
|
324
|
+
/// }
|
|
325
|
+
/// fn supports_language(&self, _: &str) -> bool { true }
|
|
326
|
+
/// fn backend_type(&self) -> OcrBackendType { OcrBackendType::Custom }
|
|
327
|
+
/// }
|
|
328
|
+
///
|
|
329
|
+
/// # tokio_test::block_on(async {
|
|
330
|
+
/// let backend = Arc::new(CustomOcr);
|
|
331
|
+
/// register_ocr_backend(backend)?;
|
|
332
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
333
|
+
/// # });
|
|
334
|
+
/// ```
|
|
335
|
+
pub fn register_ocr_backend(backend: Arc<dyn OcrBackend>) -> crate::Result<()> {
|
|
336
|
+
use crate::plugins::registry::get_ocr_backend_registry;
|
|
337
|
+
|
|
338
|
+
let registry = get_ocr_backend_registry();
|
|
339
|
+
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
340
|
+
let mut registry = registry
|
|
341
|
+
.write()
|
|
342
|
+
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
343
|
+
|
|
344
|
+
registry.register(backend)
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/// Unregister an OCR backend by name.
|
|
348
|
+
///
|
|
349
|
+
/// Removes the OCR backend from the global registry and calls its `shutdown()` method.
|
|
350
|
+
///
|
|
351
|
+
/// # Arguments
|
|
352
|
+
///
|
|
353
|
+
/// * `name` - Name of the OCR backend to unregister
|
|
354
|
+
///
|
|
355
|
+
/// # Returns
|
|
356
|
+
///
|
|
357
|
+
/// - `Ok(())` if the backend was unregistered or didn't exist
|
|
358
|
+
/// - `Err(...)` if the shutdown method failed
|
|
359
|
+
///
|
|
360
|
+
/// # Example
|
|
361
|
+
///
|
|
362
|
+
/// ```rust
|
|
363
|
+
/// use kreuzberg::plugins::unregister_ocr_backend;
|
|
364
|
+
///
|
|
365
|
+
/// # tokio_test::block_on(async {
|
|
366
|
+
/// unregister_ocr_backend("custom-ocr")?;
|
|
367
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
368
|
+
/// # });
|
|
369
|
+
/// ```
|
|
370
|
+
pub fn unregister_ocr_backend(name: &str) -> crate::Result<()> {
|
|
371
|
+
use crate::plugins::registry::get_ocr_backend_registry;
|
|
372
|
+
|
|
373
|
+
let registry = get_ocr_backend_registry();
|
|
374
|
+
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
375
|
+
let mut registry = registry
|
|
376
|
+
.write()
|
|
377
|
+
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
378
|
+
|
|
379
|
+
registry.remove(name)
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/// List all registered OCR backends.
|
|
383
|
+
///
|
|
384
|
+
/// Returns the names of all OCR backends currently registered in the global registry.
|
|
385
|
+
///
|
|
386
|
+
/// # Returns
|
|
387
|
+
///
|
|
388
|
+
/// A vector of OCR backend names.
|
|
389
|
+
///
|
|
390
|
+
/// # Example
|
|
391
|
+
///
|
|
392
|
+
/// ```rust
|
|
393
|
+
/// use kreuzberg::plugins::list_ocr_backends;
|
|
394
|
+
///
|
|
395
|
+
/// # tokio_test::block_on(async {
|
|
396
|
+
/// let backends = list_ocr_backends()?;
|
|
397
|
+
/// for name in backends {
|
|
398
|
+
/// println!("Registered OCR backend: {}", name);
|
|
399
|
+
/// }
|
|
400
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
401
|
+
/// # });
|
|
402
|
+
/// ```
|
|
403
|
+
pub fn list_ocr_backends() -> crate::Result<Vec<String>> {
|
|
404
|
+
use crate::plugins::registry::get_ocr_backend_registry;
|
|
405
|
+
|
|
406
|
+
let registry = get_ocr_backend_registry();
|
|
407
|
+
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
408
|
+
let registry = registry
|
|
409
|
+
.read()
|
|
410
|
+
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
411
|
+
|
|
412
|
+
Ok(registry.list())
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/// Clear all OCR backends from the global registry.
|
|
416
|
+
///
|
|
417
|
+
/// Removes all OCR backends and calls their `shutdown()` methods.
|
|
418
|
+
///
|
|
419
|
+
/// # Returns
|
|
420
|
+
///
|
|
421
|
+
/// - `Ok(())` if all backends were cleared successfully
|
|
422
|
+
/// - `Err(...)` if any shutdown method failed
|
|
423
|
+
///
|
|
424
|
+
/// # Example
|
|
425
|
+
///
|
|
426
|
+
/// ```rust
|
|
427
|
+
/// use kreuzberg::plugins::clear_ocr_backends;
|
|
428
|
+
///
|
|
429
|
+
/// # tokio_test::block_on(async {
|
|
430
|
+
/// clear_ocr_backends()?;
|
|
431
|
+
/// # Ok::<(), kreuzberg::KreuzbergError>(())
|
|
432
|
+
/// # });
|
|
433
|
+
/// ```
|
|
434
|
+
pub fn clear_ocr_backends() -> crate::Result<()> {
|
|
435
|
+
use crate::plugins::registry::get_ocr_backend_registry;
|
|
436
|
+
|
|
437
|
+
let registry = get_ocr_backend_registry();
|
|
438
|
+
// ~keep: Lock poisoning indicates a panic in another thread holding the lock.
|
|
439
|
+
let mut registry = registry
|
|
440
|
+
.write()
|
|
441
|
+
.expect("OCR backend registry lock poisoned - critical runtime error");
|
|
442
|
+
|
|
443
|
+
registry.shutdown_all()
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
#[cfg(test)]
|
|
447
|
+
mod tests {
|
|
448
|
+
use super::*;
|
|
449
|
+
|
|
450
|
+
struct MockOcrBackend {
|
|
451
|
+
languages: Vec<String>,
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
impl Plugin for MockOcrBackend {
|
|
455
|
+
fn name(&self) -> &str {
|
|
456
|
+
"mock-ocr"
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
fn version(&self) -> String {
|
|
460
|
+
"1.0.0".to_string()
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
fn initialize(&self) -> Result<()> {
|
|
464
|
+
Ok(())
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
fn shutdown(&self) -> Result<()> {
|
|
468
|
+
Ok(())
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
#[async_trait]
|
|
473
|
+
impl OcrBackend for MockOcrBackend {
|
|
474
|
+
async fn process_image(&self, _image_bytes: &[u8], _config: &OcrConfig) -> Result<ExtractionResult> {
|
|
475
|
+
Ok(ExtractionResult {
|
|
476
|
+
content: "Mocked OCR text".to_string(),
|
|
477
|
+
mime_type: "text/plain".to_string(),
|
|
478
|
+
metadata: crate::types::Metadata::default(),
|
|
479
|
+
tables: vec![],
|
|
480
|
+
detected_languages: None,
|
|
481
|
+
chunks: None,
|
|
482
|
+
images: None,
|
|
483
|
+
pages: None,
|
|
484
|
+
})
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
fn supports_language(&self, lang: &str) -> bool {
|
|
488
|
+
self.languages.iter().any(|l| l == lang)
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
fn backend_type(&self) -> OcrBackendType {
|
|
492
|
+
OcrBackendType::Custom
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
fn supported_languages(&self) -> Vec<String> {
|
|
496
|
+
self.languages.clone()
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
#[tokio::test]
|
|
501
|
+
async fn test_ocr_backend_process_image() {
|
|
502
|
+
let backend = MockOcrBackend {
|
|
503
|
+
languages: vec!["eng".to_string(), "deu".to_string()],
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
let config = OcrConfig {
|
|
507
|
+
backend: "mock".to_string(),
|
|
508
|
+
language: "eng".to_string(),
|
|
509
|
+
tesseract_config: None,
|
|
510
|
+
};
|
|
511
|
+
|
|
512
|
+
let result = backend.process_image(b"fake image data", &config).await.unwrap();
|
|
513
|
+
assert_eq!(result.content, "Mocked OCR text");
|
|
514
|
+
assert_eq!(result.mime_type, "text/plain");
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
#[test]
|
|
518
|
+
fn test_ocr_backend_supports_language() {
|
|
519
|
+
let backend = MockOcrBackend {
|
|
520
|
+
languages: vec!["eng".to_string(), "deu".to_string()],
|
|
521
|
+
};
|
|
522
|
+
|
|
523
|
+
assert!(backend.supports_language("eng"));
|
|
524
|
+
assert!(backend.supports_language("deu"));
|
|
525
|
+
assert!(!backend.supports_language("fra"));
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
#[test]
|
|
529
|
+
fn test_ocr_backend_type() {
|
|
530
|
+
let backend = MockOcrBackend {
|
|
531
|
+
languages: vec!["eng".to_string()],
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
assert_eq!(backend.backend_type(), OcrBackendType::Custom);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
#[test]
|
|
538
|
+
fn test_ocr_backend_supported_languages() {
|
|
539
|
+
let backend = MockOcrBackend {
|
|
540
|
+
languages: vec!["eng".to_string(), "deu".to_string(), "fra".to_string()],
|
|
541
|
+
};
|
|
542
|
+
|
|
543
|
+
let supported = backend.supported_languages();
|
|
544
|
+
assert_eq!(supported.len(), 3);
|
|
545
|
+
assert!(supported.contains(&"eng".to_string()));
|
|
546
|
+
assert!(supported.contains(&"deu".to_string()));
|
|
547
|
+
assert!(supported.contains(&"fra".to_string()));
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
#[test]
|
|
551
|
+
fn test_ocr_backend_type_variants() {
|
|
552
|
+
assert_eq!(OcrBackendType::Tesseract, OcrBackendType::Tesseract);
|
|
553
|
+
assert_ne!(OcrBackendType::Tesseract, OcrBackendType::EasyOCR);
|
|
554
|
+
assert_ne!(OcrBackendType::EasyOCR, OcrBackendType::PaddleOCR);
|
|
555
|
+
assert_ne!(OcrBackendType::PaddleOCR, OcrBackendType::Custom);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
#[test]
|
|
559
|
+
fn test_ocr_backend_type_debug() {
|
|
560
|
+
let backend_type = OcrBackendType::Tesseract;
|
|
561
|
+
let debug_str = format!("{:?}", backend_type);
|
|
562
|
+
assert!(debug_str.contains("Tesseract"));
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
#[test]
|
|
566
|
+
fn test_ocr_backend_type_clone() {
|
|
567
|
+
let backend_type = OcrBackendType::EasyOCR;
|
|
568
|
+
let cloned = backend_type;
|
|
569
|
+
assert_eq!(backend_type, cloned);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
#[test]
|
|
573
|
+
fn test_ocr_backend_default_table_detection() {
|
|
574
|
+
let backend = MockOcrBackend {
|
|
575
|
+
languages: vec!["eng".to_string()],
|
|
576
|
+
};
|
|
577
|
+
assert!(!backend.supports_table_detection());
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
#[tokio::test]
|
|
581
|
+
async fn test_ocr_backend_process_file_default_impl() {
|
|
582
|
+
use std::io::Write;
|
|
583
|
+
use tempfile::NamedTempFile;
|
|
584
|
+
|
|
585
|
+
let backend = MockOcrBackend {
|
|
586
|
+
languages: vec!["eng".to_string()],
|
|
587
|
+
};
|
|
588
|
+
|
|
589
|
+
let mut temp_file = NamedTempFile::new().unwrap();
|
|
590
|
+
temp_file.write_all(b"fake image data").unwrap();
|
|
591
|
+
let path = temp_file.path();
|
|
592
|
+
|
|
593
|
+
let config = OcrConfig {
|
|
594
|
+
backend: "mock".to_string(),
|
|
595
|
+
language: "eng".to_string(),
|
|
596
|
+
tesseract_config: None,
|
|
597
|
+
};
|
|
598
|
+
|
|
599
|
+
let result = backend.process_file(path, &config).await.unwrap();
|
|
600
|
+
assert_eq!(result.content, "Mocked OCR text");
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
#[test]
|
|
604
|
+
fn test_ocr_backend_plugin_interface() {
|
|
605
|
+
let backend = MockOcrBackend {
|
|
606
|
+
languages: vec!["eng".to_string()],
|
|
607
|
+
};
|
|
608
|
+
|
|
609
|
+
assert_eq!(backend.name(), "mock-ocr");
|
|
610
|
+
assert_eq!(backend.version(), "1.0.0");
|
|
611
|
+
assert!(backend.initialize().is_ok());
|
|
612
|
+
assert!(backend.shutdown().is_ok());
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
#[test]
|
|
616
|
+
fn test_ocr_backend_empty_languages() {
|
|
617
|
+
let backend = MockOcrBackend { languages: vec![] };
|
|
618
|
+
|
|
619
|
+
let supported = backend.supported_languages();
|
|
620
|
+
assert_eq!(supported.len(), 0);
|
|
621
|
+
assert!(!backend.supports_language("eng"));
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
#[tokio::test]
|
|
625
|
+
async fn test_ocr_backend_with_empty_image() {
|
|
626
|
+
let backend = MockOcrBackend {
|
|
627
|
+
languages: vec!["eng".to_string()],
|
|
628
|
+
};
|
|
629
|
+
|
|
630
|
+
let config = OcrConfig {
|
|
631
|
+
backend: "mock".to_string(),
|
|
632
|
+
language: "eng".to_string(),
|
|
633
|
+
tesseract_config: None,
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
let result = backend.process_image(b"", &config).await;
|
|
637
|
+
assert!(result.is_ok());
|
|
638
|
+
}
|
|
639
|
+
}
|