kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +105 -2
- data/README.md +454 -454
- data/Rakefile +33 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6940 -6941
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3158
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -214
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -81
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -80
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -340
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -109
- data/lib/{pdfium.dll → libpdfium.dylib} +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -546
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +1 -1
- data/vendor/kreuzberg/Cargo.toml +5 -5
- data/vendor/kreuzberg/README.md +230 -230
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -843
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1184
- data/vendor/kreuzberg/src/embeddings.rs +500 -500
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -601
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -574
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -749
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -44
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -346
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -68
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -420
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -164
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +1 -1
- data/vendor/kreuzberg-ffi/README.md +851 -851
- data/vendor/kreuzberg-ffi/build.rs +176 -176
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -27
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -1087
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -3616
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -247
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -470
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
- data/vendor/kreuzberg-tesseract/LICENSE +22 -22
- data/vendor/kreuzberg-tesseract/README.md +399 -399
- data/vendor/kreuzberg-tesseract/build.rs +1354 -1354
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -211
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +73 -4
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +0 -12
|
@@ -1,374 +1,374 @@
|
|
|
1
|
-
//! PDFium linking integration tests.
|
|
2
|
-
//!
|
|
3
|
-
//! This module tests PDF extraction functionality across all PDFium linking strategies
|
|
4
|
-
//! (default, static, bundled, system). Tests verify that:
|
|
5
|
-
//!
|
|
6
|
-
//! 1. PDF extraction works regardless of linking strategy
|
|
7
|
-
//! 2. Extracted content is valid and non-empty
|
|
8
|
-
//! 3. Bundled PDFium extraction works when feature is enabled
|
|
9
|
-
//! 4. Metadata extraction functions correctly
|
|
10
|
-
//! 5. Error handling works for invalid PDFs
|
|
11
|
-
//!
|
|
12
|
-
//! These tests are strategy-agnostic - they verify functionality works with ANY
|
|
13
|
-
//! linking strategy, not implementation details of specific strategies.
|
|
14
|
-
//!
|
|
15
|
-
//! Test philosophy:
|
|
16
|
-
//! - Use small test PDFs from test_documents/
|
|
17
|
-
//! - Assert on extracted content and behavior, not linking internals
|
|
18
|
-
//! - Test both successful extraction and error cases
|
|
19
|
-
//! - Skip tests gracefully if test files are missing
|
|
20
|
-
|
|
21
|
-
#![cfg(feature = "pdf")]
|
|
22
|
-
|
|
23
|
-
mod helpers;
|
|
24
|
-
|
|
25
|
-
use helpers::*;
|
|
26
|
-
use kreuzberg::core::config::ExtractionConfig;
|
|
27
|
-
use kreuzberg::extract_file_sync;
|
|
28
|
-
|
|
29
|
-
/// Test basic PDF extraction works regardless of linking strategy.
|
|
30
|
-
///
|
|
31
|
-
/// This is the primary integration test - it verifies that PDF extraction
|
|
32
|
-
/// functions correctly with any linking strategy. We use a small test PDF
|
|
33
|
-
/// and verify that:
|
|
34
|
-
/// - Extraction completes successfully
|
|
35
|
-
/// - Content is non-empty and contains expected text
|
|
36
|
-
/// - MIME type is correctly detected
|
|
37
|
-
#[test]
|
|
38
|
-
fn test_pdf_extraction_basic() {
|
|
39
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
40
|
-
return;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
44
|
-
let config = ExtractionConfig::default();
|
|
45
|
-
|
|
46
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF successfully");
|
|
47
|
-
|
|
48
|
-
// Verify MIME type detection
|
|
49
|
-
assert_mime_type(&result, "application/pdf");
|
|
50
|
-
|
|
51
|
-
// Verify content was extracted
|
|
52
|
-
assert_non_empty_content(&result);
|
|
53
|
-
|
|
54
|
-
// Verify reasonable content length for a small PDF
|
|
55
|
-
assert_min_content_length(&result, 10);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
/// Test PDF extraction with a medium-sized test PDF.
|
|
59
|
-
///
|
|
60
|
-
/// Verifies that extraction works with slightly larger PDFs and produces
|
|
61
|
-
/// substantive content. This test helps ensure linking strategy works
|
|
62
|
-
/// with real-world document sizes.
|
|
63
|
-
#[test]
|
|
64
|
-
fn test_pdf_extraction_medium() {
|
|
65
|
-
if skip_if_missing("pdfs_with_tables/medium.pdf") {
|
|
66
|
-
return;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
let file_path = get_test_file_path("pdfs_with_tables/medium.pdf");
|
|
70
|
-
let config = ExtractionConfig::default();
|
|
71
|
-
|
|
72
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract medium PDF successfully");
|
|
73
|
-
|
|
74
|
-
assert_mime_type(&result, "application/pdf");
|
|
75
|
-
assert_non_empty_content(&result);
|
|
76
|
-
|
|
77
|
-
// Medium PDF should have more content than tiny
|
|
78
|
-
assert!(result.content.len() >= 50, "Medium PDF should have substantial content");
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/// Test PDF extraction with rotated pages.
|
|
82
|
-
///
|
|
83
|
-
/// Some PDFs have rotated pages. This test verifies that the linking
|
|
84
|
-
/// strategy handles page rotation correctly (PDFium should handle this
|
|
85
|
-
/// transparently). Note: This specific PDF may have OCR-only content,
|
|
86
|
-
/// so we just verify extraction completes without error.
|
|
87
|
-
#[test]
|
|
88
|
-
fn test_pdf_extraction_rotated_pages() {
|
|
89
|
-
if skip_if_missing("pdfs/ocr_test_rotated_90.pdf") {
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
let file_path = get_test_file_path("pdfs/ocr_test_rotated_90.pdf");
|
|
94
|
-
let config = ExtractionConfig::default();
|
|
95
|
-
|
|
96
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract rotated PDF successfully");
|
|
97
|
-
|
|
98
|
-
assert_mime_type(&result, "application/pdf");
|
|
99
|
-
// Rotated PDFs may have OCR-only content, just verify extraction completes
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/// Test PDF text extraction with code and formula content.
|
|
103
|
-
///
|
|
104
|
-
/// Verifies extraction works with technical PDFs containing code blocks,
|
|
105
|
-
/// mathematical formulas, and special formatting.
|
|
106
|
-
#[test]
|
|
107
|
-
fn test_pdf_extraction_code_and_formulas() {
|
|
108
|
-
if skip_if_missing("pdfs/code_and_formula.pdf") {
|
|
109
|
-
return;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
let file_path = get_test_file_path("pdfs/code_and_formula.pdf");
|
|
113
|
-
let config = ExtractionConfig::default();
|
|
114
|
-
|
|
115
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with code and formulas");
|
|
116
|
-
|
|
117
|
-
assert_mime_type(&result, "application/pdf");
|
|
118
|
-
assert_non_empty_content(&result);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/// Test PDF extraction preserves right-to-left text direction.
|
|
122
|
-
///
|
|
123
|
-
/// PDFs with RTL text (Arabic, Hebrew, etc.) require special handling.
|
|
124
|
-
/// This verifies extraction works with RTL content.
|
|
125
|
-
#[test]
|
|
126
|
-
fn test_pdf_extraction_right_to_left() {
|
|
127
|
-
if skip_if_missing("pdfs/right_to_left_01.pdf") {
|
|
128
|
-
return;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
|
|
132
|
-
let config = ExtractionConfig::default();
|
|
133
|
-
|
|
134
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract RTL PDF successfully");
|
|
135
|
-
|
|
136
|
-
assert_mime_type(&result, "application/pdf");
|
|
137
|
-
// RTL PDFs might have minimal or no extracted text depending on PDF structure
|
|
138
|
-
// Just verify extraction completes without error
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/// Test PDF metadata extraction.
|
|
142
|
-
///
|
|
143
|
-
/// Verifies that PDF metadata (title, author, creation date, etc.) can be
|
|
144
|
-
/// extracted correctly, which is independent of linking strategy.
|
|
145
|
-
#[test]
|
|
146
|
-
fn test_pdf_metadata_extraction() {
|
|
147
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
148
|
-
return;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
152
|
-
let config = ExtractionConfig::default();
|
|
153
|
-
|
|
154
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF metadata");
|
|
155
|
-
|
|
156
|
-
// Metadata field is always present (may have empty fields for some PDFs)
|
|
157
|
-
// Just verify the structure exists
|
|
158
|
-
let _metadata = &result.metadata;
|
|
159
|
-
assert!(!result.mime_type.is_empty(), "MIME type should be set");
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/// Test extraction of byte array from PDF.
|
|
163
|
-
///
|
|
164
|
-
/// Verifies that extract_bytes_sync works for PDF content, which is
|
|
165
|
-
/// important for in-memory processing.
|
|
166
|
-
#[test]
|
|
167
|
-
fn test_pdf_extraction_from_bytes() {
|
|
168
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
169
|
-
return;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
173
|
-
let pdf_bytes = std::fs::read(&file_path).expect("Should read PDF file");
|
|
174
|
-
|
|
175
|
-
let config = ExtractionConfig::default();
|
|
176
|
-
let result = kreuzberg::extract_bytes_sync(&pdf_bytes, "application/pdf", &config)
|
|
177
|
-
.expect("Should extract PDF from bytes successfully");
|
|
178
|
-
|
|
179
|
-
assert_mime_type(&result, "application/pdf");
|
|
180
|
-
assert_non_empty_content(&result);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/// Test bundled PDFium extraction when pdf-bundled feature is enabled.
|
|
184
|
-
///
|
|
185
|
-
/// When `pdf-bundled` feature is enabled, PDFium is embedded in the binary.
|
|
186
|
-
/// This test verifies the bundled extraction mechanism works correctly.
|
|
187
|
-
#[test]
|
|
188
|
-
#[cfg(feature = "pdf-bundled")]
|
|
189
|
-
fn test_bundled_pdfium_extraction() {
|
|
190
|
-
use kreuzberg::pdf::extract_bundled_pdfium;
|
|
191
|
-
|
|
192
|
-
// Extract bundled PDFium library to temporary directory
|
|
193
|
-
let lib_path = extract_bundled_pdfium().expect("Should extract bundled PDFium library");
|
|
194
|
-
|
|
195
|
-
// Verify extracted library exists and is a file
|
|
196
|
-
assert!(
|
|
197
|
-
lib_path.exists(),
|
|
198
|
-
"Extracted PDFium library should exist at {}",
|
|
199
|
-
lib_path.display()
|
|
200
|
-
);
|
|
201
|
-
assert!(
|
|
202
|
-
lib_path.is_file(),
|
|
203
|
-
"Extracted PDFium library should be a file, not directory"
|
|
204
|
-
);
|
|
205
|
-
|
|
206
|
-
// Verify library has a reasonable file size (at least a few MB)
|
|
207
|
-
let metadata = std::fs::metadata(&lib_path).expect("Should read library metadata");
|
|
208
|
-
assert!(
|
|
209
|
-
metadata.len() > 1_000_000,
|
|
210
|
-
"Bundled PDFium library should be at least 1MB (got {} bytes)",
|
|
211
|
-
metadata.len()
|
|
212
|
-
);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
/// Test bundled PDFium extracts consistently.
|
|
216
|
-
///
|
|
217
|
-
/// Verify that repeated calls to extract_bundled_pdfium return the same path
|
|
218
|
-
/// and don't create duplicate extracted libraries.
|
|
219
|
-
#[test]
|
|
220
|
-
#[cfg(feature = "pdf-bundled")]
|
|
221
|
-
fn test_bundled_pdfium_caching() {
|
|
222
|
-
use kreuzberg::pdf::extract_bundled_pdfium;
|
|
223
|
-
|
|
224
|
-
let path1 = extract_bundled_pdfium().expect("First extraction should succeed");
|
|
225
|
-
let path2 = extract_bundled_pdfium().expect("Second extraction should succeed");
|
|
226
|
-
|
|
227
|
-
// Both calls should return the same path (caching in place)
|
|
228
|
-
assert_eq!(path1, path2, "Multiple extractions should return consistent path");
|
|
229
|
-
|
|
230
|
-
// Path should still exist
|
|
231
|
-
assert!(path1.exists(), "Extracted library should still exist");
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/// Test bundled PDFium works with PDF extraction.
|
|
235
|
-
///
|
|
236
|
-
/// This integration test verifies that PDFium extracted via bundled mechanism
|
|
237
|
-
/// actually works for PDF content extraction.
|
|
238
|
-
#[test]
|
|
239
|
-
#[cfg(feature = "pdf-bundled")]
|
|
240
|
-
fn test_bundled_pdfium_with_pdf_extraction() {
|
|
241
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
// Ensure bundled PDFium is extracted and available
|
|
246
|
-
let _lib_path = kreuzberg::pdf::extract_bundled_pdfium().expect("Should extract bundled PDFium library");
|
|
247
|
-
|
|
248
|
-
// Now test that PDF extraction works with the bundled library
|
|
249
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
250
|
-
let config = ExtractionConfig::default();
|
|
251
|
-
|
|
252
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with bundled PDFium");
|
|
253
|
-
|
|
254
|
-
assert_mime_type(&result, "application/pdf");
|
|
255
|
-
assert_non_empty_content(&result);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/// Test extraction with custom PDF configuration.
|
|
259
|
-
///
|
|
260
|
-
/// Verifies that custom PDF extraction settings work correctly.
|
|
261
|
-
#[test]
|
|
262
|
-
fn test_pdf_extraction_with_config() {
|
|
263
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
264
|
-
return;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
268
|
-
|
|
269
|
-
// Create custom extraction config
|
|
270
|
-
let config = ExtractionConfig::default();
|
|
271
|
-
|
|
272
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with custom config");
|
|
273
|
-
|
|
274
|
-
assert_mime_type(&result, "application/pdf");
|
|
275
|
-
assert_non_empty_content(&result);
|
|
276
|
-
|
|
277
|
-
// Verify metadata is present
|
|
278
|
-
assert!(!result.mime_type.is_empty(), "MIME type should be set in result");
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
/// Test handling of empty or minimal PDFs.
|
|
282
|
-
///
|
|
283
|
-
/// Some edge case PDFs might be empty or contain no text. Verify extraction
|
|
284
|
-
/// handles these gracefully.
|
|
285
|
-
#[test]
|
|
286
|
-
fn test_pdf_extraction_edge_cases() {
|
|
287
|
-
// Test extraction of a minimal PDF
|
|
288
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
289
|
-
return;
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
293
|
-
let config = ExtractionConfig::default();
|
|
294
|
-
|
|
295
|
-
// Should complete without panic, regardless of content
|
|
296
|
-
let result = extract_file_sync(&file_path, None, &config);
|
|
297
|
-
assert!(result.is_ok(), "PDF extraction should succeed");
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/// Test batch extraction of multiple PDFs.
|
|
301
|
-
///
|
|
302
|
-
/// Verifies that linking strategy handles batch processing of multiple
|
|
303
|
-
/// PDF files correctly.
|
|
304
|
-
#[test]
|
|
305
|
-
#[cfg(feature = "tokio-runtime")]
|
|
306
|
-
fn test_pdf_batch_extraction() {
|
|
307
|
-
if skip_if_missing("pdfs_with_tables/tiny.pdf") || skip_if_missing("pdfs_with_tables/medium.pdf") {
|
|
308
|
-
return;
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
let paths = vec![
|
|
312
|
-
get_test_file_path("pdfs_with_tables/tiny.pdf"),
|
|
313
|
-
get_test_file_path("pdfs_with_tables/medium.pdf"),
|
|
314
|
-
];
|
|
315
|
-
|
|
316
|
-
let config = ExtractionConfig::default();
|
|
317
|
-
|
|
318
|
-
// Use synchronous batch extraction
|
|
319
|
-
let results = kreuzberg::batch_extract_file_sync(paths, &config).expect("Should extract PDFs in batch");
|
|
320
|
-
|
|
321
|
-
// Verify all PDFs were extracted
|
|
322
|
-
assert_eq!(results.len(), 2, "Should extract both PDFs");
|
|
323
|
-
|
|
324
|
-
// Verify all results are valid
|
|
325
|
-
for result in &results {
|
|
326
|
-
assert_mime_type(result, "application/pdf");
|
|
327
|
-
assert_non_empty_content(result);
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/// Test PDF with Unicode content extraction.
|
|
332
|
-
///
|
|
333
|
-
/// Verifies that PDFium linking strategy handles PDFs with international
|
|
334
|
-
/// characters correctly.
|
|
335
|
-
#[test]
|
|
336
|
-
fn test_pdf_unicode_content() {
|
|
337
|
-
// Right-to-left PDFs often contain Unicode content
|
|
338
|
-
if skip_if_missing("pdfs/right_to_left_01.pdf") {
|
|
339
|
-
return;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
|
|
343
|
-
let config = ExtractionConfig::default();
|
|
344
|
-
|
|
345
|
-
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with Unicode content");
|
|
346
|
-
|
|
347
|
-
// Just verify extraction completes - Unicode content handling
|
|
348
|
-
// varies by PDF structure
|
|
349
|
-
assert_mime_type(&result, "application/pdf");
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
/// Verify PDF module compiles with all feature combinations.
|
|
353
|
-
///
|
|
354
|
-
/// This is a compile-time test that ensures the pdf module and bundled
|
|
355
|
-
/// submodule (when enabled) compile correctly. It's implicitly tested
|
|
356
|
-
/// by the fact that these test functions compile.
|
|
357
|
-
#[test]
|
|
358
|
-
#[cfg(feature = "pdf")]
|
|
359
|
-
fn test_pdf_module_availability() {
|
|
360
|
-
// This test verifies that:
|
|
361
|
-
// 1. The pdf module is available (feature = "pdf")
|
|
362
|
-
// 2. Key functions are accessible
|
|
363
|
-
// 3. pdf-bundled module is available when feature enabled
|
|
364
|
-
|
|
365
|
-
// Verify we can access the text extraction function
|
|
366
|
-
let _ = kreuzberg::pdf::extract_text_from_pdf;
|
|
367
|
-
|
|
368
|
-
// Verify we can access the metadata function
|
|
369
|
-
let _ = kreuzberg::pdf::extract_metadata;
|
|
370
|
-
|
|
371
|
-
// Verify bundled module is available when feature enabled
|
|
372
|
-
#[cfg(feature = "pdf-bundled")]
|
|
373
|
-
let _ = kreuzberg::pdf::extract_bundled_pdfium;
|
|
374
|
-
}
|
|
1
|
+
//! PDFium linking integration tests.
|
|
2
|
+
//!
|
|
3
|
+
//! This module tests PDF extraction functionality across all PDFium linking strategies
|
|
4
|
+
//! (default, static, bundled, system). Tests verify that:
|
|
5
|
+
//!
|
|
6
|
+
//! 1. PDF extraction works regardless of linking strategy
|
|
7
|
+
//! 2. Extracted content is valid and non-empty
|
|
8
|
+
//! 3. Bundled PDFium extraction works when feature is enabled
|
|
9
|
+
//! 4. Metadata extraction functions correctly
|
|
10
|
+
//! 5. Error handling works for invalid PDFs
|
|
11
|
+
//!
|
|
12
|
+
//! These tests are strategy-agnostic - they verify functionality works with ANY
|
|
13
|
+
//! linking strategy, not implementation details of specific strategies.
|
|
14
|
+
//!
|
|
15
|
+
//! Test philosophy:
|
|
16
|
+
//! - Use small test PDFs from test_documents/
|
|
17
|
+
//! - Assert on extracted content and behavior, not linking internals
|
|
18
|
+
//! - Test both successful extraction and error cases
|
|
19
|
+
//! - Skip tests gracefully if test files are missing
|
|
20
|
+
|
|
21
|
+
#![cfg(feature = "pdf")]
|
|
22
|
+
|
|
23
|
+
mod helpers;
|
|
24
|
+
|
|
25
|
+
use helpers::*;
|
|
26
|
+
use kreuzberg::core::config::ExtractionConfig;
|
|
27
|
+
use kreuzberg::extract_file_sync;
|
|
28
|
+
|
|
29
|
+
/// Test basic PDF extraction works regardless of linking strategy.
|
|
30
|
+
///
|
|
31
|
+
/// This is the primary integration test - it verifies that PDF extraction
|
|
32
|
+
/// functions correctly with any linking strategy. We use a small test PDF
|
|
33
|
+
/// and verify that:
|
|
34
|
+
/// - Extraction completes successfully
|
|
35
|
+
/// - Content is non-empty and contains expected text
|
|
36
|
+
/// - MIME type is correctly detected
|
|
37
|
+
#[test]
|
|
38
|
+
fn test_pdf_extraction_basic() {
|
|
39
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
44
|
+
let config = ExtractionConfig::default();
|
|
45
|
+
|
|
46
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF successfully");
|
|
47
|
+
|
|
48
|
+
// Verify MIME type detection
|
|
49
|
+
assert_mime_type(&result, "application/pdf");
|
|
50
|
+
|
|
51
|
+
// Verify content was extracted
|
|
52
|
+
assert_non_empty_content(&result);
|
|
53
|
+
|
|
54
|
+
// Verify reasonable content length for a small PDF
|
|
55
|
+
assert_min_content_length(&result, 10);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/// Test PDF extraction with a medium-sized test PDF.
|
|
59
|
+
///
|
|
60
|
+
/// Verifies that extraction works with slightly larger PDFs and produces
|
|
61
|
+
/// substantive content. This test helps ensure linking strategy works
|
|
62
|
+
/// with real-world document sizes.
|
|
63
|
+
#[test]
|
|
64
|
+
fn test_pdf_extraction_medium() {
|
|
65
|
+
if skip_if_missing("pdfs_with_tables/medium.pdf") {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
let file_path = get_test_file_path("pdfs_with_tables/medium.pdf");
|
|
70
|
+
let config = ExtractionConfig::default();
|
|
71
|
+
|
|
72
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract medium PDF successfully");
|
|
73
|
+
|
|
74
|
+
assert_mime_type(&result, "application/pdf");
|
|
75
|
+
assert_non_empty_content(&result);
|
|
76
|
+
|
|
77
|
+
// Medium PDF should have more content than tiny
|
|
78
|
+
assert!(result.content.len() >= 50, "Medium PDF should have substantial content");
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Test PDF extraction with rotated pages.
|
|
82
|
+
///
|
|
83
|
+
/// Some PDFs have rotated pages. This test verifies that the linking
|
|
84
|
+
/// strategy handles page rotation correctly (PDFium should handle this
|
|
85
|
+
/// transparently). Note: This specific PDF may have OCR-only content,
|
|
86
|
+
/// so we just verify extraction completes without error.
|
|
87
|
+
#[test]
|
|
88
|
+
fn test_pdf_extraction_rotated_pages() {
|
|
89
|
+
if skip_if_missing("pdfs/ocr_test_rotated_90.pdf") {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let file_path = get_test_file_path("pdfs/ocr_test_rotated_90.pdf");
|
|
94
|
+
let config = ExtractionConfig::default();
|
|
95
|
+
|
|
96
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract rotated PDF successfully");
|
|
97
|
+
|
|
98
|
+
assert_mime_type(&result, "application/pdf");
|
|
99
|
+
// Rotated PDFs may have OCR-only content, just verify extraction completes
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/// Test PDF text extraction with code and formula content.
|
|
103
|
+
///
|
|
104
|
+
/// Verifies extraction works with technical PDFs containing code blocks,
|
|
105
|
+
/// mathematical formulas, and special formatting.
|
|
106
|
+
#[test]
|
|
107
|
+
fn test_pdf_extraction_code_and_formulas() {
|
|
108
|
+
if skip_if_missing("pdfs/code_and_formula.pdf") {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let file_path = get_test_file_path("pdfs/code_and_formula.pdf");
|
|
113
|
+
let config = ExtractionConfig::default();
|
|
114
|
+
|
|
115
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with code and formulas");
|
|
116
|
+
|
|
117
|
+
assert_mime_type(&result, "application/pdf");
|
|
118
|
+
assert_non_empty_content(&result);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/// Test PDF extraction preserves right-to-left text direction.
|
|
122
|
+
///
|
|
123
|
+
/// PDFs with RTL text (Arabic, Hebrew, etc.) require special handling.
|
|
124
|
+
/// This verifies extraction works with RTL content.
|
|
125
|
+
#[test]
|
|
126
|
+
fn test_pdf_extraction_right_to_left() {
|
|
127
|
+
if skip_if_missing("pdfs/right_to_left_01.pdf") {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
|
|
132
|
+
let config = ExtractionConfig::default();
|
|
133
|
+
|
|
134
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract RTL PDF successfully");
|
|
135
|
+
|
|
136
|
+
assert_mime_type(&result, "application/pdf");
|
|
137
|
+
// RTL PDFs might have minimal or no extracted text depending on PDF structure
|
|
138
|
+
// Just verify extraction completes without error
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/// Test PDF metadata extraction.
|
|
142
|
+
///
|
|
143
|
+
/// Verifies that PDF metadata (title, author, creation date, etc.) can be
|
|
144
|
+
/// extracted correctly, which is independent of linking strategy.
|
|
145
|
+
#[test]
|
|
146
|
+
fn test_pdf_metadata_extraction() {
|
|
147
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
152
|
+
let config = ExtractionConfig::default();
|
|
153
|
+
|
|
154
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF metadata");
|
|
155
|
+
|
|
156
|
+
// Metadata field is always present (may have empty fields for some PDFs)
|
|
157
|
+
// Just verify the structure exists
|
|
158
|
+
let _metadata = &result.metadata;
|
|
159
|
+
assert!(!result.mime_type.is_empty(), "MIME type should be set");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/// Test extraction of byte array from PDF.
|
|
163
|
+
///
|
|
164
|
+
/// Verifies that extract_bytes_sync works for PDF content, which is
|
|
165
|
+
/// important for in-memory processing.
|
|
166
|
+
#[test]
|
|
167
|
+
fn test_pdf_extraction_from_bytes() {
|
|
168
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
173
|
+
let pdf_bytes = std::fs::read(&file_path).expect("Should read PDF file");
|
|
174
|
+
|
|
175
|
+
let config = ExtractionConfig::default();
|
|
176
|
+
let result = kreuzberg::extract_bytes_sync(&pdf_bytes, "application/pdf", &config)
|
|
177
|
+
.expect("Should extract PDF from bytes successfully");
|
|
178
|
+
|
|
179
|
+
assert_mime_type(&result, "application/pdf");
|
|
180
|
+
assert_non_empty_content(&result);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// Test bundled PDFium extraction when pdf-bundled feature is enabled.
|
|
184
|
+
///
|
|
185
|
+
/// When `pdf-bundled` feature is enabled, PDFium is embedded in the binary.
|
|
186
|
+
/// This test verifies the bundled extraction mechanism works correctly.
|
|
187
|
+
#[test]
|
|
188
|
+
#[cfg(feature = "pdf-bundled")]
|
|
189
|
+
fn test_bundled_pdfium_extraction() {
|
|
190
|
+
use kreuzberg::pdf::extract_bundled_pdfium;
|
|
191
|
+
|
|
192
|
+
// Extract bundled PDFium library to temporary directory
|
|
193
|
+
let lib_path = extract_bundled_pdfium().expect("Should extract bundled PDFium library");
|
|
194
|
+
|
|
195
|
+
// Verify extracted library exists and is a file
|
|
196
|
+
assert!(
|
|
197
|
+
lib_path.exists(),
|
|
198
|
+
"Extracted PDFium library should exist at {}",
|
|
199
|
+
lib_path.display()
|
|
200
|
+
);
|
|
201
|
+
assert!(
|
|
202
|
+
lib_path.is_file(),
|
|
203
|
+
"Extracted PDFium library should be a file, not directory"
|
|
204
|
+
);
|
|
205
|
+
|
|
206
|
+
// Verify library has a reasonable file size (at least a few MB)
|
|
207
|
+
let metadata = std::fs::metadata(&lib_path).expect("Should read library metadata");
|
|
208
|
+
assert!(
|
|
209
|
+
metadata.len() > 1_000_000,
|
|
210
|
+
"Bundled PDFium library should be at least 1MB (got {} bytes)",
|
|
211
|
+
metadata.len()
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/// Test bundled PDFium extracts consistently.
|
|
216
|
+
///
|
|
217
|
+
/// Verify that repeated calls to extract_bundled_pdfium return the same path
|
|
218
|
+
/// and don't create duplicate extracted libraries.
|
|
219
|
+
#[test]
|
|
220
|
+
#[cfg(feature = "pdf-bundled")]
|
|
221
|
+
fn test_bundled_pdfium_caching() {
|
|
222
|
+
use kreuzberg::pdf::extract_bundled_pdfium;
|
|
223
|
+
|
|
224
|
+
let path1 = extract_bundled_pdfium().expect("First extraction should succeed");
|
|
225
|
+
let path2 = extract_bundled_pdfium().expect("Second extraction should succeed");
|
|
226
|
+
|
|
227
|
+
// Both calls should return the same path (caching in place)
|
|
228
|
+
assert_eq!(path1, path2, "Multiple extractions should return consistent path");
|
|
229
|
+
|
|
230
|
+
// Path should still exist
|
|
231
|
+
assert!(path1.exists(), "Extracted library should still exist");
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/// Test bundled PDFium works with PDF extraction.
|
|
235
|
+
///
|
|
236
|
+
/// This integration test verifies that PDFium extracted via bundled mechanism
|
|
237
|
+
/// actually works for PDF content extraction.
|
|
238
|
+
#[test]
|
|
239
|
+
#[cfg(feature = "pdf-bundled")]
|
|
240
|
+
fn test_bundled_pdfium_with_pdf_extraction() {
|
|
241
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Ensure bundled PDFium is extracted and available
|
|
246
|
+
let _lib_path = kreuzberg::pdf::extract_bundled_pdfium().expect("Should extract bundled PDFium library");
|
|
247
|
+
|
|
248
|
+
// Now test that PDF extraction works with the bundled library
|
|
249
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
250
|
+
let config = ExtractionConfig::default();
|
|
251
|
+
|
|
252
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with bundled PDFium");
|
|
253
|
+
|
|
254
|
+
assert_mime_type(&result, "application/pdf");
|
|
255
|
+
assert_non_empty_content(&result);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/// Test extraction with custom PDF configuration.
|
|
259
|
+
///
|
|
260
|
+
/// Verifies that custom PDF extraction settings work correctly.
|
|
261
|
+
#[test]
|
|
262
|
+
fn test_pdf_extraction_with_config() {
|
|
263
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
268
|
+
|
|
269
|
+
// Create custom extraction config
|
|
270
|
+
let config = ExtractionConfig::default();
|
|
271
|
+
|
|
272
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with custom config");
|
|
273
|
+
|
|
274
|
+
assert_mime_type(&result, "application/pdf");
|
|
275
|
+
assert_non_empty_content(&result);
|
|
276
|
+
|
|
277
|
+
// Verify metadata is present
|
|
278
|
+
assert!(!result.mime_type.is_empty(), "MIME type should be set in result");
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/// Test handling of empty or minimal PDFs.
|
|
282
|
+
///
|
|
283
|
+
/// Some edge case PDFs might be empty or contain no text. Verify extraction
|
|
284
|
+
/// handles these gracefully.
|
|
285
|
+
#[test]
|
|
286
|
+
fn test_pdf_extraction_edge_cases() {
|
|
287
|
+
// Test extraction of a minimal PDF
|
|
288
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") {
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
|
|
293
|
+
let config = ExtractionConfig::default();
|
|
294
|
+
|
|
295
|
+
// Should complete without panic, regardless of content
|
|
296
|
+
let result = extract_file_sync(&file_path, None, &config);
|
|
297
|
+
assert!(result.is_ok(), "PDF extraction should succeed");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/// Test batch extraction of multiple PDFs.
|
|
301
|
+
///
|
|
302
|
+
/// Verifies that linking strategy handles batch processing of multiple
|
|
303
|
+
/// PDF files correctly.
|
|
304
|
+
#[test]
|
|
305
|
+
#[cfg(feature = "tokio-runtime")]
|
|
306
|
+
fn test_pdf_batch_extraction() {
|
|
307
|
+
if skip_if_missing("pdfs_with_tables/tiny.pdf") || skip_if_missing("pdfs_with_tables/medium.pdf") {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
let paths = vec![
|
|
312
|
+
get_test_file_path("pdfs_with_tables/tiny.pdf"),
|
|
313
|
+
get_test_file_path("pdfs_with_tables/medium.pdf"),
|
|
314
|
+
];
|
|
315
|
+
|
|
316
|
+
let config = ExtractionConfig::default();
|
|
317
|
+
|
|
318
|
+
// Use synchronous batch extraction
|
|
319
|
+
let results = kreuzberg::batch_extract_file_sync(paths, &config).expect("Should extract PDFs in batch");
|
|
320
|
+
|
|
321
|
+
// Verify all PDFs were extracted
|
|
322
|
+
assert_eq!(results.len(), 2, "Should extract both PDFs");
|
|
323
|
+
|
|
324
|
+
// Verify all results are valid
|
|
325
|
+
for result in &results {
|
|
326
|
+
assert_mime_type(result, "application/pdf");
|
|
327
|
+
assert_non_empty_content(result);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/// Test PDF with Unicode content extraction.
|
|
332
|
+
///
|
|
333
|
+
/// Verifies that PDFium linking strategy handles PDFs with international
|
|
334
|
+
/// characters correctly.
|
|
335
|
+
#[test]
|
|
336
|
+
fn test_pdf_unicode_content() {
|
|
337
|
+
// Right-to-left PDFs often contain Unicode content
|
|
338
|
+
if skip_if_missing("pdfs/right_to_left_01.pdf") {
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
|
|
343
|
+
let config = ExtractionConfig::default();
|
|
344
|
+
|
|
345
|
+
let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with Unicode content");
|
|
346
|
+
|
|
347
|
+
// Just verify extraction completes - Unicode content handling
|
|
348
|
+
// varies by PDF structure
|
|
349
|
+
assert_mime_type(&result, "application/pdf");
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/// Verify PDF module compiles with all feature combinations.
|
|
353
|
+
///
|
|
354
|
+
/// This is a compile-time test that ensures the pdf module and bundled
|
|
355
|
+
/// submodule (when enabled) compile correctly. It's implicitly tested
|
|
356
|
+
/// by the fact that these test functions compile.
|
|
357
|
+
#[test]
|
|
358
|
+
#[cfg(feature = "pdf")]
|
|
359
|
+
fn test_pdf_module_availability() {
|
|
360
|
+
// This test verifies that:
|
|
361
|
+
// 1. The pdf module is available (feature = "pdf")
|
|
362
|
+
// 2. Key functions are accessible
|
|
363
|
+
// 3. pdf-bundled module is available when feature enabled
|
|
364
|
+
|
|
365
|
+
// Verify we can access the text extraction function
|
|
366
|
+
let _ = kreuzberg::pdf::extract_text_from_pdf;
|
|
367
|
+
|
|
368
|
+
// Verify we can access the metadata function
|
|
369
|
+
let _ = kreuzberg::pdf::extract_metadata;
|
|
370
|
+
|
|
371
|
+
// Verify bundled module is available when feature enabled
|
|
372
|
+
#[cfg(feature = "pdf-bundled")]
|
|
373
|
+
let _ = kreuzberg::pdf::extract_bundled_pdfium;
|
|
374
|
+
}
|