kreuzberg 4.0.0.pre.rc.8 → 4.0.0.pre.rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +4 -104
- data/README.md +454 -432
- data/Rakefile +25 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6941 -6721
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3135
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -182
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -46
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -32
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -85
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -103
- data/lib/pdfium.dll +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -537
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +45 -0
- data/vendor/kreuzberg/Cargo.toml +61 -38
- data/vendor/kreuzberg/README.md +230 -221
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -891
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1171
- data/vendor/kreuzberg/src/embeddings.rs +500 -432
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -569
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -562
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -673
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -0
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -328
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -66
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -417
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -161
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +63 -0
- data/vendor/kreuzberg-ffi/README.md +851 -0
- data/vendor/kreuzberg-ffi/build.rs +176 -0
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -0
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +12 -0
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -0
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -0
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -0
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -0
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -0
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -0
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -0
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -0
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -0
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -0
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -0
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -0
- data/vendor/kreuzberg-tesseract/Cargo.toml +48 -0
- data/vendor/kreuzberg-tesseract/LICENSE +22 -0
- data/vendor/kreuzberg-tesseract/README.md +399 -0
- data/vendor/kreuzberg-tesseract/build.rs +1354 -0
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -0
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -0
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -0
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -0
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -0
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -0
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -0
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -0
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -0
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -0
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -0
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -0
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -0
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +44 -81
- data/vendor/rb-sys/bin/release.sh +0 -21
|
@@ -1,628 +1,628 @@
|
|
|
1
|
-
#![cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
2
|
-
|
|
3
|
-
//! ODT (OpenDocument Text) extractor using native Rust parsing.
|
|
4
|
-
//!
|
|
5
|
-
//! Supports: OpenDocument Text (.odt)
|
|
6
|
-
|
|
7
|
-
use crate::Result;
|
|
8
|
-
use crate::core::config::ExtractionConfig;
|
|
9
|
-
use crate::extraction::{cells_to_markdown, office_metadata};
|
|
10
|
-
use crate::plugins::{DocumentExtractor, Plugin};
|
|
11
|
-
use crate::types::{ExtractionResult, Metadata, Table};
|
|
12
|
-
use async_trait::async_trait;
|
|
13
|
-
use roxmltree::Document;
|
|
14
|
-
use std::io::Cursor;
|
|
15
|
-
|
|
16
|
-
/// High-performance ODT extractor using native Rust XML parsing.
|
|
17
|
-
///
|
|
18
|
-
/// This extractor provides:
|
|
19
|
-
/// - Fast text extraction via roxmltree XML parsing
|
|
20
|
-
/// - Comprehensive metadata extraction from meta.xml
|
|
21
|
-
/// - Table extraction with row and cell support
|
|
22
|
-
/// - Formatting preservation (bold, italic, strikeout)
|
|
23
|
-
/// - Support for headings, paragraphs, and special elements
|
|
24
|
-
pub struct OdtExtractor;
|
|
25
|
-
|
|
26
|
-
impl OdtExtractor {
|
|
27
|
-
/// Create a new ODT extractor.
|
|
28
|
-
pub fn new() -> Self {
|
|
29
|
-
Self
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
impl Default for OdtExtractor {
|
|
34
|
-
fn default() -> Self {
|
|
35
|
-
Self::new()
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
impl Plugin for OdtExtractor {
|
|
40
|
-
fn name(&self) -> &str {
|
|
41
|
-
"odt-extractor"
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
fn version(&self) -> String {
|
|
45
|
-
env!("CARGO_PKG_VERSION").to_string()
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
fn initialize(&self) -> Result<()> {
|
|
49
|
-
Ok(())
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
fn shutdown(&self) -> Result<()> {
|
|
53
|
-
Ok(())
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
fn description(&self) -> &str {
|
|
57
|
-
"Native Rust ODT (OpenDocument Text) extractor with metadata and table support"
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
fn author(&self) -> &str {
|
|
61
|
-
"Kreuzberg Team"
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/// Extract text from MathML formula element
|
|
66
|
-
///
|
|
67
|
-
/// # Arguments
|
|
68
|
-
/// * `math_node` - The math XML node
|
|
69
|
-
///
|
|
70
|
-
/// # Returns
|
|
71
|
-
/// * `Option<String>` - The extracted formula text
|
|
72
|
-
fn extract_mathml_text(math_node: roxmltree::Node) -> Option<String> {
|
|
73
|
-
for node in math_node.descendants() {
|
|
74
|
-
if node.tag_name().name() == "annotation"
|
|
75
|
-
&& let Some(encoding) = node.attribute("encoding")
|
|
76
|
-
&& encoding.contains("StarMath")
|
|
77
|
-
&& let Some(text) = node.text()
|
|
78
|
-
{
|
|
79
|
-
return Some(text.to_string());
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
let mut formula_parts = Vec::new();
|
|
84
|
-
for node in math_node.descendants() {
|
|
85
|
-
match node.tag_name().name() {
|
|
86
|
-
"mi" | "mo" | "mn" | "ms" | "mtext" => {
|
|
87
|
-
if let Some(text) = node.text() {
|
|
88
|
-
formula_parts.push(text.to_string());
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
_ => {}
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
if !formula_parts.is_empty() {
|
|
96
|
-
Some(formula_parts.join(" "))
|
|
97
|
-
} else {
|
|
98
|
-
None
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/// Extract text from embedded formula objects
|
|
103
|
-
///
|
|
104
|
-
/// # Arguments
|
|
105
|
-
/// * `archive` - ZIP archive containing the ODT document
|
|
106
|
-
///
|
|
107
|
-
/// # Returns
|
|
108
|
-
/// * `String` - Extracted formula content from embedded objects
|
|
109
|
-
fn extract_embedded_formulas(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<String> {
|
|
110
|
-
use std::io::Read;
|
|
111
|
-
let mut formula_parts = Vec::new();
|
|
112
|
-
|
|
113
|
-
let file_names: Vec<String> = archive.file_names().map(|s| s.to_string()).collect();
|
|
114
|
-
|
|
115
|
-
for file_name in file_names {
|
|
116
|
-
if file_name.contains("Object")
|
|
117
|
-
&& file_name.ends_with("content.xml")
|
|
118
|
-
&& let Ok(mut file) = archive.by_name(&file_name)
|
|
119
|
-
{
|
|
120
|
-
let mut xml_content = String::new();
|
|
121
|
-
if file.read_to_string(&mut xml_content).is_ok()
|
|
122
|
-
&& let Ok(doc) = Document::parse(&xml_content)
|
|
123
|
-
{
|
|
124
|
-
let root = doc.root_element();
|
|
125
|
-
|
|
126
|
-
if root.tag_name().name() == "math" {
|
|
127
|
-
if let Some(formula_text) = extract_mathml_text(root) {
|
|
128
|
-
formula_parts.push(formula_text);
|
|
129
|
-
}
|
|
130
|
-
} else {
|
|
131
|
-
for node in root.descendants() {
|
|
132
|
-
if node.tag_name().name() == "math"
|
|
133
|
-
&& let Some(formula_text) = extract_mathml_text(node)
|
|
134
|
-
{
|
|
135
|
-
formula_parts.push(formula_text);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
Ok(formula_parts.join("\n"))
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
/// Extract text content from ODT content.xml
|
|
147
|
-
///
|
|
148
|
-
/// # Arguments
|
|
149
|
-
/// * `archive` - ZIP archive containing the ODT document
|
|
150
|
-
///
|
|
151
|
-
/// # Returns
|
|
152
|
-
/// * `String` - Extracted text content
|
|
153
|
-
fn extract_content_text(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<String> {
|
|
154
|
-
let mut xml_content = String::new();
|
|
155
|
-
|
|
156
|
-
match archive.by_name("content.xml") {
|
|
157
|
-
Ok(mut file) => {
|
|
158
|
-
use std::io::Read;
|
|
159
|
-
file.read_to_string(&mut xml_content)
|
|
160
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to read content.xml: {}", e)))?;
|
|
161
|
-
}
|
|
162
|
-
Err(_) => {
|
|
163
|
-
return Ok(String::new());
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
let doc = Document::parse(&xml_content)
|
|
168
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse content.xml: {}", e)))?;
|
|
169
|
-
|
|
170
|
-
let root = doc.root_element();
|
|
171
|
-
|
|
172
|
-
let mut text_parts: Vec<String> = Vec::new();
|
|
173
|
-
|
|
174
|
-
for body_child in root.children() {
|
|
175
|
-
if body_child.tag_name().name() == "body" {
|
|
176
|
-
for text_elem in body_child.children() {
|
|
177
|
-
if text_elem.tag_name().name() == "text" {
|
|
178
|
-
process_document_elements(text_elem, &mut text_parts);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
Ok(text_parts.join("\n").trim().to_string())
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/// Helper function to process document elements (paragraphs, headings, tables)
|
|
188
|
-
/// Only processes direct children, avoiding nested content like table cells
|
|
189
|
-
fn process_document_elements(parent: roxmltree::Node, text_parts: &mut Vec<String>) {
|
|
190
|
-
for node in parent.children() {
|
|
191
|
-
match node.tag_name().name() {
|
|
192
|
-
"h" => {
|
|
193
|
-
if let Some(text) = extract_node_text(node)
|
|
194
|
-
&& !text.trim().is_empty()
|
|
195
|
-
{
|
|
196
|
-
text_parts.push(format!("# {}", text.trim()));
|
|
197
|
-
text_parts.push(String::new());
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
"p" => {
|
|
201
|
-
if let Some(text) = extract_node_text(node)
|
|
202
|
-
&& !text.trim().is_empty()
|
|
203
|
-
{
|
|
204
|
-
text_parts.push(text.trim().to_string());
|
|
205
|
-
text_parts.push(String::new());
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
"table" => {
|
|
209
|
-
if let Some(table_text) = extract_table_text(node) {
|
|
210
|
-
text_parts.push(table_text);
|
|
211
|
-
text_parts.push(String::new());
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
_ => {}
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
/// Extract text from a single XML node, handling spans and formatting
|
|
220
|
-
///
|
|
221
|
-
/// # Arguments
|
|
222
|
-
/// * `node` - The XML node to extract text from
|
|
223
|
-
///
|
|
224
|
-
/// # Returns
|
|
225
|
-
/// * `Option<String>` - The extracted text with formatting preserved
|
|
226
|
-
fn extract_node_text(node: roxmltree::Node) -> Option<String> {
|
|
227
|
-
let mut text_parts = Vec::new();
|
|
228
|
-
|
|
229
|
-
for child in node.children() {
|
|
230
|
-
match child.tag_name().name() {
|
|
231
|
-
"span" => {
|
|
232
|
-
if let Some(text) = child.text() {
|
|
233
|
-
text_parts.push(text.to_string());
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
"tab" => {
|
|
237
|
-
text_parts.push("\t".to_string());
|
|
238
|
-
}
|
|
239
|
-
"line-break" => {
|
|
240
|
-
text_parts.push("\n".to_string());
|
|
241
|
-
}
|
|
242
|
-
_ => {
|
|
243
|
-
if let Some(text) = child.text() {
|
|
244
|
-
text_parts.push(text.to_string());
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
if text_parts.is_empty() {
|
|
251
|
-
node.text().map(|s| s.to_string())
|
|
252
|
-
} else {
|
|
253
|
-
Some(text_parts.join(""))
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/// Extract table content as text with markdown formatting
|
|
258
|
-
///
|
|
259
|
-
/// # Arguments
|
|
260
|
-
/// * `table_node` - The table XML node
|
|
261
|
-
///
|
|
262
|
-
/// # Returns
|
|
263
|
-
/// * `Option<String>` - Markdown formatted table
|
|
264
|
-
fn extract_table_text(table_node: roxmltree::Node) -> Option<String> {
|
|
265
|
-
let mut rows = Vec::new();
|
|
266
|
-
let mut max_cols = 0;
|
|
267
|
-
|
|
268
|
-
for row_node in table_node.children() {
|
|
269
|
-
if row_node.tag_name().name() == "table-row" {
|
|
270
|
-
let mut row_cells = Vec::new();
|
|
271
|
-
|
|
272
|
-
for cell_node in row_node.children() {
|
|
273
|
-
if cell_node.tag_name().name() == "table-cell" {
|
|
274
|
-
let cell_text = extract_node_text(cell_node).unwrap_or_default();
|
|
275
|
-
row_cells.push(cell_text.trim().to_string());
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
if !row_cells.is_empty() {
|
|
280
|
-
max_cols = max_cols.max(row_cells.len());
|
|
281
|
-
rows.push(row_cells);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
if rows.is_empty() {
|
|
287
|
-
return None;
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
for row in &mut rows {
|
|
291
|
-
while row.len() < max_cols {
|
|
292
|
-
row.push(String::new());
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
let mut markdown = String::new();
|
|
297
|
-
|
|
298
|
-
if !rows.is_empty() {
|
|
299
|
-
markdown.push('|');
|
|
300
|
-
for cell in &rows[0] {
|
|
301
|
-
markdown.push(' ');
|
|
302
|
-
markdown.push_str(cell);
|
|
303
|
-
markdown.push_str(" |");
|
|
304
|
-
}
|
|
305
|
-
markdown.push('\n');
|
|
306
|
-
|
|
307
|
-
markdown.push('|');
|
|
308
|
-
for _ in 0..rows[0].len() {
|
|
309
|
-
markdown.push_str(" --- |");
|
|
310
|
-
}
|
|
311
|
-
markdown.push('\n');
|
|
312
|
-
|
|
313
|
-
for row in rows.iter().skip(1) {
|
|
314
|
-
markdown.push('|');
|
|
315
|
-
for cell in row {
|
|
316
|
-
markdown.push(' ');
|
|
317
|
-
markdown.push_str(cell);
|
|
318
|
-
markdown.push_str(" |");
|
|
319
|
-
}
|
|
320
|
-
markdown.push('\n');
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
Some(markdown)
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
/// Extract tables from ODT content.xml
|
|
328
|
-
///
|
|
329
|
-
/// # Arguments
|
|
330
|
-
/// * `archive` - ZIP archive containing the ODT document
|
|
331
|
-
///
|
|
332
|
-
/// # Returns
|
|
333
|
-
/// * `Result<Vec<Table>>` - Extracted tables
|
|
334
|
-
fn extract_tables(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<Vec<Table>> {
|
|
335
|
-
let mut xml_content = String::new();
|
|
336
|
-
|
|
337
|
-
match archive.by_name("content.xml") {
|
|
338
|
-
Ok(mut file) => {
|
|
339
|
-
use std::io::Read;
|
|
340
|
-
file.read_to_string(&mut xml_content)
|
|
341
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to read content.xml: {}", e)))?;
|
|
342
|
-
}
|
|
343
|
-
Err(_) => {
|
|
344
|
-
return Ok(Vec::new());
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
let doc = Document::parse(&xml_content)
|
|
349
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse content.xml: {}", e)))?;
|
|
350
|
-
|
|
351
|
-
let root = doc.root_element();
|
|
352
|
-
let mut tables = Vec::new();
|
|
353
|
-
let mut table_index = 0;
|
|
354
|
-
|
|
355
|
-
for node in root.descendants() {
|
|
356
|
-
if node.tag_name().name() == "table"
|
|
357
|
-
&& let Some(table) = parse_odt_table(node, table_index)
|
|
358
|
-
{
|
|
359
|
-
tables.push(table);
|
|
360
|
-
table_index += 1;
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
Ok(tables)
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
/// Parse a single ODT table element into a Table struct
|
|
368
|
-
///
|
|
369
|
-
/// # Arguments
|
|
370
|
-
/// * `table_node` - The table XML node
|
|
371
|
-
/// * `table_index` - Index of the table in the document
|
|
372
|
-
///
|
|
373
|
-
/// # Returns
|
|
374
|
-
/// * `Option<Table>` - Parsed table
|
|
375
|
-
fn parse_odt_table(table_node: roxmltree::Node, table_index: usize) -> Option<Table> {
|
|
376
|
-
let mut cells: Vec<Vec<String>> = Vec::new();
|
|
377
|
-
|
|
378
|
-
for row_node in table_node.children() {
|
|
379
|
-
if row_node.tag_name().name() == "table-row" {
|
|
380
|
-
let mut row_cells = Vec::new();
|
|
381
|
-
|
|
382
|
-
for cell_node in row_node.children() {
|
|
383
|
-
if cell_node.tag_name().name() == "table-cell" {
|
|
384
|
-
let cell_text = extract_node_text(cell_node).unwrap_or_default();
|
|
385
|
-
row_cells.push(cell_text.trim().to_string());
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
if !row_cells.is_empty() {
|
|
390
|
-
cells.push(row_cells);
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
if cells.is_empty() {
|
|
396
|
-
return None;
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
let markdown = cells_to_markdown(&cells);
|
|
400
|
-
|
|
401
|
-
Some(Table {
|
|
402
|
-
cells,
|
|
403
|
-
markdown,
|
|
404
|
-
page_number: table_index + 1,
|
|
405
|
-
})
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
#[async_trait]
|
|
409
|
-
impl DocumentExtractor for OdtExtractor {
|
|
410
|
-
#[cfg_attr(
|
|
411
|
-
feature = "otel",
|
|
412
|
-
tracing::instrument(
|
|
413
|
-
skip(self, content, _config),
|
|
414
|
-
fields(
|
|
415
|
-
extractor.name = self.name(),
|
|
416
|
-
content.size_bytes = content.len(),
|
|
417
|
-
)
|
|
418
|
-
)
|
|
419
|
-
)]
|
|
420
|
-
async fn extract_bytes(
|
|
421
|
-
&self,
|
|
422
|
-
content: &[u8],
|
|
423
|
-
mime_type: &str,
|
|
424
|
-
_config: &ExtractionConfig,
|
|
425
|
-
) -> Result<ExtractionResult> {
|
|
426
|
-
let content_owned = content.to_vec();
|
|
427
|
-
|
|
428
|
-
let (text, tables) = if crate::core::batch_mode::is_batch_mode() {
|
|
429
|
-
let content_for_task = content_owned.clone();
|
|
430
|
-
let span = tracing::Span::current();
|
|
431
|
-
tokio::task::spawn_blocking(move || -> crate::error::Result<(String, Vec<Table>)> {
|
|
432
|
-
let _guard = span.entered();
|
|
433
|
-
|
|
434
|
-
let cursor = Cursor::new(content_for_task);
|
|
435
|
-
let mut archive = zip::ZipArchive::new(cursor)
|
|
436
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
|
|
437
|
-
|
|
438
|
-
let text = extract_content_text(&mut archive)?;
|
|
439
|
-
let tables = extract_tables(&mut archive)?;
|
|
440
|
-
let embedded_formulas = extract_embedded_formulas(&mut archive)?;
|
|
441
|
-
|
|
442
|
-
let combined_text = if !embedded_formulas.is_empty() {
|
|
443
|
-
if !text.is_empty() {
|
|
444
|
-
format!("{}\n{}", text, embedded_formulas)
|
|
445
|
-
} else {
|
|
446
|
-
embedded_formulas
|
|
447
|
-
}
|
|
448
|
-
} else {
|
|
449
|
-
text
|
|
450
|
-
};
|
|
451
|
-
|
|
452
|
-
Ok((combined_text, tables))
|
|
453
|
-
})
|
|
454
|
-
.await
|
|
455
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("ODT extraction task failed: {}", e)))??
|
|
456
|
-
} else {
|
|
457
|
-
let cursor = Cursor::new(content_owned.clone());
|
|
458
|
-
let mut archive = zip::ZipArchive::new(cursor)
|
|
459
|
-
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
|
|
460
|
-
|
|
461
|
-
let text = extract_content_text(&mut archive)?;
|
|
462
|
-
let tables = extract_tables(&mut archive)?;
|
|
463
|
-
let embedded_formulas = extract_embedded_formulas(&mut archive)?;
|
|
464
|
-
|
|
465
|
-
let combined_text = if !embedded_formulas.is_empty() {
|
|
466
|
-
if !text.is_empty() {
|
|
467
|
-
format!("{}\n{}", text, embedded_formulas)
|
|
468
|
-
} else {
|
|
469
|
-
embedded_formulas
|
|
470
|
-
}
|
|
471
|
-
} else {
|
|
472
|
-
text
|
|
473
|
-
};
|
|
474
|
-
|
|
475
|
-
(combined_text, tables)
|
|
476
|
-
};
|
|
477
|
-
|
|
478
|
-
let mut metadata_map = std::collections::HashMap::new();
|
|
479
|
-
|
|
480
|
-
let cursor = Cursor::new(content_owned.clone());
|
|
481
|
-
let mut archive = zip::ZipArchive::new(cursor).map_err(|e| {
|
|
482
|
-
crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive for metadata: {}", e))
|
|
483
|
-
})?;
|
|
484
|
-
|
|
485
|
-
if let Ok(odt_props) = office_metadata::extract_odt_properties(&mut archive) {
|
|
486
|
-
if let Some(title) = odt_props.title {
|
|
487
|
-
metadata_map.insert("title".to_string(), serde_json::Value::String(title));
|
|
488
|
-
}
|
|
489
|
-
if let Some(creator) = odt_props.creator {
|
|
490
|
-
metadata_map.insert(
|
|
491
|
-
"authors".to_string(),
|
|
492
|
-
serde_json::Value::Array(vec![serde_json::Value::String(creator.clone())]),
|
|
493
|
-
);
|
|
494
|
-
metadata_map.insert("created_by".to_string(), serde_json::Value::String(creator));
|
|
495
|
-
}
|
|
496
|
-
if let Some(initial_creator) = odt_props.initial_creator {
|
|
497
|
-
metadata_map.insert(
|
|
498
|
-
"initial_creator".to_string(),
|
|
499
|
-
serde_json::Value::String(initial_creator),
|
|
500
|
-
);
|
|
501
|
-
}
|
|
502
|
-
if let Some(subject) = odt_props.subject {
|
|
503
|
-
metadata_map.insert("subject".to_string(), serde_json::Value::String(subject));
|
|
504
|
-
}
|
|
505
|
-
if let Some(keywords) = odt_props.keywords {
|
|
506
|
-
metadata_map.insert("keywords".to_string(), serde_json::Value::String(keywords));
|
|
507
|
-
}
|
|
508
|
-
if let Some(description) = odt_props.description {
|
|
509
|
-
metadata_map.insert("description".to_string(), serde_json::Value::String(description));
|
|
510
|
-
}
|
|
511
|
-
if let Some(creation_date) = odt_props.creation_date {
|
|
512
|
-
metadata_map.insert("created_at".to_string(), serde_json::Value::String(creation_date));
|
|
513
|
-
}
|
|
514
|
-
if let Some(date) = odt_props.date {
|
|
515
|
-
metadata_map.insert("modified_at".to_string(), serde_json::Value::String(date));
|
|
516
|
-
}
|
|
517
|
-
if let Some(language) = odt_props.language {
|
|
518
|
-
metadata_map.insert("language".to_string(), serde_json::Value::String(language));
|
|
519
|
-
}
|
|
520
|
-
if let Some(generator) = odt_props.generator {
|
|
521
|
-
metadata_map.insert("generator".to_string(), serde_json::Value::String(generator));
|
|
522
|
-
}
|
|
523
|
-
if let Some(editing_duration) = odt_props.editing_duration {
|
|
524
|
-
metadata_map.insert(
|
|
525
|
-
"editing_duration".to_string(),
|
|
526
|
-
serde_json::Value::String(editing_duration),
|
|
527
|
-
);
|
|
528
|
-
}
|
|
529
|
-
if let Some(editing_cycles) = odt_props.editing_cycles {
|
|
530
|
-
metadata_map.insert("editing_cycles".to_string(), serde_json::Value::String(editing_cycles));
|
|
531
|
-
}
|
|
532
|
-
if let Some(page_count) = odt_props.page_count {
|
|
533
|
-
metadata_map.insert("page_count".to_string(), serde_json::Value::Number(page_count.into()));
|
|
534
|
-
}
|
|
535
|
-
if let Some(word_count) = odt_props.word_count {
|
|
536
|
-
metadata_map.insert("word_count".to_string(), serde_json::Value::Number(word_count.into()));
|
|
537
|
-
}
|
|
538
|
-
if let Some(character_count) = odt_props.character_count {
|
|
539
|
-
metadata_map.insert(
|
|
540
|
-
"character_count".to_string(),
|
|
541
|
-
serde_json::Value::Number(character_count.into()),
|
|
542
|
-
);
|
|
543
|
-
}
|
|
544
|
-
if let Some(paragraph_count) = odt_props.paragraph_count {
|
|
545
|
-
metadata_map.insert(
|
|
546
|
-
"paragraph_count".to_string(),
|
|
547
|
-
serde_json::Value::Number(paragraph_count.into()),
|
|
548
|
-
);
|
|
549
|
-
}
|
|
550
|
-
if let Some(table_count) = odt_props.table_count {
|
|
551
|
-
metadata_map.insert("table_count".to_string(), serde_json::Value::Number(table_count.into()));
|
|
552
|
-
}
|
|
553
|
-
if let Some(image_count) = odt_props.image_count {
|
|
554
|
-
metadata_map.insert("image_count".to_string(), serde_json::Value::Number(image_count.into()));
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
Ok(ExtractionResult {
|
|
559
|
-
content: text,
|
|
560
|
-
mime_type: mime_type.to_string(),
|
|
561
|
-
metadata: Metadata {
|
|
562
|
-
additional: metadata_map,
|
|
563
|
-
..Default::default()
|
|
564
|
-
},
|
|
565
|
-
pages: None,
|
|
566
|
-
tables,
|
|
567
|
-
detected_languages: None,
|
|
568
|
-
chunks: None,
|
|
569
|
-
images: None,
|
|
570
|
-
})
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
fn supported_mime_types(&self) -> &[&str] {
|
|
574
|
-
&["application/vnd.oasis.opendocument.text"]
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
fn priority(&self) -> i32 {
|
|
578
|
-
60
|
|
579
|
-
}
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
#[cfg(test)]
|
|
583
|
-
mod tests {
|
|
584
|
-
use super::*;
|
|
585
|
-
|
|
586
|
-
#[tokio::test]
|
|
587
|
-
async fn test_odt_extractor_plugin_interface() {
|
|
588
|
-
let extractor = OdtExtractor::new();
|
|
589
|
-
assert_eq!(extractor.name(), "odt-extractor");
|
|
590
|
-
assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
|
|
591
|
-
assert_eq!(extractor.priority(), 60);
|
|
592
|
-
assert_eq!(extractor.supported_mime_types().len(), 1);
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
#[tokio::test]
|
|
596
|
-
async fn test_odt_extractor_supports_odt() {
|
|
597
|
-
let extractor = OdtExtractor::new();
|
|
598
|
-
assert!(
|
|
599
|
-
extractor
|
|
600
|
-
.supported_mime_types()
|
|
601
|
-
.contains(&"application/vnd.oasis.opendocument.text")
|
|
602
|
-
);
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
#[tokio::test]
|
|
606
|
-
async fn test_odt_extractor_default() {
|
|
607
|
-
let extractor = OdtExtractor;
|
|
608
|
-
assert_eq!(extractor.name(), "odt-extractor");
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
#[tokio::test]
|
|
612
|
-
async fn test_odt_extractor_initialize_shutdown() {
|
|
613
|
-
let extractor = OdtExtractor::new();
|
|
614
|
-
assert!(extractor.initialize().is_ok());
|
|
615
|
-
assert!(extractor.shutdown().is_ok());
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
#[test]
|
|
619
|
-
fn test_extract_node_text_simple() {
|
|
620
|
-
let xml = r#"<p xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0">Hello world</p>"#;
|
|
621
|
-
let doc = roxmltree::Document::parse(xml).unwrap();
|
|
622
|
-
let node = doc.root_element();
|
|
623
|
-
|
|
624
|
-
let result = extract_node_text(node);
|
|
625
|
-
assert!(result.is_some());
|
|
626
|
-
assert!(!result.unwrap().is_empty());
|
|
627
|
-
}
|
|
628
|
-
}
|
|
1
|
+
#![cfg(all(feature = "tokio-runtime", feature = "office"))]
|
|
2
|
+
|
|
3
|
+
//! ODT (OpenDocument Text) extractor using native Rust parsing.
|
|
4
|
+
//!
|
|
5
|
+
//! Supports: OpenDocument Text (.odt)
|
|
6
|
+
|
|
7
|
+
use crate::Result;
|
|
8
|
+
use crate::core::config::ExtractionConfig;
|
|
9
|
+
use crate::extraction::{cells_to_markdown, office_metadata};
|
|
10
|
+
use crate::plugins::{DocumentExtractor, Plugin};
|
|
11
|
+
use crate::types::{ExtractionResult, Metadata, Table};
|
|
12
|
+
use async_trait::async_trait;
|
|
13
|
+
use roxmltree::Document;
|
|
14
|
+
use std::io::Cursor;
|
|
15
|
+
|
|
16
|
+
/// High-performance ODT extractor using native Rust XML parsing.
|
|
17
|
+
///
|
|
18
|
+
/// This extractor provides:
|
|
19
|
+
/// - Fast text extraction via roxmltree XML parsing
|
|
20
|
+
/// - Comprehensive metadata extraction from meta.xml
|
|
21
|
+
/// - Table extraction with row and cell support
|
|
22
|
+
/// - Formatting preservation (bold, italic, strikeout)
|
|
23
|
+
/// - Support for headings, paragraphs, and special elements
|
|
24
|
+
pub struct OdtExtractor;
|
|
25
|
+
|
|
26
|
+
impl OdtExtractor {
|
|
27
|
+
/// Create a new ODT extractor.
|
|
28
|
+
pub fn new() -> Self {
|
|
29
|
+
Self
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
impl Default for OdtExtractor {
|
|
34
|
+
fn default() -> Self {
|
|
35
|
+
Self::new()
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
impl Plugin for OdtExtractor {
|
|
40
|
+
fn name(&self) -> &str {
|
|
41
|
+
"odt-extractor"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
fn version(&self) -> String {
|
|
45
|
+
env!("CARGO_PKG_VERSION").to_string()
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
fn initialize(&self) -> Result<()> {
|
|
49
|
+
Ok(())
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
fn shutdown(&self) -> Result<()> {
|
|
53
|
+
Ok(())
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fn description(&self) -> &str {
|
|
57
|
+
"Native Rust ODT (OpenDocument Text) extractor with metadata and table support"
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fn author(&self) -> &str {
|
|
61
|
+
"Kreuzberg Team"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/// Extract text from MathML formula element
|
|
66
|
+
///
|
|
67
|
+
/// # Arguments
|
|
68
|
+
/// * `math_node` - The math XML node
|
|
69
|
+
///
|
|
70
|
+
/// # Returns
|
|
71
|
+
/// * `Option<String>` - The extracted formula text
|
|
72
|
+
fn extract_mathml_text(math_node: roxmltree::Node) -> Option<String> {
|
|
73
|
+
for node in math_node.descendants() {
|
|
74
|
+
if node.tag_name().name() == "annotation"
|
|
75
|
+
&& let Some(encoding) = node.attribute("encoding")
|
|
76
|
+
&& encoding.contains("StarMath")
|
|
77
|
+
&& let Some(text) = node.text()
|
|
78
|
+
{
|
|
79
|
+
return Some(text.to_string());
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
let mut formula_parts = Vec::new();
|
|
84
|
+
for node in math_node.descendants() {
|
|
85
|
+
match node.tag_name().name() {
|
|
86
|
+
"mi" | "mo" | "mn" | "ms" | "mtext" => {
|
|
87
|
+
if let Some(text) = node.text() {
|
|
88
|
+
formula_parts.push(text.to_string());
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
_ => {}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if !formula_parts.is_empty() {
|
|
96
|
+
Some(formula_parts.join(" "))
|
|
97
|
+
} else {
|
|
98
|
+
None
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/// Extract text from embedded formula objects
|
|
103
|
+
///
|
|
104
|
+
/// # Arguments
|
|
105
|
+
/// * `archive` - ZIP archive containing the ODT document
|
|
106
|
+
///
|
|
107
|
+
/// # Returns
|
|
108
|
+
/// * `String` - Extracted formula content from embedded objects
|
|
109
|
+
fn extract_embedded_formulas(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<String> {
|
|
110
|
+
use std::io::Read;
|
|
111
|
+
let mut formula_parts = Vec::new();
|
|
112
|
+
|
|
113
|
+
let file_names: Vec<String> = archive.file_names().map(|s| s.to_string()).collect();
|
|
114
|
+
|
|
115
|
+
for file_name in file_names {
|
|
116
|
+
if file_name.contains("Object")
|
|
117
|
+
&& file_name.ends_with("content.xml")
|
|
118
|
+
&& let Ok(mut file) = archive.by_name(&file_name)
|
|
119
|
+
{
|
|
120
|
+
let mut xml_content = String::new();
|
|
121
|
+
if file.read_to_string(&mut xml_content).is_ok()
|
|
122
|
+
&& let Ok(doc) = Document::parse(&xml_content)
|
|
123
|
+
{
|
|
124
|
+
let root = doc.root_element();
|
|
125
|
+
|
|
126
|
+
if root.tag_name().name() == "math" {
|
|
127
|
+
if let Some(formula_text) = extract_mathml_text(root) {
|
|
128
|
+
formula_parts.push(formula_text);
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
for node in root.descendants() {
|
|
132
|
+
if node.tag_name().name() == "math"
|
|
133
|
+
&& let Some(formula_text) = extract_mathml_text(node)
|
|
134
|
+
{
|
|
135
|
+
formula_parts.push(formula_text);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
Ok(formula_parts.join("\n"))
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/// Extract text content from ODT content.xml
|
|
147
|
+
///
|
|
148
|
+
/// # Arguments
|
|
149
|
+
/// * `archive` - ZIP archive containing the ODT document
|
|
150
|
+
///
|
|
151
|
+
/// # Returns
|
|
152
|
+
/// * `String` - Extracted text content
|
|
153
|
+
fn extract_content_text(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<String> {
|
|
154
|
+
let mut xml_content = String::new();
|
|
155
|
+
|
|
156
|
+
match archive.by_name("content.xml") {
|
|
157
|
+
Ok(mut file) => {
|
|
158
|
+
use std::io::Read;
|
|
159
|
+
file.read_to_string(&mut xml_content)
|
|
160
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to read content.xml: {}", e)))?;
|
|
161
|
+
}
|
|
162
|
+
Err(_) => {
|
|
163
|
+
return Ok(String::new());
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
let doc = Document::parse(&xml_content)
|
|
168
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse content.xml: {}", e)))?;
|
|
169
|
+
|
|
170
|
+
let root = doc.root_element();
|
|
171
|
+
|
|
172
|
+
let mut text_parts: Vec<String> = Vec::new();
|
|
173
|
+
|
|
174
|
+
for body_child in root.children() {
|
|
175
|
+
if body_child.tag_name().name() == "body" {
|
|
176
|
+
for text_elem in body_child.children() {
|
|
177
|
+
if text_elem.tag_name().name() == "text" {
|
|
178
|
+
process_document_elements(text_elem, &mut text_parts);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
Ok(text_parts.join("\n").trim().to_string())
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/// Helper function to process document elements (paragraphs, headings, tables)
|
|
188
|
+
/// Only processes direct children, avoiding nested content like table cells
|
|
189
|
+
fn process_document_elements(parent: roxmltree::Node, text_parts: &mut Vec<String>) {
|
|
190
|
+
for node in parent.children() {
|
|
191
|
+
match node.tag_name().name() {
|
|
192
|
+
"h" => {
|
|
193
|
+
if let Some(text) = extract_node_text(node)
|
|
194
|
+
&& !text.trim().is_empty()
|
|
195
|
+
{
|
|
196
|
+
text_parts.push(format!("# {}", text.trim()));
|
|
197
|
+
text_parts.push(String::new());
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
"p" => {
|
|
201
|
+
if let Some(text) = extract_node_text(node)
|
|
202
|
+
&& !text.trim().is_empty()
|
|
203
|
+
{
|
|
204
|
+
text_parts.push(text.trim().to_string());
|
|
205
|
+
text_parts.push(String::new());
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
"table" => {
|
|
209
|
+
if let Some(table_text) = extract_table_text(node) {
|
|
210
|
+
text_parts.push(table_text);
|
|
211
|
+
text_parts.push(String::new());
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
_ => {}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/// Extract text from a single XML node, handling spans and formatting
|
|
220
|
+
///
|
|
221
|
+
/// # Arguments
|
|
222
|
+
/// * `node` - The XML node to extract text from
|
|
223
|
+
///
|
|
224
|
+
/// # Returns
|
|
225
|
+
/// * `Option<String>` - The extracted text with formatting preserved
|
|
226
|
+
fn extract_node_text(node: roxmltree::Node) -> Option<String> {
|
|
227
|
+
let mut text_parts = Vec::new();
|
|
228
|
+
|
|
229
|
+
for child in node.children() {
|
|
230
|
+
match child.tag_name().name() {
|
|
231
|
+
"span" => {
|
|
232
|
+
if let Some(text) = child.text() {
|
|
233
|
+
text_parts.push(text.to_string());
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
"tab" => {
|
|
237
|
+
text_parts.push("\t".to_string());
|
|
238
|
+
}
|
|
239
|
+
"line-break" => {
|
|
240
|
+
text_parts.push("\n".to_string());
|
|
241
|
+
}
|
|
242
|
+
_ => {
|
|
243
|
+
if let Some(text) = child.text() {
|
|
244
|
+
text_parts.push(text.to_string());
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if text_parts.is_empty() {
|
|
251
|
+
node.text().map(|s| s.to_string())
|
|
252
|
+
} else {
|
|
253
|
+
Some(text_parts.join(""))
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/// Extract table content as text with markdown formatting
|
|
258
|
+
///
|
|
259
|
+
/// # Arguments
|
|
260
|
+
/// * `table_node` - The table XML node
|
|
261
|
+
///
|
|
262
|
+
/// # Returns
|
|
263
|
+
/// * `Option<String>` - Markdown formatted table
|
|
264
|
+
fn extract_table_text(table_node: roxmltree::Node) -> Option<String> {
|
|
265
|
+
let mut rows = Vec::new();
|
|
266
|
+
let mut max_cols = 0;
|
|
267
|
+
|
|
268
|
+
for row_node in table_node.children() {
|
|
269
|
+
if row_node.tag_name().name() == "table-row" {
|
|
270
|
+
let mut row_cells = Vec::new();
|
|
271
|
+
|
|
272
|
+
for cell_node in row_node.children() {
|
|
273
|
+
if cell_node.tag_name().name() == "table-cell" {
|
|
274
|
+
let cell_text = extract_node_text(cell_node).unwrap_or_default();
|
|
275
|
+
row_cells.push(cell_text.trim().to_string());
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if !row_cells.is_empty() {
|
|
280
|
+
max_cols = max_cols.max(row_cells.len());
|
|
281
|
+
rows.push(row_cells);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if rows.is_empty() {
|
|
287
|
+
return None;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
for row in &mut rows {
|
|
291
|
+
while row.len() < max_cols {
|
|
292
|
+
row.push(String::new());
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
let mut markdown = String::new();
|
|
297
|
+
|
|
298
|
+
if !rows.is_empty() {
|
|
299
|
+
markdown.push('|');
|
|
300
|
+
for cell in &rows[0] {
|
|
301
|
+
markdown.push(' ');
|
|
302
|
+
markdown.push_str(cell);
|
|
303
|
+
markdown.push_str(" |");
|
|
304
|
+
}
|
|
305
|
+
markdown.push('\n');
|
|
306
|
+
|
|
307
|
+
markdown.push('|');
|
|
308
|
+
for _ in 0..rows[0].len() {
|
|
309
|
+
markdown.push_str(" --- |");
|
|
310
|
+
}
|
|
311
|
+
markdown.push('\n');
|
|
312
|
+
|
|
313
|
+
for row in rows.iter().skip(1) {
|
|
314
|
+
markdown.push('|');
|
|
315
|
+
for cell in row {
|
|
316
|
+
markdown.push(' ');
|
|
317
|
+
markdown.push_str(cell);
|
|
318
|
+
markdown.push_str(" |");
|
|
319
|
+
}
|
|
320
|
+
markdown.push('\n');
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
Some(markdown)
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/// Extract tables from ODT content.xml
|
|
328
|
+
///
|
|
329
|
+
/// # Arguments
|
|
330
|
+
/// * `archive` - ZIP archive containing the ODT document
|
|
331
|
+
///
|
|
332
|
+
/// # Returns
|
|
333
|
+
/// * `Result<Vec<Table>>` - Extracted tables
|
|
334
|
+
fn extract_tables(archive: &mut zip::ZipArchive<Cursor<Vec<u8>>>) -> crate::error::Result<Vec<Table>> {
|
|
335
|
+
let mut xml_content = String::new();
|
|
336
|
+
|
|
337
|
+
match archive.by_name("content.xml") {
|
|
338
|
+
Ok(mut file) => {
|
|
339
|
+
use std::io::Read;
|
|
340
|
+
file.read_to_string(&mut xml_content)
|
|
341
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to read content.xml: {}", e)))?;
|
|
342
|
+
}
|
|
343
|
+
Err(_) => {
|
|
344
|
+
return Ok(Vec::new());
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
let doc = Document::parse(&xml_content)
|
|
349
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to parse content.xml: {}", e)))?;
|
|
350
|
+
|
|
351
|
+
let root = doc.root_element();
|
|
352
|
+
let mut tables = Vec::new();
|
|
353
|
+
let mut table_index = 0;
|
|
354
|
+
|
|
355
|
+
for node in root.descendants() {
|
|
356
|
+
if node.tag_name().name() == "table"
|
|
357
|
+
&& let Some(table) = parse_odt_table(node, table_index)
|
|
358
|
+
{
|
|
359
|
+
tables.push(table);
|
|
360
|
+
table_index += 1;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
Ok(tables)
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/// Parse a single ODT table element into a Table struct
|
|
368
|
+
///
|
|
369
|
+
/// # Arguments
|
|
370
|
+
/// * `table_node` - The table XML node
|
|
371
|
+
/// * `table_index` - Index of the table in the document
|
|
372
|
+
///
|
|
373
|
+
/// # Returns
|
|
374
|
+
/// * `Option<Table>` - Parsed table
|
|
375
|
+
fn parse_odt_table(table_node: roxmltree::Node, table_index: usize) -> Option<Table> {
|
|
376
|
+
let mut cells: Vec<Vec<String>> = Vec::new();
|
|
377
|
+
|
|
378
|
+
for row_node in table_node.children() {
|
|
379
|
+
if row_node.tag_name().name() == "table-row" {
|
|
380
|
+
let mut row_cells = Vec::new();
|
|
381
|
+
|
|
382
|
+
for cell_node in row_node.children() {
|
|
383
|
+
if cell_node.tag_name().name() == "table-cell" {
|
|
384
|
+
let cell_text = extract_node_text(cell_node).unwrap_or_default();
|
|
385
|
+
row_cells.push(cell_text.trim().to_string());
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if !row_cells.is_empty() {
|
|
390
|
+
cells.push(row_cells);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
if cells.is_empty() {
|
|
396
|
+
return None;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
let markdown = cells_to_markdown(&cells);
|
|
400
|
+
|
|
401
|
+
Some(Table {
|
|
402
|
+
cells,
|
|
403
|
+
markdown,
|
|
404
|
+
page_number: table_index + 1,
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
#[async_trait]
|
|
409
|
+
impl DocumentExtractor for OdtExtractor {
|
|
410
|
+
#[cfg_attr(
|
|
411
|
+
feature = "otel",
|
|
412
|
+
tracing::instrument(
|
|
413
|
+
skip(self, content, _config),
|
|
414
|
+
fields(
|
|
415
|
+
extractor.name = self.name(),
|
|
416
|
+
content.size_bytes = content.len(),
|
|
417
|
+
)
|
|
418
|
+
)
|
|
419
|
+
)]
|
|
420
|
+
async fn extract_bytes(
|
|
421
|
+
&self,
|
|
422
|
+
content: &[u8],
|
|
423
|
+
mime_type: &str,
|
|
424
|
+
_config: &ExtractionConfig,
|
|
425
|
+
) -> Result<ExtractionResult> {
|
|
426
|
+
let content_owned = content.to_vec();
|
|
427
|
+
|
|
428
|
+
let (text, tables) = if crate::core::batch_mode::is_batch_mode() {
|
|
429
|
+
let content_for_task = content_owned.clone();
|
|
430
|
+
let span = tracing::Span::current();
|
|
431
|
+
tokio::task::spawn_blocking(move || -> crate::error::Result<(String, Vec<Table>)> {
|
|
432
|
+
let _guard = span.entered();
|
|
433
|
+
|
|
434
|
+
let cursor = Cursor::new(content_for_task);
|
|
435
|
+
let mut archive = zip::ZipArchive::new(cursor)
|
|
436
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
|
|
437
|
+
|
|
438
|
+
let text = extract_content_text(&mut archive)?;
|
|
439
|
+
let tables = extract_tables(&mut archive)?;
|
|
440
|
+
let embedded_formulas = extract_embedded_formulas(&mut archive)?;
|
|
441
|
+
|
|
442
|
+
let combined_text = if !embedded_formulas.is_empty() {
|
|
443
|
+
if !text.is_empty() {
|
|
444
|
+
format!("{}\n{}", text, embedded_formulas)
|
|
445
|
+
} else {
|
|
446
|
+
embedded_formulas
|
|
447
|
+
}
|
|
448
|
+
} else {
|
|
449
|
+
text
|
|
450
|
+
};
|
|
451
|
+
|
|
452
|
+
Ok((combined_text, tables))
|
|
453
|
+
})
|
|
454
|
+
.await
|
|
455
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("ODT extraction task failed: {}", e)))??
|
|
456
|
+
} else {
|
|
457
|
+
let cursor = Cursor::new(content_owned.clone());
|
|
458
|
+
let mut archive = zip::ZipArchive::new(cursor)
|
|
459
|
+
.map_err(|e| crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
|
|
460
|
+
|
|
461
|
+
let text = extract_content_text(&mut archive)?;
|
|
462
|
+
let tables = extract_tables(&mut archive)?;
|
|
463
|
+
let embedded_formulas = extract_embedded_formulas(&mut archive)?;
|
|
464
|
+
|
|
465
|
+
let combined_text = if !embedded_formulas.is_empty() {
|
|
466
|
+
if !text.is_empty() {
|
|
467
|
+
format!("{}\n{}", text, embedded_formulas)
|
|
468
|
+
} else {
|
|
469
|
+
embedded_formulas
|
|
470
|
+
}
|
|
471
|
+
} else {
|
|
472
|
+
text
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
(combined_text, tables)
|
|
476
|
+
};
|
|
477
|
+
|
|
478
|
+
let mut metadata_map = std::collections::HashMap::new();
|
|
479
|
+
|
|
480
|
+
let cursor = Cursor::new(content_owned.clone());
|
|
481
|
+
let mut archive = zip::ZipArchive::new(cursor).map_err(|e| {
|
|
482
|
+
crate::error::KreuzbergError::parsing(format!("Failed to open ZIP archive for metadata: {}", e))
|
|
483
|
+
})?;
|
|
484
|
+
|
|
485
|
+
if let Ok(odt_props) = office_metadata::extract_odt_properties(&mut archive) {
|
|
486
|
+
if let Some(title) = odt_props.title {
|
|
487
|
+
metadata_map.insert("title".to_string(), serde_json::Value::String(title));
|
|
488
|
+
}
|
|
489
|
+
if let Some(creator) = odt_props.creator {
|
|
490
|
+
metadata_map.insert(
|
|
491
|
+
"authors".to_string(),
|
|
492
|
+
serde_json::Value::Array(vec![serde_json::Value::String(creator.clone())]),
|
|
493
|
+
);
|
|
494
|
+
metadata_map.insert("created_by".to_string(), serde_json::Value::String(creator));
|
|
495
|
+
}
|
|
496
|
+
if let Some(initial_creator) = odt_props.initial_creator {
|
|
497
|
+
metadata_map.insert(
|
|
498
|
+
"initial_creator".to_string(),
|
|
499
|
+
serde_json::Value::String(initial_creator),
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
if let Some(subject) = odt_props.subject {
|
|
503
|
+
metadata_map.insert("subject".to_string(), serde_json::Value::String(subject));
|
|
504
|
+
}
|
|
505
|
+
if let Some(keywords) = odt_props.keywords {
|
|
506
|
+
metadata_map.insert("keywords".to_string(), serde_json::Value::String(keywords));
|
|
507
|
+
}
|
|
508
|
+
if let Some(description) = odt_props.description {
|
|
509
|
+
metadata_map.insert("description".to_string(), serde_json::Value::String(description));
|
|
510
|
+
}
|
|
511
|
+
if let Some(creation_date) = odt_props.creation_date {
|
|
512
|
+
metadata_map.insert("created_at".to_string(), serde_json::Value::String(creation_date));
|
|
513
|
+
}
|
|
514
|
+
if let Some(date) = odt_props.date {
|
|
515
|
+
metadata_map.insert("modified_at".to_string(), serde_json::Value::String(date));
|
|
516
|
+
}
|
|
517
|
+
if let Some(language) = odt_props.language {
|
|
518
|
+
metadata_map.insert("language".to_string(), serde_json::Value::String(language));
|
|
519
|
+
}
|
|
520
|
+
if let Some(generator) = odt_props.generator {
|
|
521
|
+
metadata_map.insert("generator".to_string(), serde_json::Value::String(generator));
|
|
522
|
+
}
|
|
523
|
+
if let Some(editing_duration) = odt_props.editing_duration {
|
|
524
|
+
metadata_map.insert(
|
|
525
|
+
"editing_duration".to_string(),
|
|
526
|
+
serde_json::Value::String(editing_duration),
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
if let Some(editing_cycles) = odt_props.editing_cycles {
|
|
530
|
+
metadata_map.insert("editing_cycles".to_string(), serde_json::Value::String(editing_cycles));
|
|
531
|
+
}
|
|
532
|
+
if let Some(page_count) = odt_props.page_count {
|
|
533
|
+
metadata_map.insert("page_count".to_string(), serde_json::Value::Number(page_count.into()));
|
|
534
|
+
}
|
|
535
|
+
if let Some(word_count) = odt_props.word_count {
|
|
536
|
+
metadata_map.insert("word_count".to_string(), serde_json::Value::Number(word_count.into()));
|
|
537
|
+
}
|
|
538
|
+
if let Some(character_count) = odt_props.character_count {
|
|
539
|
+
metadata_map.insert(
|
|
540
|
+
"character_count".to_string(),
|
|
541
|
+
serde_json::Value::Number(character_count.into()),
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
if let Some(paragraph_count) = odt_props.paragraph_count {
|
|
545
|
+
metadata_map.insert(
|
|
546
|
+
"paragraph_count".to_string(),
|
|
547
|
+
serde_json::Value::Number(paragraph_count.into()),
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
if let Some(table_count) = odt_props.table_count {
|
|
551
|
+
metadata_map.insert("table_count".to_string(), serde_json::Value::Number(table_count.into()));
|
|
552
|
+
}
|
|
553
|
+
if let Some(image_count) = odt_props.image_count {
|
|
554
|
+
metadata_map.insert("image_count".to_string(), serde_json::Value::Number(image_count.into()));
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
Ok(ExtractionResult {
|
|
559
|
+
content: text,
|
|
560
|
+
mime_type: mime_type.to_string(),
|
|
561
|
+
metadata: Metadata {
|
|
562
|
+
additional: metadata_map,
|
|
563
|
+
..Default::default()
|
|
564
|
+
},
|
|
565
|
+
pages: None,
|
|
566
|
+
tables,
|
|
567
|
+
detected_languages: None,
|
|
568
|
+
chunks: None,
|
|
569
|
+
images: None,
|
|
570
|
+
})
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
fn supported_mime_types(&self) -> &[&str] {
|
|
574
|
+
&["application/vnd.oasis.opendocument.text"]
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
fn priority(&self) -> i32 {
|
|
578
|
+
60
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
#[cfg(test)]
|
|
583
|
+
mod tests {
|
|
584
|
+
use super::*;
|
|
585
|
+
|
|
586
|
+
#[tokio::test]
|
|
587
|
+
async fn test_odt_extractor_plugin_interface() {
|
|
588
|
+
let extractor = OdtExtractor::new();
|
|
589
|
+
assert_eq!(extractor.name(), "odt-extractor");
|
|
590
|
+
assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
|
|
591
|
+
assert_eq!(extractor.priority(), 60);
|
|
592
|
+
assert_eq!(extractor.supported_mime_types().len(), 1);
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
#[tokio::test]
|
|
596
|
+
async fn test_odt_extractor_supports_odt() {
|
|
597
|
+
let extractor = OdtExtractor::new();
|
|
598
|
+
assert!(
|
|
599
|
+
extractor
|
|
600
|
+
.supported_mime_types()
|
|
601
|
+
.contains(&"application/vnd.oasis.opendocument.text")
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
#[tokio::test]
|
|
606
|
+
async fn test_odt_extractor_default() {
|
|
607
|
+
let extractor = OdtExtractor;
|
|
608
|
+
assert_eq!(extractor.name(), "odt-extractor");
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
#[tokio::test]
|
|
612
|
+
async fn test_odt_extractor_initialize_shutdown() {
|
|
613
|
+
let extractor = OdtExtractor::new();
|
|
614
|
+
assert!(extractor.initialize().is_ok());
|
|
615
|
+
assert!(extractor.shutdown().is_ok());
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
#[test]
|
|
619
|
+
fn test_extract_node_text_simple() {
|
|
620
|
+
let xml = r#"<p xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0">Hello world</p>"#;
|
|
621
|
+
let doc = roxmltree::Document::parse(xml).unwrap();
|
|
622
|
+
let node = doc.root_element();
|
|
623
|
+
|
|
624
|
+
let result = extract_node_text(node);
|
|
625
|
+
assert!(result.is_some());
|
|
626
|
+
assert!(!result.unwrap().is_empty());
|
|
627
|
+
}
|
|
628
|
+
}
|