kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -14
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -1
- data/.rubocop.yml +538 -538
- data/Gemfile +8 -8
- data/Gemfile.lock +105 -2
- data/README.md +454 -454
- data/Rakefile +33 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -341
- data/ext/kreuzberg_rb/extconf.rb +45 -45
- data/ext/kreuzberg_rb/native/.cargo/config.toml +2 -2
- data/ext/kreuzberg_rb/native/Cargo.lock +6940 -6941
- data/ext/kreuzberg_rb/native/Cargo.toml +54 -54
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -15
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +3158 -3158
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +214 -214
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +81 -81
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +724 -724
- data/lib/kreuzberg/error_context.rb +80 -80
- data/lib/kreuzberg/errors.rb +118 -118
- data/lib/kreuzberg/extraction_api.rb +340 -340
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +279 -279
- data/lib/kreuzberg/setup_lib_path.rb +80 -80
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +109 -109
- data/lib/{pdfium.dll → libpdfium.dylib} +0 -0
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +546 -546
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -85
- data/spec/binding/cli_spec.rb +55 -55
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -41
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/Cargo.toml +1 -1
- data/vendor/kreuzberg/Cargo.toml +5 -5
- data/vendor/kreuzberg/README.md +230 -230
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -48
- data/vendor/kreuzberg/build.rs +843 -843
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
- data/vendor/kreuzberg/src/chunking/mod.rs +1877 -1877
- data/vendor/kreuzberg/src/chunking/processor.rs +220 -220
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
- data/vendor/kreuzberg/src/core/config.rs +1080 -1080
- data/vendor/kreuzberg/src/core/extractor.rs +1156 -1156
- data/vendor/kreuzberg/src/core/io.rs +329 -329
- data/vendor/kreuzberg/src/core/mime.rs +605 -605
- data/vendor/kreuzberg/src/core/mod.rs +47 -47
- data/vendor/kreuzberg/src/core/pipeline.rs +1184 -1184
- data/vendor/kreuzberg/src/embeddings.rs +500 -500
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +398 -398
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +601 -601
- data/vendor/kreuzberg/src/extraction/image.rs +491 -491
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -574
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -213
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -81
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -284
- data/vendor/kreuzberg/src/extraction/pptx.rs +3100 -3100
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +447 -447
- data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +504 -504
- data/vendor/kreuzberg/src/extractors/docx.rs +400 -400
- data/vendor/kreuzberg/src/extractors/email.rs +157 -157
- data/vendor/kreuzberg/src/extractors/epub.rs +708 -708
- data/vendor/kreuzberg/src/extractors/excel.rs +345 -345
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -492
- data/vendor/kreuzberg/src/extractors/html.rs +407 -407
- data/vendor/kreuzberg/src/extractors/image.rs +219 -219
- data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +653 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +701 -701
- data/vendor/kreuzberg/src/extractors/mod.rs +429 -429
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +635 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -529
- data/vendor/kreuzberg/src/extractors/pdf.rs +749 -749
- data/vendor/kreuzberg/src/extractors/pptx.rs +267 -267
- data/vendor/kreuzberg/src/extractors/rst.rs +577 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +809 -809
- data/vendor/kreuzberg/src/extractors/security.rs +484 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
- data/vendor/kreuzberg/src/extractors/structured.rs +142 -142
- data/vendor/kreuzberg/src/extractors/text.rs +265 -265
- data/vendor/kreuzberg/src/extractors/typst.rs +651 -651
- data/vendor/kreuzberg/src/extractors/xml.rs +147 -147
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +275 -275
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +985 -985
- data/vendor/kreuzberg/src/language_detection/processor.rs +219 -219
- data/vendor/kreuzberg/src/lib.rs +113 -113
- data/vendor/kreuzberg/src/mcp/mod.rs +35 -35
- data/vendor/kreuzberg/src/mcp/server.rs +2076 -2076
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -863
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +452 -452
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +44 -44
- data/vendor/kreuzberg/src/pdf/bundled.rs +346 -346
- data/vendor/kreuzberg/src/pdf/error.rs +130 -130
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +489 -489
- data/vendor/kreuzberg/src/pdf/mod.rs +68 -68
- data/vendor/kreuzberg/src/pdf/rendering.rs +368 -368
- data/vendor/kreuzberg/src/pdf/table.rs +420 -420
- data/vendor/kreuzberg/src/pdf/text.rs +240 -240
- data/vendor/kreuzberg/src/plugins/extractor.rs +1044 -1044
- data/vendor/kreuzberg/src/plugins/mod.rs +212 -212
- data/vendor/kreuzberg/src/plugins/ocr.rs +639 -639
- data/vendor/kreuzberg/src/plugins/processor.rs +650 -650
- data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1339
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +967 -967
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +25 -25
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/quality_processor.rs +219 -219
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +1055 -1055
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +545 -545
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -556
- data/vendor/kreuzberg/tests/batch_processing.rs +318 -318
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
- data/vendor/kreuzberg/tests/concurrency_stress.rs +533 -533
- data/vendor/kreuzberg/tests/config_features.rs +612 -612
- data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -416
- data/vendor/kreuzberg/tests/core_integration.rs +510 -510
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -500
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
- data/vendor/kreuzberg/tests/email_integration.rs +327 -327
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
- data/vendor/kreuzberg/tests/error_handling.rs +402 -402
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
- data/vendor/kreuzberg/tests/format_integration.rs +164 -164
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
- data/vendor/kreuzberg/tests/image_integration.rs +255 -255
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
- data/vendor/kreuzberg/tests/mime_detection.rs +429 -429
- data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -514
- data/vendor/kreuzberg/tests/ocr_errors.rs +698 -698
- data/vendor/kreuzberg/tests/ocr_quality.rs +629 -629
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
- data/vendor/kreuzberg/tests/pdf_integration.rs +45 -45
- data/vendor/kreuzberg/tests/pdfium_linking.rs +374 -374
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1436 -1436
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -776
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -560
- data/vendor/kreuzberg/tests/plugin_system.rs +927 -927
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -587
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -775
- data/vendor/kreuzberg/tests/security_validation.rs +416 -416
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +631 -631
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -648
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/kreuzberg-ffi/Cargo.toml +1 -1
- data/vendor/kreuzberg-ffi/README.md +851 -851
- data/vendor/kreuzberg-ffi/build.rs +176 -176
- data/vendor/kreuzberg-ffi/cbindgen.toml +27 -27
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +1087 -1087
- data/vendor/kreuzberg-ffi/src/lib.rs +3616 -3616
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +247 -247
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -470
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +2 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
- data/vendor/kreuzberg-tesseract/LICENSE +22 -22
- data/vendor/kreuzberg-tesseract/README.md +399 -399
- data/vendor/kreuzberg-tesseract/build.rs +1354 -1354
- data/vendor/kreuzberg-tesseract/patches/README.md +71 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +1371 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +297 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +81 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +145 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -211
- data/vendor/rb-sys/.cargo_vcs_info.json +5 -5
- data/vendor/rb-sys/Cargo.lock +393 -393
- data/vendor/rb-sys/Cargo.toml +70 -70
- data/vendor/rb-sys/Cargo.toml.orig +57 -57
- data/vendor/rb-sys/LICENSE-APACHE +190 -190
- data/vendor/rb-sys/LICENSE-MIT +21 -21
- data/vendor/rb-sys/build/features.rs +111 -111
- data/vendor/rb-sys/build/main.rs +286 -286
- data/vendor/rb-sys/build/stable_api_config.rs +155 -155
- data/vendor/rb-sys/build/version.rs +50 -50
- data/vendor/rb-sys/readme.md +36 -36
- data/vendor/rb-sys/src/bindings.rs +21 -21
- data/vendor/rb-sys/src/hidden.rs +11 -11
- data/vendor/rb-sys/src/lib.rs +35 -35
- data/vendor/rb-sys/src/macros.rs +371 -371
- data/vendor/rb-sys/src/memory.rs +53 -53
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -38
- data/vendor/rb-sys/src/special_consts.rs +31 -31
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -179
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -257
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +324 -324
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +332 -332
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +325 -325
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +323 -323
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +339 -339
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +339 -339
- data/vendor/rb-sys/src/stable_api.rs +260 -260
- data/vendor/rb-sys/src/symbol.rs +31 -31
- data/vendor/rb-sys/src/tracking_allocator.rs +330 -330
- data/vendor/rb-sys/src/utils.rs +89 -89
- data/vendor/rb-sys/src/value_type.rs +7 -7
- metadata +73 -4
- data/vendor/kreuzberg-ffi/kreuzberg-ffi-install.pc +0 -12
|
@@ -1,551 +1,551 @@
|
|
|
1
|
-
//! HTML table parsing tests for `html-to-markdown-rs`.
|
|
2
|
-
//!
|
|
3
|
-
//! Tests to verify that `html-to-markdown-rs` handles HTML table parsing correctly.
|
|
4
|
-
//! These tests help determine if we can safely remove the `scraper` dependency
|
|
5
|
-
//! by confirming that `html-to-markdown-rs` already handles table content preservation.
|
|
6
|
-
|
|
7
|
-
#[cfg(feature = "html")]
|
|
8
|
-
mod html_table_tests {
|
|
9
|
-
use kreuzberg::extraction::html::convert_html_to_markdown;
|
|
10
|
-
|
|
11
|
-
/// Test basic table HTML to markdown conversion.
|
|
12
|
-
///
|
|
13
|
-
/// Verifies that:
|
|
14
|
-
/// - Table structure is recognized
|
|
15
|
-
/// - Header row (th) content is preserved
|
|
16
|
-
/// - Data rows (td) content is preserved
|
|
17
|
-
/// - All cell values are retained in output
|
|
18
|
-
#[test]
|
|
19
|
-
fn test_basic_table_parsing() {
|
|
20
|
-
let html = r#"
|
|
21
|
-
<table>
|
|
22
|
-
<tr>
|
|
23
|
-
<th>Name</th>
|
|
24
|
-
<th>Age</th>
|
|
25
|
-
</tr>
|
|
26
|
-
<tr>
|
|
27
|
-
<td>Alice</td>
|
|
28
|
-
<td>30</td>
|
|
29
|
-
</tr>
|
|
30
|
-
<tr>
|
|
31
|
-
<td>Bob</td>
|
|
32
|
-
<td>25</td>
|
|
33
|
-
</tr>
|
|
34
|
-
</table>
|
|
35
|
-
"#;
|
|
36
|
-
|
|
37
|
-
let result = convert_html_to_markdown(html, None);
|
|
38
|
-
assert!(result.is_ok(), "HTML to markdown conversion should succeed");
|
|
39
|
-
|
|
40
|
-
let markdown = result.unwrap();
|
|
41
|
-
|
|
42
|
-
println!("=== Basic Table Test ===");
|
|
43
|
-
println!("Input HTML:\n{}", html);
|
|
44
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
45
|
-
println!("========================\n");
|
|
46
|
-
|
|
47
|
-
assert!(markdown.contains("Name"), "Should contain header 'Name'");
|
|
48
|
-
assert!(markdown.contains("Age"), "Should contain header 'Age'");
|
|
49
|
-
|
|
50
|
-
assert!(markdown.contains("Alice"), "Should contain cell 'Alice'");
|
|
51
|
-
assert!(markdown.contains("Bob"), "Should contain cell 'Bob'");
|
|
52
|
-
assert!(markdown.contains("30"), "Should contain cell '30'");
|
|
53
|
-
assert!(markdown.contains("25"), "Should contain cell '25'");
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/// Test markdown table format output.
|
|
57
|
-
///
|
|
58
|
-
/// Verifies that the library outputs proper markdown table syntax
|
|
59
|
-
/// with pipe separators and alignment markers.
|
|
60
|
-
#[test]
|
|
61
|
-
fn test_markdown_table_format() {
|
|
62
|
-
let html = r#"
|
|
63
|
-
<table>
|
|
64
|
-
<thead>
|
|
65
|
-
<tr>
|
|
66
|
-
<th>Column 1</th>
|
|
67
|
-
<th>Column 2</th>
|
|
68
|
-
</tr>
|
|
69
|
-
</thead>
|
|
70
|
-
<tbody>
|
|
71
|
-
<tr>
|
|
72
|
-
<td>Value 1</td>
|
|
73
|
-
<td>Value 2</td>
|
|
74
|
-
</tr>
|
|
75
|
-
</tbody>
|
|
76
|
-
</table>
|
|
77
|
-
"#;
|
|
78
|
-
|
|
79
|
-
let result = convert_html_to_markdown(html, None);
|
|
80
|
-
assert!(result.is_ok(), "Should convert to markdown");
|
|
81
|
-
|
|
82
|
-
let markdown = result.unwrap();
|
|
83
|
-
|
|
84
|
-
println!("=== Table Format Test ===");
|
|
85
|
-
println!("Input HTML:\n{}", html);
|
|
86
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
87
|
-
println!("==========================\n");
|
|
88
|
-
|
|
89
|
-
if markdown.contains("|") {
|
|
90
|
-
println!("✓ Table uses pipe (|) separators (standard markdown table format)");
|
|
91
|
-
assert!(
|
|
92
|
-
markdown.contains("Column 1") && markdown.contains("Column 2"),
|
|
93
|
-
"Headers should be present in pipe-separated format"
|
|
94
|
-
);
|
|
95
|
-
} else {
|
|
96
|
-
println!("✓ Table content preserved but in alternative format");
|
|
97
|
-
assert!(
|
|
98
|
-
markdown.contains("Column 1") && markdown.contains("Column 2"),
|
|
99
|
-
"Headers should still be present in output"
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
assert!(
|
|
104
|
-
markdown.contains("Value 1") && markdown.contains("Value 2"),
|
|
105
|
-
"Data should be preserved"
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/// Test complex table with nested HTML content in cells.
|
|
110
|
-
///
|
|
111
|
-
/// Verifies that:
|
|
112
|
-
/// - Bold text (strong/b) in cells is handled
|
|
113
|
-
/// - Italic text (em/i) in cells is handled
|
|
114
|
-
/// - Links in cells are handled
|
|
115
|
-
/// - Nested formatting doesn't break table structure
|
|
116
|
-
#[test]
|
|
117
|
-
fn test_complex_table_with_formatting() {
|
|
118
|
-
let html = r#"
|
|
119
|
-
<table>
|
|
120
|
-
<tr>
|
|
121
|
-
<th>Feature</th>
|
|
122
|
-
<th>Status</th>
|
|
123
|
-
<th>Link</th>
|
|
124
|
-
</tr>
|
|
125
|
-
<tr>
|
|
126
|
-
<td>Headers</td>
|
|
127
|
-
<td><strong>Working</strong></td>
|
|
128
|
-
<td><a href="https://example.com">docs</a></td>
|
|
129
|
-
</tr>
|
|
130
|
-
<tr>
|
|
131
|
-
<td>Data cells</td>
|
|
132
|
-
<td><em>Implemented</em></td>
|
|
133
|
-
<td><a href="https://test.com">test</a></td>
|
|
134
|
-
</tr>
|
|
135
|
-
<tr>
|
|
136
|
-
<td><strong>Bold Cell</strong></td>
|
|
137
|
-
<td><em>Italic Cell</em></td>
|
|
138
|
-
<td><strong><em>Both</em></strong></td>
|
|
139
|
-
</tr>
|
|
140
|
-
</table>
|
|
141
|
-
"#;
|
|
142
|
-
|
|
143
|
-
let result = convert_html_to_markdown(html, None);
|
|
144
|
-
assert!(result.is_ok(), "Should convert complex table");
|
|
145
|
-
|
|
146
|
-
let markdown = result.unwrap();
|
|
147
|
-
|
|
148
|
-
println!("=== Complex Table Test ===");
|
|
149
|
-
println!("Input HTML:\n{}", html);
|
|
150
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
151
|
-
println!("===========================\n");
|
|
152
|
-
|
|
153
|
-
assert!(markdown.contains("Feature"), "Should preserve 'Feature' header");
|
|
154
|
-
assert!(markdown.contains("Status"), "Should preserve 'Status' header");
|
|
155
|
-
assert!(markdown.contains("Link"), "Should preserve 'Link' header");
|
|
156
|
-
|
|
157
|
-
assert!(markdown.contains("Headers"), "Should preserve 'Headers' cell");
|
|
158
|
-
assert!(markdown.contains("Data cells"), "Should preserve 'Data cells' cell");
|
|
159
|
-
|
|
160
|
-
assert!(
|
|
161
|
-
markdown.contains("Working"),
|
|
162
|
-
"Should preserve 'Working' (from strong tag)"
|
|
163
|
-
);
|
|
164
|
-
assert!(
|
|
165
|
-
markdown.contains("Implemented"),
|
|
166
|
-
"Should preserve 'Implemented' (from em tag)"
|
|
167
|
-
);
|
|
168
|
-
|
|
169
|
-
assert!(
|
|
170
|
-
markdown.contains("docs") || markdown.contains("example.com"),
|
|
171
|
-
"Should preserve link content or URL"
|
|
172
|
-
);
|
|
173
|
-
|
|
174
|
-
println!("✓ All content preserved in complex table");
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/// Test table with colspan and rowspan attributes.
|
|
178
|
-
///
|
|
179
|
-
/// Verifies how the library handles merged cells.
|
|
180
|
-
#[test]
|
|
181
|
-
fn test_table_with_merged_cells() {
|
|
182
|
-
let html = r#"
|
|
183
|
-
<table>
|
|
184
|
-
<tr>
|
|
185
|
-
<th colspan="2">Merged Header</th>
|
|
186
|
-
</tr>
|
|
187
|
-
<tr>
|
|
188
|
-
<td>Cell 1</td>
|
|
189
|
-
<td>Cell 2</td>
|
|
190
|
-
</tr>
|
|
191
|
-
</table>
|
|
192
|
-
"#;
|
|
193
|
-
|
|
194
|
-
let result = convert_html_to_markdown(html, None);
|
|
195
|
-
assert!(result.is_ok(), "Should handle merged cell table");
|
|
196
|
-
|
|
197
|
-
let markdown = result.unwrap();
|
|
198
|
-
|
|
199
|
-
println!("=== Merged Cells Test ===");
|
|
200
|
-
println!("Input HTML:\n{}", html);
|
|
201
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
202
|
-
println!("==========================\n");
|
|
203
|
-
|
|
204
|
-
assert!(
|
|
205
|
-
markdown.contains("Merged Header"),
|
|
206
|
-
"Should preserve merged header content"
|
|
207
|
-
);
|
|
208
|
-
assert!(
|
|
209
|
-
markdown.contains("Cell 1") && markdown.contains("Cell 2"),
|
|
210
|
-
"Should preserve all cell content"
|
|
211
|
-
);
|
|
212
|
-
|
|
213
|
-
println!("✓ Merged cell content preserved");
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
/// Test multiple tables in same HTML document.
|
|
217
|
-
///
|
|
218
|
-
/// Verifies that the library can handle multiple tables
|
|
219
|
-
/// without losing data or mixing them up.
|
|
220
|
-
#[test]
|
|
221
|
-
fn test_multiple_tables() {
|
|
222
|
-
let html = r#"
|
|
223
|
-
<h2>First Table</h2>
|
|
224
|
-
<table>
|
|
225
|
-
<tr>
|
|
226
|
-
<th>A</th>
|
|
227
|
-
<th>B</th>
|
|
228
|
-
</tr>
|
|
229
|
-
<tr>
|
|
230
|
-
<td>1</td>
|
|
231
|
-
<td>2</td>
|
|
232
|
-
</tr>
|
|
233
|
-
</table>
|
|
234
|
-
|
|
235
|
-
<h2>Second Table</h2>
|
|
236
|
-
<table>
|
|
237
|
-
<tr>
|
|
238
|
-
<th>X</th>
|
|
239
|
-
<th>Y</th>
|
|
240
|
-
</tr>
|
|
241
|
-
<tr>
|
|
242
|
-
<td>10</td>
|
|
243
|
-
<td>20</td>
|
|
244
|
-
</tr>
|
|
245
|
-
</table>
|
|
246
|
-
"#;
|
|
247
|
-
|
|
248
|
-
let result = convert_html_to_markdown(html, None);
|
|
249
|
-
assert!(result.is_ok(), "Should handle multiple tables");
|
|
250
|
-
|
|
251
|
-
let markdown = result.unwrap();
|
|
252
|
-
|
|
253
|
-
println!("=== Multiple Tables Test ===");
|
|
254
|
-
println!("Input HTML:\n{}", html);
|
|
255
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
256
|
-
println!("==============================\n");
|
|
257
|
-
|
|
258
|
-
assert!(markdown.contains("First Table"), "Should preserve first table heading");
|
|
259
|
-
assert!(
|
|
260
|
-
markdown.contains("Second Table"),
|
|
261
|
-
"Should preserve second table heading"
|
|
262
|
-
);
|
|
263
|
-
assert!(
|
|
264
|
-
markdown.contains("A") && markdown.contains("B"),
|
|
265
|
-
"Should preserve first table headers"
|
|
266
|
-
);
|
|
267
|
-
assert!(
|
|
268
|
-
markdown.contains("X") && markdown.contains("Y"),
|
|
269
|
-
"Should preserve second table headers"
|
|
270
|
-
);
|
|
271
|
-
assert!(
|
|
272
|
-
markdown.contains("1") && markdown.contains("2"),
|
|
273
|
-
"Should preserve first table data"
|
|
274
|
-
);
|
|
275
|
-
assert!(
|
|
276
|
-
markdown.contains("10") && markdown.contains("20"),
|
|
277
|
-
"Should preserve second table data"
|
|
278
|
-
);
|
|
279
|
-
|
|
280
|
-
println!("✓ Multiple tables handled correctly");
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
/// Test table with th in data rows (mixed headers and data).
|
|
284
|
-
///
|
|
285
|
-
/// Some HTML tables use th elements in tbody, not just thead.
|
|
286
|
-
#[test]
|
|
287
|
-
fn test_table_with_mixed_header_cells() {
|
|
288
|
-
let html = r#"
|
|
289
|
-
<table>
|
|
290
|
-
<tr>
|
|
291
|
-
<th>Row Header</th>
|
|
292
|
-
<td>Data 1</td>
|
|
293
|
-
<td>Data 2</td>
|
|
294
|
-
</tr>
|
|
295
|
-
<tr>
|
|
296
|
-
<th>Row Header 2</th>
|
|
297
|
-
<td>Data 3</td>
|
|
298
|
-
<td>Data 4</td>
|
|
299
|
-
</tr>
|
|
300
|
-
</table>
|
|
301
|
-
"#;
|
|
302
|
-
|
|
303
|
-
let result = convert_html_to_markdown(html, None);
|
|
304
|
-
assert!(result.is_ok(), "Should handle mixed header cells");
|
|
305
|
-
|
|
306
|
-
let markdown = result.unwrap();
|
|
307
|
-
|
|
308
|
-
println!("=== Mixed Header Cells Test ===");
|
|
309
|
-
println!("Input HTML:\n{}", html);
|
|
310
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
311
|
-
println!("=================================\n");
|
|
312
|
-
|
|
313
|
-
assert!(markdown.contains("Row Header"), "Should preserve first row header");
|
|
314
|
-
assert!(markdown.contains("Row Header 2"), "Should preserve second row header");
|
|
315
|
-
assert!(
|
|
316
|
-
markdown.contains("Data 1")
|
|
317
|
-
&& markdown.contains("Data 2")
|
|
318
|
-
&& markdown.contains("Data 3")
|
|
319
|
-
&& markdown.contains("Data 4"),
|
|
320
|
-
"Should preserve all data cells"
|
|
321
|
-
);
|
|
322
|
-
|
|
323
|
-
println!("✓ Mixed header cells preserved");
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
/// Test table with caption and other structural elements.
|
|
327
|
-
///
|
|
328
|
-
/// Verifies that additional table structure elements are handled.
|
|
329
|
-
#[test]
|
|
330
|
-
fn test_table_with_caption() {
|
|
331
|
-
let html = r#"
|
|
332
|
-
<table>
|
|
333
|
-
<caption>Sales Report 2024</caption>
|
|
334
|
-
<tr>
|
|
335
|
-
<th>Product</th>
|
|
336
|
-
<th>Sales</th>
|
|
337
|
-
</tr>
|
|
338
|
-
<tr>
|
|
339
|
-
<td>Widget A</td>
|
|
340
|
-
<td>$1,000</td>
|
|
341
|
-
</tr>
|
|
342
|
-
<tr>
|
|
343
|
-
<td>Widget B</td>
|
|
344
|
-
<td>$2,500</td>
|
|
345
|
-
</tr>
|
|
346
|
-
</table>
|
|
347
|
-
"#;
|
|
348
|
-
|
|
349
|
-
let result = convert_html_to_markdown(html, None);
|
|
350
|
-
assert!(result.is_ok(), "Should handle table with caption");
|
|
351
|
-
|
|
352
|
-
let markdown = result.unwrap();
|
|
353
|
-
|
|
354
|
-
println!("=== Table with Caption Test ===");
|
|
355
|
-
println!("Input HTML:\n{}", html);
|
|
356
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
357
|
-
println!("=================================\n");
|
|
358
|
-
|
|
359
|
-
if markdown.contains("Sales Report 2024") {
|
|
360
|
-
println!("✓ Caption is preserved in output");
|
|
361
|
-
} else {
|
|
362
|
-
println!("✓ Caption may be handled separately but content is present");
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
assert!(
|
|
366
|
-
markdown.contains("Product") && markdown.contains("Sales"),
|
|
367
|
-
"Should preserve headers"
|
|
368
|
-
);
|
|
369
|
-
assert!(
|
|
370
|
-
markdown.contains("Widget A")
|
|
371
|
-
&& markdown.contains("Widget B")
|
|
372
|
-
&& markdown.contains("1,000")
|
|
373
|
-
&& markdown.contains("2,500"),
|
|
374
|
-
"Should preserve all table data"
|
|
375
|
-
);
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/// Test simple flat table data structure.
|
|
379
|
-
///
|
|
380
|
-
/// This is the most common table format and should work reliably.
|
|
381
|
-
#[test]
|
|
382
|
-
fn test_simple_flat_table() {
|
|
383
|
-
let html = r#"<table><tr><td>A</td><td>B</td></tr><tr><td>C</td><td>D</td></tr></table>"#;
|
|
384
|
-
|
|
385
|
-
let result = convert_html_to_markdown(html, None);
|
|
386
|
-
assert!(result.is_ok(), "Should handle flat table");
|
|
387
|
-
|
|
388
|
-
let markdown = result.unwrap();
|
|
389
|
-
|
|
390
|
-
println!("=== Simple Flat Table Test ===");
|
|
391
|
-
println!("Input HTML:\n{}", html);
|
|
392
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
393
|
-
println!("==============================\n");
|
|
394
|
-
|
|
395
|
-
assert!(
|
|
396
|
-
markdown.contains("A") && markdown.contains("B") && markdown.contains("C") && markdown.contains("D"),
|
|
397
|
-
"Should preserve all cells in flat table"
|
|
398
|
-
);
|
|
399
|
-
|
|
400
|
-
println!("✓ Flat table structure preserved");
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
/// Test empty table cells.
|
|
404
|
-
///
|
|
405
|
-
/// Verifies handling of tables with empty or whitespace-only cells.
|
|
406
|
-
#[test]
|
|
407
|
-
fn test_table_with_empty_cells() {
|
|
408
|
-
let html = r#"
|
|
409
|
-
<table>
|
|
410
|
-
<tr>
|
|
411
|
-
<td>Data</td>
|
|
412
|
-
<td></td>
|
|
413
|
-
</tr>
|
|
414
|
-
<tr>
|
|
415
|
-
<td> </td>
|
|
416
|
-
<td>More Data</td>
|
|
417
|
-
</tr>
|
|
418
|
-
</table>
|
|
419
|
-
"#;
|
|
420
|
-
|
|
421
|
-
let result = convert_html_to_markdown(html, None);
|
|
422
|
-
assert!(result.is_ok(), "Should handle empty cells");
|
|
423
|
-
|
|
424
|
-
let markdown = result.unwrap();
|
|
425
|
-
|
|
426
|
-
println!("=== Empty Cells Test ===");
|
|
427
|
-
println!("Input HTML:\n{}", html);
|
|
428
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
429
|
-
println!("========================\n");
|
|
430
|
-
|
|
431
|
-
assert!(markdown.contains("Data"), "Should preserve non-empty cell");
|
|
432
|
-
assert!(markdown.contains("More Data"), "Should preserve other non-empty cell");
|
|
433
|
-
|
|
434
|
-
println!("✓ Table with empty cells handled");
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
/// Test table with numeric data.
|
|
438
|
-
///
|
|
439
|
-
/// Ensures that numeric content is preserved correctly.
|
|
440
|
-
#[test]
|
|
441
|
-
fn test_table_with_numeric_data() {
|
|
442
|
-
let html = r#"
|
|
443
|
-
<table>
|
|
444
|
-
<tr>
|
|
445
|
-
<th>Value</th>
|
|
446
|
-
<th>Amount</th>
|
|
447
|
-
</tr>
|
|
448
|
-
<tr>
|
|
449
|
-
<td>123456</td>
|
|
450
|
-
<td>789.45</td>
|
|
451
|
-
</tr>
|
|
452
|
-
<tr>
|
|
453
|
-
<td>999</td>
|
|
454
|
-
<td>0.01</td>
|
|
455
|
-
</tr>
|
|
456
|
-
</table>
|
|
457
|
-
"#;
|
|
458
|
-
|
|
459
|
-
let result = convert_html_to_markdown(html, None);
|
|
460
|
-
assert!(result.is_ok(), "Should handle numeric table");
|
|
461
|
-
|
|
462
|
-
let markdown = result.unwrap();
|
|
463
|
-
|
|
464
|
-
println!("=== Numeric Data Test ===");
|
|
465
|
-
println!("Input HTML:\n{}", html);
|
|
466
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
467
|
-
println!("=========================\n");
|
|
468
|
-
|
|
469
|
-
assert!(markdown.contains("123456"), "Should preserve numeric data");
|
|
470
|
-
assert!(markdown.contains("789.45"), "Should preserve decimal numbers");
|
|
471
|
-
assert!(markdown.contains("0.01"), "Should preserve small decimals");
|
|
472
|
-
|
|
473
|
-
println!("✓ Numeric data preserved");
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
/// Test table with special characters and unicode.
|
|
477
|
-
///
|
|
478
|
-
/// Verifies handling of non-ASCII characters in table cells.
|
|
479
|
-
#[test]
|
|
480
|
-
fn test_table_with_special_characters() {
|
|
481
|
-
let html = r#"
|
|
482
|
-
<table>
|
|
483
|
-
<tr>
|
|
484
|
-
<th>Name</th>
|
|
485
|
-
<th>Description</th>
|
|
486
|
-
</tr>
|
|
487
|
-
<tr>
|
|
488
|
-
<td>Café</td>
|
|
489
|
-
<td>Résumé with accents</td>
|
|
490
|
-
</tr>
|
|
491
|
-
<tr>
|
|
492
|
-
<td>北京</td>
|
|
493
|
-
<td>Chinese characters</td>
|
|
494
|
-
</tr>
|
|
495
|
-
<tr>
|
|
496
|
-
<td>Ñoño</td>
|
|
497
|
-
<td>Spanish tilde</td>
|
|
498
|
-
</tr>
|
|
499
|
-
</table>
|
|
500
|
-
"#;
|
|
501
|
-
|
|
502
|
-
let result = convert_html_to_markdown(html, None);
|
|
503
|
-
assert!(result.is_ok(), "Should handle unicode characters");
|
|
504
|
-
|
|
505
|
-
let markdown = result.unwrap();
|
|
506
|
-
|
|
507
|
-
println!("=== Special Characters Test ===");
|
|
508
|
-
println!("Input HTML:\n{}", html);
|
|
509
|
-
println!("\nOutput Markdown:\n{}", markdown);
|
|
510
|
-
println!("=================================\n");
|
|
511
|
-
|
|
512
|
-
assert!(markdown.contains("Café"), "Should preserve accented characters");
|
|
513
|
-
assert!(markdown.contains("北京"), "Should preserve Chinese characters");
|
|
514
|
-
assert!(markdown.contains("Ñoño"), "Should preserve Spanish tilde");
|
|
515
|
-
|
|
516
|
-
println!("✓ Special characters preserved");
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
/// Summary test providing an overall assessment of html-to-markdown-rs capabilities.
|
|
521
|
-
///
|
|
522
|
-
/// Run with: cargo test --test html_table_test --features html -- --nocapture --test-threads=1
|
|
523
|
-
#[cfg(feature = "html")]
|
|
524
|
-
#[test]
|
|
525
|
-
fn html_table_support_summary() {
|
|
526
|
-
println!("\n");
|
|
527
|
-
println!("╔════════════════════════════════════════════════════════════════╗");
|
|
528
|
-
println!("║ HTML Table Parsing Support Assessment Summary ║");
|
|
529
|
-
println!("╠════════════════════════════════════════════════════════════════╣");
|
|
530
|
-
println!("║ Testing html-to-markdown-rs capabilities for table parsing ║");
|
|
531
|
-
println!("║ to determine if scraper dependency can be safely removed. ║");
|
|
532
|
-
println!("╚════════════════════════════════════════════════════════════════╝");
|
|
533
|
-
println!();
|
|
534
|
-
println!("Test Results:");
|
|
535
|
-
println!(" ✓ Basic table parsing with th/td elements");
|
|
536
|
-
println!(" ✓ Markdown table format validation");
|
|
537
|
-
println!(" ✓ Complex tables with nested HTML content");
|
|
538
|
-
println!(" ✓ Tables with merged cells (colspan/rowspan)");
|
|
539
|
-
println!(" ✓ Multiple tables in same document");
|
|
540
|
-
println!(" ✓ Mixed header cells within tbody");
|
|
541
|
-
println!(" ✓ Tables with caption elements");
|
|
542
|
-
println!(" ✓ Simple flat table structures");
|
|
543
|
-
println!(" ✓ Empty and whitespace-only cells");
|
|
544
|
-
println!(" ✓ Numeric data preservation");
|
|
545
|
-
println!(" ✓ Unicode and special characters");
|
|
546
|
-
println!();
|
|
547
|
-
println!("Assessment:");
|
|
548
|
-
println!(" If all tests pass: html-to-markdown-rs is sufficient");
|
|
549
|
-
println!(" If content is preserved: scraper dependency may be removable");
|
|
550
|
-
println!();
|
|
551
|
-
}
|
|
1
|
+
//! HTML table parsing tests for `html-to-markdown-rs`.
|
|
2
|
+
//!
|
|
3
|
+
//! Tests to verify that `html-to-markdown-rs` handles HTML table parsing correctly.
|
|
4
|
+
//! These tests help determine if we can safely remove the `scraper` dependency
|
|
5
|
+
//! by confirming that `html-to-markdown-rs` already handles table content preservation.
|
|
6
|
+
|
|
7
|
+
#[cfg(feature = "html")]
|
|
8
|
+
mod html_table_tests {
|
|
9
|
+
use kreuzberg::extraction::html::convert_html_to_markdown;
|
|
10
|
+
|
|
11
|
+
/// Test basic table HTML to markdown conversion.
|
|
12
|
+
///
|
|
13
|
+
/// Verifies that:
|
|
14
|
+
/// - Table structure is recognized
|
|
15
|
+
/// - Header row (th) content is preserved
|
|
16
|
+
/// - Data rows (td) content is preserved
|
|
17
|
+
/// - All cell values are retained in output
|
|
18
|
+
#[test]
|
|
19
|
+
fn test_basic_table_parsing() {
|
|
20
|
+
let html = r#"
|
|
21
|
+
<table>
|
|
22
|
+
<tr>
|
|
23
|
+
<th>Name</th>
|
|
24
|
+
<th>Age</th>
|
|
25
|
+
</tr>
|
|
26
|
+
<tr>
|
|
27
|
+
<td>Alice</td>
|
|
28
|
+
<td>30</td>
|
|
29
|
+
</tr>
|
|
30
|
+
<tr>
|
|
31
|
+
<td>Bob</td>
|
|
32
|
+
<td>25</td>
|
|
33
|
+
</tr>
|
|
34
|
+
</table>
|
|
35
|
+
"#;
|
|
36
|
+
|
|
37
|
+
let result = convert_html_to_markdown(html, None);
|
|
38
|
+
assert!(result.is_ok(), "HTML to markdown conversion should succeed");
|
|
39
|
+
|
|
40
|
+
let markdown = result.unwrap();
|
|
41
|
+
|
|
42
|
+
println!("=== Basic Table Test ===");
|
|
43
|
+
println!("Input HTML:\n{}", html);
|
|
44
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
45
|
+
println!("========================\n");
|
|
46
|
+
|
|
47
|
+
assert!(markdown.contains("Name"), "Should contain header 'Name'");
|
|
48
|
+
assert!(markdown.contains("Age"), "Should contain header 'Age'");
|
|
49
|
+
|
|
50
|
+
assert!(markdown.contains("Alice"), "Should contain cell 'Alice'");
|
|
51
|
+
assert!(markdown.contains("Bob"), "Should contain cell 'Bob'");
|
|
52
|
+
assert!(markdown.contains("30"), "Should contain cell '30'");
|
|
53
|
+
assert!(markdown.contains("25"), "Should contain cell '25'");
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// Test markdown table format output.
|
|
57
|
+
///
|
|
58
|
+
/// Verifies that the library outputs proper markdown table syntax
|
|
59
|
+
/// with pipe separators and alignment markers.
|
|
60
|
+
#[test]
|
|
61
|
+
fn test_markdown_table_format() {
|
|
62
|
+
let html = r#"
|
|
63
|
+
<table>
|
|
64
|
+
<thead>
|
|
65
|
+
<tr>
|
|
66
|
+
<th>Column 1</th>
|
|
67
|
+
<th>Column 2</th>
|
|
68
|
+
</tr>
|
|
69
|
+
</thead>
|
|
70
|
+
<tbody>
|
|
71
|
+
<tr>
|
|
72
|
+
<td>Value 1</td>
|
|
73
|
+
<td>Value 2</td>
|
|
74
|
+
</tr>
|
|
75
|
+
</tbody>
|
|
76
|
+
</table>
|
|
77
|
+
"#;
|
|
78
|
+
|
|
79
|
+
let result = convert_html_to_markdown(html, None);
|
|
80
|
+
assert!(result.is_ok(), "Should convert to markdown");
|
|
81
|
+
|
|
82
|
+
let markdown = result.unwrap();
|
|
83
|
+
|
|
84
|
+
println!("=== Table Format Test ===");
|
|
85
|
+
println!("Input HTML:\n{}", html);
|
|
86
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
87
|
+
println!("==========================\n");
|
|
88
|
+
|
|
89
|
+
if markdown.contains("|") {
|
|
90
|
+
println!("✓ Table uses pipe (|) separators (standard markdown table format)");
|
|
91
|
+
assert!(
|
|
92
|
+
markdown.contains("Column 1") && markdown.contains("Column 2"),
|
|
93
|
+
"Headers should be present in pipe-separated format"
|
|
94
|
+
);
|
|
95
|
+
} else {
|
|
96
|
+
println!("✓ Table content preserved but in alternative format");
|
|
97
|
+
assert!(
|
|
98
|
+
markdown.contains("Column 1") && markdown.contains("Column 2"),
|
|
99
|
+
"Headers should still be present in output"
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
assert!(
|
|
104
|
+
markdown.contains("Value 1") && markdown.contains("Value 2"),
|
|
105
|
+
"Data should be preserved"
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/// Test complex table with nested HTML content in cells.
|
|
110
|
+
///
|
|
111
|
+
/// Verifies that:
|
|
112
|
+
/// - Bold text (strong/b) in cells is handled
|
|
113
|
+
/// - Italic text (em/i) in cells is handled
|
|
114
|
+
/// - Links in cells are handled
|
|
115
|
+
/// - Nested formatting doesn't break table structure
|
|
116
|
+
#[test]
|
|
117
|
+
fn test_complex_table_with_formatting() {
|
|
118
|
+
let html = r#"
|
|
119
|
+
<table>
|
|
120
|
+
<tr>
|
|
121
|
+
<th>Feature</th>
|
|
122
|
+
<th>Status</th>
|
|
123
|
+
<th>Link</th>
|
|
124
|
+
</tr>
|
|
125
|
+
<tr>
|
|
126
|
+
<td>Headers</td>
|
|
127
|
+
<td><strong>Working</strong></td>
|
|
128
|
+
<td><a href="https://example.com">docs</a></td>
|
|
129
|
+
</tr>
|
|
130
|
+
<tr>
|
|
131
|
+
<td>Data cells</td>
|
|
132
|
+
<td><em>Implemented</em></td>
|
|
133
|
+
<td><a href="https://test.com">test</a></td>
|
|
134
|
+
</tr>
|
|
135
|
+
<tr>
|
|
136
|
+
<td><strong>Bold Cell</strong></td>
|
|
137
|
+
<td><em>Italic Cell</em></td>
|
|
138
|
+
<td><strong><em>Both</em></strong></td>
|
|
139
|
+
</tr>
|
|
140
|
+
</table>
|
|
141
|
+
"#;
|
|
142
|
+
|
|
143
|
+
let result = convert_html_to_markdown(html, None);
|
|
144
|
+
assert!(result.is_ok(), "Should convert complex table");
|
|
145
|
+
|
|
146
|
+
let markdown = result.unwrap();
|
|
147
|
+
|
|
148
|
+
println!("=== Complex Table Test ===");
|
|
149
|
+
println!("Input HTML:\n{}", html);
|
|
150
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
151
|
+
println!("===========================\n");
|
|
152
|
+
|
|
153
|
+
assert!(markdown.contains("Feature"), "Should preserve 'Feature' header");
|
|
154
|
+
assert!(markdown.contains("Status"), "Should preserve 'Status' header");
|
|
155
|
+
assert!(markdown.contains("Link"), "Should preserve 'Link' header");
|
|
156
|
+
|
|
157
|
+
assert!(markdown.contains("Headers"), "Should preserve 'Headers' cell");
|
|
158
|
+
assert!(markdown.contains("Data cells"), "Should preserve 'Data cells' cell");
|
|
159
|
+
|
|
160
|
+
assert!(
|
|
161
|
+
markdown.contains("Working"),
|
|
162
|
+
"Should preserve 'Working' (from strong tag)"
|
|
163
|
+
);
|
|
164
|
+
assert!(
|
|
165
|
+
markdown.contains("Implemented"),
|
|
166
|
+
"Should preserve 'Implemented' (from em tag)"
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
assert!(
|
|
170
|
+
markdown.contains("docs") || markdown.contains("example.com"),
|
|
171
|
+
"Should preserve link content or URL"
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
println!("✓ All content preserved in complex table");
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/// Test table with colspan and rowspan attributes.
|
|
178
|
+
///
|
|
179
|
+
/// Verifies how the library handles merged cells.
|
|
180
|
+
#[test]
|
|
181
|
+
fn test_table_with_merged_cells() {
|
|
182
|
+
let html = r#"
|
|
183
|
+
<table>
|
|
184
|
+
<tr>
|
|
185
|
+
<th colspan="2">Merged Header</th>
|
|
186
|
+
</tr>
|
|
187
|
+
<tr>
|
|
188
|
+
<td>Cell 1</td>
|
|
189
|
+
<td>Cell 2</td>
|
|
190
|
+
</tr>
|
|
191
|
+
</table>
|
|
192
|
+
"#;
|
|
193
|
+
|
|
194
|
+
let result = convert_html_to_markdown(html, None);
|
|
195
|
+
assert!(result.is_ok(), "Should handle merged cell table");
|
|
196
|
+
|
|
197
|
+
let markdown = result.unwrap();
|
|
198
|
+
|
|
199
|
+
println!("=== Merged Cells Test ===");
|
|
200
|
+
println!("Input HTML:\n{}", html);
|
|
201
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
202
|
+
println!("==========================\n");
|
|
203
|
+
|
|
204
|
+
assert!(
|
|
205
|
+
markdown.contains("Merged Header"),
|
|
206
|
+
"Should preserve merged header content"
|
|
207
|
+
);
|
|
208
|
+
assert!(
|
|
209
|
+
markdown.contains("Cell 1") && markdown.contains("Cell 2"),
|
|
210
|
+
"Should preserve all cell content"
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
println!("✓ Merged cell content preserved");
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/// Test multiple tables in same HTML document.
|
|
217
|
+
///
|
|
218
|
+
/// Verifies that the library can handle multiple tables
|
|
219
|
+
/// without losing data or mixing them up.
|
|
220
|
+
#[test]
|
|
221
|
+
fn test_multiple_tables() {
|
|
222
|
+
let html = r#"
|
|
223
|
+
<h2>First Table</h2>
|
|
224
|
+
<table>
|
|
225
|
+
<tr>
|
|
226
|
+
<th>A</th>
|
|
227
|
+
<th>B</th>
|
|
228
|
+
</tr>
|
|
229
|
+
<tr>
|
|
230
|
+
<td>1</td>
|
|
231
|
+
<td>2</td>
|
|
232
|
+
</tr>
|
|
233
|
+
</table>
|
|
234
|
+
|
|
235
|
+
<h2>Second Table</h2>
|
|
236
|
+
<table>
|
|
237
|
+
<tr>
|
|
238
|
+
<th>X</th>
|
|
239
|
+
<th>Y</th>
|
|
240
|
+
</tr>
|
|
241
|
+
<tr>
|
|
242
|
+
<td>10</td>
|
|
243
|
+
<td>20</td>
|
|
244
|
+
</tr>
|
|
245
|
+
</table>
|
|
246
|
+
"#;
|
|
247
|
+
|
|
248
|
+
let result = convert_html_to_markdown(html, None);
|
|
249
|
+
assert!(result.is_ok(), "Should handle multiple tables");
|
|
250
|
+
|
|
251
|
+
let markdown = result.unwrap();
|
|
252
|
+
|
|
253
|
+
println!("=== Multiple Tables Test ===");
|
|
254
|
+
println!("Input HTML:\n{}", html);
|
|
255
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
256
|
+
println!("==============================\n");
|
|
257
|
+
|
|
258
|
+
assert!(markdown.contains("First Table"), "Should preserve first table heading");
|
|
259
|
+
assert!(
|
|
260
|
+
markdown.contains("Second Table"),
|
|
261
|
+
"Should preserve second table heading"
|
|
262
|
+
);
|
|
263
|
+
assert!(
|
|
264
|
+
markdown.contains("A") && markdown.contains("B"),
|
|
265
|
+
"Should preserve first table headers"
|
|
266
|
+
);
|
|
267
|
+
assert!(
|
|
268
|
+
markdown.contains("X") && markdown.contains("Y"),
|
|
269
|
+
"Should preserve second table headers"
|
|
270
|
+
);
|
|
271
|
+
assert!(
|
|
272
|
+
markdown.contains("1") && markdown.contains("2"),
|
|
273
|
+
"Should preserve first table data"
|
|
274
|
+
);
|
|
275
|
+
assert!(
|
|
276
|
+
markdown.contains("10") && markdown.contains("20"),
|
|
277
|
+
"Should preserve second table data"
|
|
278
|
+
);
|
|
279
|
+
|
|
280
|
+
println!("✓ Multiple tables handled correctly");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/// Test table with th in data rows (mixed headers and data).
|
|
284
|
+
///
|
|
285
|
+
/// Some HTML tables use th elements in tbody, not just thead.
|
|
286
|
+
#[test]
|
|
287
|
+
fn test_table_with_mixed_header_cells() {
|
|
288
|
+
let html = r#"
|
|
289
|
+
<table>
|
|
290
|
+
<tr>
|
|
291
|
+
<th>Row Header</th>
|
|
292
|
+
<td>Data 1</td>
|
|
293
|
+
<td>Data 2</td>
|
|
294
|
+
</tr>
|
|
295
|
+
<tr>
|
|
296
|
+
<th>Row Header 2</th>
|
|
297
|
+
<td>Data 3</td>
|
|
298
|
+
<td>Data 4</td>
|
|
299
|
+
</tr>
|
|
300
|
+
</table>
|
|
301
|
+
"#;
|
|
302
|
+
|
|
303
|
+
let result = convert_html_to_markdown(html, None);
|
|
304
|
+
assert!(result.is_ok(), "Should handle mixed header cells");
|
|
305
|
+
|
|
306
|
+
let markdown = result.unwrap();
|
|
307
|
+
|
|
308
|
+
println!("=== Mixed Header Cells Test ===");
|
|
309
|
+
println!("Input HTML:\n{}", html);
|
|
310
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
311
|
+
println!("=================================\n");
|
|
312
|
+
|
|
313
|
+
assert!(markdown.contains("Row Header"), "Should preserve first row header");
|
|
314
|
+
assert!(markdown.contains("Row Header 2"), "Should preserve second row header");
|
|
315
|
+
assert!(
|
|
316
|
+
markdown.contains("Data 1")
|
|
317
|
+
&& markdown.contains("Data 2")
|
|
318
|
+
&& markdown.contains("Data 3")
|
|
319
|
+
&& markdown.contains("Data 4"),
|
|
320
|
+
"Should preserve all data cells"
|
|
321
|
+
);
|
|
322
|
+
|
|
323
|
+
println!("✓ Mixed header cells preserved");
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/// Test table with caption and other structural elements.
|
|
327
|
+
///
|
|
328
|
+
/// Verifies that additional table structure elements are handled.
|
|
329
|
+
#[test]
|
|
330
|
+
fn test_table_with_caption() {
|
|
331
|
+
let html = r#"
|
|
332
|
+
<table>
|
|
333
|
+
<caption>Sales Report 2024</caption>
|
|
334
|
+
<tr>
|
|
335
|
+
<th>Product</th>
|
|
336
|
+
<th>Sales</th>
|
|
337
|
+
</tr>
|
|
338
|
+
<tr>
|
|
339
|
+
<td>Widget A</td>
|
|
340
|
+
<td>$1,000</td>
|
|
341
|
+
</tr>
|
|
342
|
+
<tr>
|
|
343
|
+
<td>Widget B</td>
|
|
344
|
+
<td>$2,500</td>
|
|
345
|
+
</tr>
|
|
346
|
+
</table>
|
|
347
|
+
"#;
|
|
348
|
+
|
|
349
|
+
let result = convert_html_to_markdown(html, None);
|
|
350
|
+
assert!(result.is_ok(), "Should handle table with caption");
|
|
351
|
+
|
|
352
|
+
let markdown = result.unwrap();
|
|
353
|
+
|
|
354
|
+
println!("=== Table with Caption Test ===");
|
|
355
|
+
println!("Input HTML:\n{}", html);
|
|
356
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
357
|
+
println!("=================================\n");
|
|
358
|
+
|
|
359
|
+
if markdown.contains("Sales Report 2024") {
|
|
360
|
+
println!("✓ Caption is preserved in output");
|
|
361
|
+
} else {
|
|
362
|
+
println!("✓ Caption may be handled separately but content is present");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
assert!(
|
|
366
|
+
markdown.contains("Product") && markdown.contains("Sales"),
|
|
367
|
+
"Should preserve headers"
|
|
368
|
+
);
|
|
369
|
+
assert!(
|
|
370
|
+
markdown.contains("Widget A")
|
|
371
|
+
&& markdown.contains("Widget B")
|
|
372
|
+
&& markdown.contains("1,000")
|
|
373
|
+
&& markdown.contains("2,500"),
|
|
374
|
+
"Should preserve all table data"
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/// Test simple flat table data structure.
|
|
379
|
+
///
|
|
380
|
+
/// This is the most common table format and should work reliably.
|
|
381
|
+
#[test]
|
|
382
|
+
fn test_simple_flat_table() {
|
|
383
|
+
let html = r#"<table><tr><td>A</td><td>B</td></tr><tr><td>C</td><td>D</td></tr></table>"#;
|
|
384
|
+
|
|
385
|
+
let result = convert_html_to_markdown(html, None);
|
|
386
|
+
assert!(result.is_ok(), "Should handle flat table");
|
|
387
|
+
|
|
388
|
+
let markdown = result.unwrap();
|
|
389
|
+
|
|
390
|
+
println!("=== Simple Flat Table Test ===");
|
|
391
|
+
println!("Input HTML:\n{}", html);
|
|
392
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
393
|
+
println!("==============================\n");
|
|
394
|
+
|
|
395
|
+
assert!(
|
|
396
|
+
markdown.contains("A") && markdown.contains("B") && markdown.contains("C") && markdown.contains("D"),
|
|
397
|
+
"Should preserve all cells in flat table"
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
println!("✓ Flat table structure preserved");
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/// Test empty table cells.
|
|
404
|
+
///
|
|
405
|
+
/// Verifies handling of tables with empty or whitespace-only cells.
|
|
406
|
+
#[test]
|
|
407
|
+
fn test_table_with_empty_cells() {
|
|
408
|
+
let html = r#"
|
|
409
|
+
<table>
|
|
410
|
+
<tr>
|
|
411
|
+
<td>Data</td>
|
|
412
|
+
<td></td>
|
|
413
|
+
</tr>
|
|
414
|
+
<tr>
|
|
415
|
+
<td> </td>
|
|
416
|
+
<td>More Data</td>
|
|
417
|
+
</tr>
|
|
418
|
+
</table>
|
|
419
|
+
"#;
|
|
420
|
+
|
|
421
|
+
let result = convert_html_to_markdown(html, None);
|
|
422
|
+
assert!(result.is_ok(), "Should handle empty cells");
|
|
423
|
+
|
|
424
|
+
let markdown = result.unwrap();
|
|
425
|
+
|
|
426
|
+
println!("=== Empty Cells Test ===");
|
|
427
|
+
println!("Input HTML:\n{}", html);
|
|
428
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
429
|
+
println!("========================\n");
|
|
430
|
+
|
|
431
|
+
assert!(markdown.contains("Data"), "Should preserve non-empty cell");
|
|
432
|
+
assert!(markdown.contains("More Data"), "Should preserve other non-empty cell");
|
|
433
|
+
|
|
434
|
+
println!("✓ Table with empty cells handled");
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/// Test table with numeric data.
|
|
438
|
+
///
|
|
439
|
+
/// Ensures that numeric content is preserved correctly.
|
|
440
|
+
#[test]
|
|
441
|
+
fn test_table_with_numeric_data() {
|
|
442
|
+
let html = r#"
|
|
443
|
+
<table>
|
|
444
|
+
<tr>
|
|
445
|
+
<th>Value</th>
|
|
446
|
+
<th>Amount</th>
|
|
447
|
+
</tr>
|
|
448
|
+
<tr>
|
|
449
|
+
<td>123456</td>
|
|
450
|
+
<td>789.45</td>
|
|
451
|
+
</tr>
|
|
452
|
+
<tr>
|
|
453
|
+
<td>999</td>
|
|
454
|
+
<td>0.01</td>
|
|
455
|
+
</tr>
|
|
456
|
+
</table>
|
|
457
|
+
"#;
|
|
458
|
+
|
|
459
|
+
let result = convert_html_to_markdown(html, None);
|
|
460
|
+
assert!(result.is_ok(), "Should handle numeric table");
|
|
461
|
+
|
|
462
|
+
let markdown = result.unwrap();
|
|
463
|
+
|
|
464
|
+
println!("=== Numeric Data Test ===");
|
|
465
|
+
println!("Input HTML:\n{}", html);
|
|
466
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
467
|
+
println!("=========================\n");
|
|
468
|
+
|
|
469
|
+
assert!(markdown.contains("123456"), "Should preserve numeric data");
|
|
470
|
+
assert!(markdown.contains("789.45"), "Should preserve decimal numbers");
|
|
471
|
+
assert!(markdown.contains("0.01"), "Should preserve small decimals");
|
|
472
|
+
|
|
473
|
+
println!("✓ Numeric data preserved");
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/// Test table with special characters and unicode.
|
|
477
|
+
///
|
|
478
|
+
/// Verifies handling of non-ASCII characters in table cells.
|
|
479
|
+
#[test]
|
|
480
|
+
fn test_table_with_special_characters() {
|
|
481
|
+
let html = r#"
|
|
482
|
+
<table>
|
|
483
|
+
<tr>
|
|
484
|
+
<th>Name</th>
|
|
485
|
+
<th>Description</th>
|
|
486
|
+
</tr>
|
|
487
|
+
<tr>
|
|
488
|
+
<td>Café</td>
|
|
489
|
+
<td>Résumé with accents</td>
|
|
490
|
+
</tr>
|
|
491
|
+
<tr>
|
|
492
|
+
<td>北京</td>
|
|
493
|
+
<td>Chinese characters</td>
|
|
494
|
+
</tr>
|
|
495
|
+
<tr>
|
|
496
|
+
<td>Ñoño</td>
|
|
497
|
+
<td>Spanish tilde</td>
|
|
498
|
+
</tr>
|
|
499
|
+
</table>
|
|
500
|
+
"#;
|
|
501
|
+
|
|
502
|
+
let result = convert_html_to_markdown(html, None);
|
|
503
|
+
assert!(result.is_ok(), "Should handle unicode characters");
|
|
504
|
+
|
|
505
|
+
let markdown = result.unwrap();
|
|
506
|
+
|
|
507
|
+
println!("=== Special Characters Test ===");
|
|
508
|
+
println!("Input HTML:\n{}", html);
|
|
509
|
+
println!("\nOutput Markdown:\n{}", markdown);
|
|
510
|
+
println!("=================================\n");
|
|
511
|
+
|
|
512
|
+
assert!(markdown.contains("Café"), "Should preserve accented characters");
|
|
513
|
+
assert!(markdown.contains("北京"), "Should preserve Chinese characters");
|
|
514
|
+
assert!(markdown.contains("Ñoño"), "Should preserve Spanish tilde");
|
|
515
|
+
|
|
516
|
+
println!("✓ Special characters preserved");
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/// Summary test providing an overall assessment of html-to-markdown-rs capabilities.
|
|
521
|
+
///
|
|
522
|
+
/// Run with: cargo test --test html_table_test --features html -- --nocapture --test-threads=1
|
|
523
|
+
#[cfg(feature = "html")]
|
|
524
|
+
#[test]
|
|
525
|
+
fn html_table_support_summary() {
|
|
526
|
+
println!("\n");
|
|
527
|
+
println!("╔════════════════════════════════════════════════════════════════╗");
|
|
528
|
+
println!("║ HTML Table Parsing Support Assessment Summary ║");
|
|
529
|
+
println!("╠════════════════════════════════════════════════════════════════╣");
|
|
530
|
+
println!("║ Testing html-to-markdown-rs capabilities for table parsing ║");
|
|
531
|
+
println!("║ to determine if scraper dependency can be safely removed. ║");
|
|
532
|
+
println!("╚════════════════════════════════════════════════════════════════╝");
|
|
533
|
+
println!();
|
|
534
|
+
println!("Test Results:");
|
|
535
|
+
println!(" ✓ Basic table parsing with th/td elements");
|
|
536
|
+
println!(" ✓ Markdown table format validation");
|
|
537
|
+
println!(" ✓ Complex tables with nested HTML content");
|
|
538
|
+
println!(" ✓ Tables with merged cells (colspan/rowspan)");
|
|
539
|
+
println!(" ✓ Multiple tables in same document");
|
|
540
|
+
println!(" ✓ Mixed header cells within tbody");
|
|
541
|
+
println!(" ✓ Tables with caption elements");
|
|
542
|
+
println!(" ✓ Simple flat table structures");
|
|
543
|
+
println!(" ✓ Empty and whitespace-only cells");
|
|
544
|
+
println!(" ✓ Numeric data preservation");
|
|
545
|
+
println!(" ✓ Unicode and special characters");
|
|
546
|
+
println!();
|
|
547
|
+
println!("Assessment:");
|
|
548
|
+
println!(" If all tests pass: html-to-markdown-rs is sufficient");
|
|
549
|
+
println!(" If content is preserved: scraper dependency may be removable");
|
|
550
|
+
println!();
|
|
551
|
+
}
|