RubyGems - kreuzberg - Versions diffs - 4.0.0.rc2 → 4.0.0 - Mend

kreuzberg 4.0.0.rc2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (446) hide show

checksums.yaml +4 -4
data/.gitignore +14 -14
data/.rspec +3 -3
data/.rubocop.yaml +1 -1
data/.rubocop.yml +543 -538
data/Gemfile +8 -8
data/Gemfile.lock +194 -6
data/README.md +396 -426
data/Rakefile +34 -25
data/Steepfile +51 -47
data/examples/async_patterns.rb +283 -341
data/ext/kreuzberg_rb/extconf.rb +65 -45
data/ext/kreuzberg_rb/native/.cargo/config.toml +23 -0
data/ext/kreuzberg_rb/native/Cargo.lock +7619 -6535
data/ext/kreuzberg_rb/native/Cargo.toml +75 -44
data/ext/kreuzberg_rb/native/README.md +425 -425
data/ext/kreuzberg_rb/native/build.rs +15 -15
data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
data/ext/kreuzberg_rb/native/include/strings.h +20 -20
data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
data/ext/kreuzberg_rb/native/src/lib.rs +3802 -2998
data/extconf.rb +60 -28
data/kreuzberg.gemspec +199 -148
data/lib/kreuzberg/api_proxy.rb +126 -142
data/lib/kreuzberg/cache_api.rb +67 -46
data/lib/kreuzberg/cli.rb +47 -55
data/lib/kreuzberg/cli_proxy.rb +117 -127
data/lib/kreuzberg/config.rb +936 -691
data/lib/kreuzberg/error_context.rb +136 -32
data/lib/kreuzberg/errors.rb +116 -118
data/lib/kreuzberg/extraction_api.rb +313 -85
data/lib/kreuzberg/mcp_proxy.rb +177 -186
data/lib/kreuzberg/ocr_backend_protocol.rb +40 -113
data/lib/kreuzberg/post_processor_protocol.rb +15 -86
data/lib/kreuzberg/result.rb +334 -216
data/lib/kreuzberg/setup_lib_path.rb +99 -80
data/lib/kreuzberg/types.rb +170 -0
data/lib/kreuzberg/validator_protocol.rb +16 -89
data/lib/kreuzberg/version.rb +5 -5
data/lib/kreuzberg.rb +96 -103
data/lib/libpdfium.so +0 -0
data/sig/kreuzberg/internal.rbs +184 -184
data/sig/kreuzberg.rbs +561 -520
data/spec/binding/async_operations_spec.rb +473 -0
data/spec/binding/batch_operations_spec.rb +595 -0
data/spec/binding/batch_spec.rb +359 -0
data/spec/binding/cache_spec.rb +227 -227
data/spec/binding/cli_proxy_spec.rb +85 -85
data/spec/binding/cli_spec.rb +55 -55
data/spec/binding/config_result_spec.rb +377 -0
data/spec/binding/config_spec.rb +419 -345
data/spec/binding/config_validation_spec.rb +377 -283
data/spec/binding/embeddings_spec.rb +816 -0
data/spec/binding/error_handling_spec.rb +399 -213
data/spec/binding/error_recovery_spec.rb +488 -0
data/spec/binding/errors_spec.rb +66 -66
data/spec/binding/font_config_spec.rb +220 -0
data/spec/binding/images_spec.rb +738 -0
data/spec/binding/keywords_extraction_spec.rb +600 -0
data/spec/binding/metadata_types_spec.rb +1228 -0
data/spec/binding/pages_extraction_spec.rb +471 -0
data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
data/spec/binding/plugins/postprocessor_spec.rb +269 -269
data/spec/binding/plugins/validator_spec.rb +273 -274
data/spec/binding/tables_spec.rb +641 -0
data/spec/fixtures/config.toml +38 -39
data/spec/fixtures/config.yaml +41 -41
data/spec/fixtures/invalid_config.toml +3 -4
data/spec/smoke/package_spec.rb +177 -178
data/spec/spec_helper.rb +40 -42
data/spec/unit/config/chunking_config_spec.rb +213 -0
data/spec/unit/config/embedding_config_spec.rb +343 -0
data/spec/unit/config/extraction_config_spec.rb +438 -0
data/spec/unit/config/font_config_spec.rb +285 -0
data/spec/unit/config/hierarchy_config_spec.rb +314 -0
data/spec/unit/config/image_extraction_config_spec.rb +209 -0
data/spec/unit/config/image_preprocessing_config_spec.rb +249 -0
data/spec/unit/config/keyword_config_spec.rb +229 -0
data/spec/unit/config/language_detection_config_spec.rb +258 -0
data/spec/unit/config/ocr_config_spec.rb +171 -0
data/spec/unit/config/page_config_spec.rb +221 -0
data/spec/unit/config/pdf_config_spec.rb +267 -0
data/spec/unit/config/postprocessor_config_spec.rb +290 -0
data/spec/unit/config/tesseract_config_spec.rb +181 -0
data/spec/unit/config/token_reduction_config_spec.rb +251 -0
data/test/metadata_types_test.rb +959 -0
data/vendor/Cargo.toml +61 -0
data/vendor/kreuzberg/Cargo.toml +259 -204
data/vendor/kreuzberg/README.md +263 -175
data/vendor/kreuzberg/build.rs +782 -474
data/vendor/kreuzberg/examples/bench_fixes.rs +71 -0
data/vendor/kreuzberg/examples/test_pdfium_fork.rs +62 -0
data/vendor/kreuzberg/src/api/error.rs +81 -81
data/vendor/kreuzberg/src/api/handlers.rs +320 -199
data/vendor/kreuzberg/src/api/mod.rs +94 -79
data/vendor/kreuzberg/src/api/server.rs +518 -353
data/vendor/kreuzberg/src/api/types.rs +206 -170
data/vendor/kreuzberg/src/cache/mod.rs +1167 -1167
data/vendor/kreuzberg/src/chunking/mod.rs +2303 -677
data/vendor/kreuzberg/src/chunking/processor.rs +219 -0
data/vendor/kreuzberg/src/core/batch_mode.rs +95 -95
data/vendor/kreuzberg/src/core/batch_optimizations.rs +385 -0
data/vendor/kreuzberg/src/core/config.rs +1914 -1032
data/vendor/kreuzberg/src/core/config_validation.rs +949 -0
data/vendor/kreuzberg/src/core/extractor.rs +1200 -1024
data/vendor/kreuzberg/src/core/formats.rs +235 -0
data/vendor/kreuzberg/src/core/io.rs +329 -329
data/vendor/kreuzberg/src/core/mime.rs +605 -605
data/vendor/kreuzberg/src/core/mod.rs +61 -45
data/vendor/kreuzberg/src/core/pipeline.rs +1223 -984
data/vendor/kreuzberg/src/core/server_config.rs +1220 -0
data/vendor/kreuzberg/src/embeddings.rs +471 -432
data/vendor/kreuzberg/src/error.rs +431 -431
data/vendor/kreuzberg/src/extraction/archive.rs +959 -954
data/vendor/kreuzberg/src/extraction/capacity.rs +263 -0
data/vendor/kreuzberg/src/extraction/docx.rs +404 -40
data/vendor/kreuzberg/src/extraction/email.rs +855 -854
data/vendor/kreuzberg/src/extraction/excel.rs +697 -688
data/vendor/kreuzberg/src/extraction/html.rs +1830 -553
data/vendor/kreuzberg/src/extraction/image.rs +492 -368
data/vendor/kreuzberg/src/extraction/libreoffice.rs +574 -563
data/vendor/kreuzberg/src/extraction/markdown.rs +216 -213
data/vendor/kreuzberg/src/extraction/mod.rs +93 -81
data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -130
data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +284 -287
data/vendor/kreuzberg/src/extraction/pptx.rs +3102 -3000
data/vendor/kreuzberg/src/extraction/structured.rs +491 -490
data/vendor/kreuzberg/src/extraction/table.rs +329 -328
data/vendor/kreuzberg/src/extraction/text.rs +277 -269
data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
data/vendor/kreuzberg/src/extractors/archive.rs +447 -446
data/vendor/kreuzberg/src/extractors/bibtex.rs +470 -469
data/vendor/kreuzberg/src/extractors/docbook.rs +504 -502
data/vendor/kreuzberg/src/extractors/docx.rs +400 -367
data/vendor/kreuzberg/src/extractors/email.rs +157 -143
data/vendor/kreuzberg/src/extractors/epub.rs +696 -707
data/vendor/kreuzberg/src/extractors/excel.rs +385 -343
data/vendor/kreuzberg/src/extractors/fictionbook.rs +492 -491
data/vendor/kreuzberg/src/extractors/html.rs +419 -393
data/vendor/kreuzberg/src/extractors/image.rs +219 -198
data/vendor/kreuzberg/src/extractors/jats.rs +1054 -1051
data/vendor/kreuzberg/src/extractors/jupyter.rs +368 -367
data/vendor/kreuzberg/src/extractors/latex.rs +653 -652
data/vendor/kreuzberg/src/extractors/markdown.rs +701 -700
data/vendor/kreuzberg/src/extractors/mod.rs +429 -365
data/vendor/kreuzberg/src/extractors/odt.rs +628 -628
data/vendor/kreuzberg/src/extractors/opml.rs +635 -634
data/vendor/kreuzberg/src/extractors/orgmode.rs +529 -528
data/vendor/kreuzberg/src/extractors/pdf.rs +761 -493
data/vendor/kreuzberg/src/extractors/pptx.rs +279 -248
data/vendor/kreuzberg/src/extractors/rst.rs +577 -576
data/vendor/kreuzberg/src/extractors/rtf.rs +809 -810
data/vendor/kreuzberg/src/extractors/security.rs +484 -484
data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -367
data/vendor/kreuzberg/src/extractors/structured.rs +142 -140
data/vendor/kreuzberg/src/extractors/text.rs +265 -260
data/vendor/kreuzberg/src/extractors/typst.rs +651 -650
data/vendor/kreuzberg/src/extractors/xml.rs +147 -135
data/vendor/kreuzberg/src/image/dpi.rs +164 -164
data/vendor/kreuzberg/src/image/mod.rs +6 -6
data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
data/vendor/kreuzberg/src/image/resize.rs +89 -89
data/vendor/kreuzberg/src/keywords/config.rs +154 -154
data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
data/vendor/kreuzberg/src/keywords/processor.rs +275 -267
data/vendor/kreuzberg/src/keywords/rake.rs +293 -293
data/vendor/kreuzberg/src/keywords/types.rs +68 -68
data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
data/vendor/kreuzberg/src/language_detection/mod.rs +985 -942
data/vendor/kreuzberg/src/language_detection/processor.rs +218 -0
data/vendor/kreuzberg/src/lib.rs +114 -105
data/vendor/kreuzberg/src/mcp/mod.rs +35 -32
data/vendor/kreuzberg/src/mcp/server.rs +2090 -1968
data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
data/vendor/kreuzberg/src/ocr/error.rs +37 -37
data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
data/vendor/kreuzberg/src/ocr/language_registry.rs +520 -0
data/vendor/kreuzberg/src/ocr/mod.rs +60 -58
data/vendor/kreuzberg/src/ocr/processor.rs +858 -863
data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +456 -450
data/vendor/kreuzberg/src/ocr/types.rs +393 -393
data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
data/vendor/kreuzberg/src/panic_context.rs +154 -154
data/vendor/kreuzberg/src/pdf/bindings.rs +306 -0
data/vendor/kreuzberg/src/pdf/bundled.rs +408 -0
data/vendor/kreuzberg/src/pdf/error.rs +214 -122
data/vendor/kreuzberg/src/pdf/fonts.rs +358 -0
data/vendor/kreuzberg/src/pdf/hierarchy.rs +903 -0
data/vendor/kreuzberg/src/pdf/images.rs +139 -139
data/vendor/kreuzberg/src/pdf/metadata.rs +509 -346
data/vendor/kreuzberg/src/pdf/mod.rs +81 -50
data/vendor/kreuzberg/src/pdf/rendering.rs +369 -369
data/vendor/kreuzberg/src/pdf/table.rs +417 -393
data/vendor/kreuzberg/src/pdf/text.rs +553 -158
data/vendor/kreuzberg/src/plugins/extractor.rs +1042 -1013
data/vendor/kreuzberg/src/plugins/mod.rs +212 -209
data/vendor/kreuzberg/src/plugins/ocr.rs +637 -620
data/vendor/kreuzberg/src/plugins/processor.rs +650 -642
data/vendor/kreuzberg/src/plugins/registry.rs +1339 -1337
data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
data/vendor/kreuzberg/src/plugins/validator.rs +967 -956
data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
data/vendor/kreuzberg/src/text/mod.rs +27 -19
data/vendor/kreuzberg/src/text/quality.rs +710 -697
data/vendor/kreuzberg/src/text/quality_processor.rs +231 -0
data/vendor/kreuzberg/src/text/string_utils.rs +229 -217
data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
data/vendor/kreuzberg/src/text/token_reduction/core.rs +832 -796
data/vendor/kreuzberg/src/text/token_reduction/filters.rs +923 -902
data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +148 -147
data/vendor/kreuzberg/src/text/utf8_validation.rs +193 -0
data/vendor/kreuzberg/src/types.rs +1713 -903
data/vendor/kreuzberg/src/utils/mod.rs +31 -17
data/vendor/kreuzberg/src/utils/pool.rs +503 -0
data/vendor/kreuzberg/src/utils/pool_sizing.rs +364 -0
data/vendor/kreuzberg/src/utils/quality.rs +968 -959
data/vendor/kreuzberg/src/utils/string_pool.rs +761 -0
data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
data/vendor/kreuzberg/tests/api_embed.rs +360 -0
data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -52
data/vendor/kreuzberg/tests/api_large_pdf_extraction.rs +471 -0
data/vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs +289 -0
data/vendor/kreuzberg/tests/api_tests.rs +1472 -966
data/vendor/kreuzberg/tests/archive_integration.rs +545 -543
data/vendor/kreuzberg/tests/batch_orchestration.rs +587 -556
data/vendor/kreuzberg/tests/batch_pooling_benchmark.rs +154 -0
data/vendor/kreuzberg/tests/batch_processing.rs +328 -316
data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -421
data/vendor/kreuzberg/tests/concurrency_stress.rs +541 -525
data/vendor/kreuzberg/tests/config_features.rs +612 -598
data/vendor/kreuzberg/tests/config_integration_test.rs +753 -0
data/vendor/kreuzberg/tests/config_loading_tests.rs +416 -415
data/vendor/kreuzberg/tests/core_integration.rs +519 -510
data/vendor/kreuzberg/tests/csv_integration.rs +414 -414
data/vendor/kreuzberg/tests/data/hierarchy_ground_truth.json +294 -0
data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +500 -498
data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -122
data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -370
data/vendor/kreuzberg/tests/email_integration.rs +327 -325
data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -275
data/vendor/kreuzberg/tests/error_handling.rs +402 -393
data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -228
data/vendor/kreuzberg/tests/format_integration.rs +165 -159
data/vendor/kreuzberg/tests/helpers/mod.rs +202 -142
data/vendor/kreuzberg/tests/html_table_test.rs +551 -551
data/vendor/kreuzberg/tests/image_integration.rs +255 -253
data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -139
data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -639
data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -704
data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -496
data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -490
data/vendor/kreuzberg/tests/mime_detection.rs +429 -428
data/vendor/kreuzberg/tests/ocr_configuration.rs +514 -510
data/vendor/kreuzberg/tests/ocr_errors.rs +698 -676
data/vendor/kreuzberg/tests/ocr_language_registry.rs +191 -0
data/vendor/kreuzberg/tests/ocr_quality.rs +629 -627
data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
data/vendor/kreuzberg/tests/odt_extractor_tests.rs +674 -695
data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -616
data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -822
data/vendor/kreuzberg/tests/page_markers.rs +297 -0
data/vendor/kreuzberg/tests/pdf_hierarchy_detection.rs +301 -0
data/vendor/kreuzberg/tests/pdf_hierarchy_quality.rs +589 -0
data/vendor/kreuzberg/tests/pdf_integration.rs +45 -43
data/vendor/kreuzberg/tests/pdf_ocr_triggering.rs +301 -0
data/vendor/kreuzberg/tests/pdf_text_merging.rs +475 -0
data/vendor/kreuzberg/tests/pdfium_linking.rs +340 -0
data/vendor/kreuzberg/tests/pipeline_integration.rs +1446 -1411
data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +776 -771
data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +577 -560
data/vendor/kreuzberg/tests/plugin_system.rs +927 -921
data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
data/vendor/kreuzberg/tests/registry_integration_tests.rs +587 -586
data/vendor/kreuzberg/tests/rst_extractor_tests.rs +694 -692
data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +775 -776
data/vendor/kreuzberg/tests/security_validation.rs +416 -415
data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
data/vendor/kreuzberg/tests/test_fastembed.rs +631 -609
data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1260 -1259
data/vendor/kreuzberg/tests/typst_extractor_tests.rs +648 -647
data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
data/vendor/kreuzberg-ffi/Cargo.toml +67 -0
data/vendor/kreuzberg-ffi/README.md +851 -0
data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs +227 -0
data/vendor/kreuzberg-ffi/build.rs +168 -0
data/vendor/kreuzberg-ffi/cbindgen.toml +37 -0
data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +12 -0
data/vendor/kreuzberg-ffi/kreuzberg.h +3012 -0
data/vendor/kreuzberg-ffi/src/batch_streaming.rs +588 -0
data/vendor/kreuzberg-ffi/src/config.rs +1341 -0
data/vendor/kreuzberg-ffi/src/error.rs +901 -0
data/vendor/kreuzberg-ffi/src/extraction.rs +555 -0
data/vendor/kreuzberg-ffi/src/helpers.rs +879 -0
data/vendor/kreuzberg-ffi/src/lib.rs +977 -0
data/vendor/kreuzberg-ffi/src/memory.rs +493 -0
data/vendor/kreuzberg-ffi/src/mime.rs +329 -0
data/vendor/kreuzberg-ffi/src/panic_shield.rs +265 -0
data/vendor/kreuzberg-ffi/src/plugins/document_extractor.rs +442 -0
data/vendor/kreuzberg-ffi/src/plugins/mod.rs +14 -0
data/vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs +628 -0
data/vendor/kreuzberg-ffi/src/plugins/post_processor.rs +438 -0
data/vendor/kreuzberg-ffi/src/plugins/validator.rs +329 -0
data/vendor/kreuzberg-ffi/src/result.rs +510 -0
data/vendor/kreuzberg-ffi/src/result_pool.rs +639 -0
data/vendor/kreuzberg-ffi/src/result_view.rs +773 -0
data/vendor/kreuzberg-ffi/src/string_intern.rs +568 -0
data/vendor/kreuzberg-ffi/src/types.rs +363 -0
data/vendor/kreuzberg-ffi/src/util.rs +210 -0
data/vendor/kreuzberg-ffi/src/validation.rs +848 -0
data/vendor/kreuzberg-ffi/tests.disabled/README.md +48 -0
data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +299 -0
data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +346 -0
data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +232 -0
data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +470 -0
data/vendor/kreuzberg-tesseract/.commitlintrc.json +13 -0
data/vendor/kreuzberg-tesseract/.crate-ignore +2 -0
data/vendor/kreuzberg-tesseract/Cargo.lock +2933 -0
data/vendor/kreuzberg-tesseract/Cargo.toml +57 -0
data/vendor/{rb-sys/LICENSE-MIT → kreuzberg-tesseract/LICENSE} +22 -21
data/vendor/kreuzberg-tesseract/README.md +399 -0
data/vendor/kreuzberg-tesseract/build.rs +1127 -0
data/vendor/kreuzberg-tesseract/patches/README.md +71 -0
data/vendor/kreuzberg-tesseract/patches/tesseract.diff +199 -0
data/vendor/kreuzberg-tesseract/src/api.rs +1371 -0
data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +77 -0
data/vendor/kreuzberg-tesseract/src/enums.rs +297 -0
data/vendor/kreuzberg-tesseract/src/error.rs +81 -0
data/vendor/kreuzberg-tesseract/src/lib.rs +145 -0
data/vendor/kreuzberg-tesseract/src/monitor.rs +57 -0
data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +197 -0
data/vendor/kreuzberg-tesseract/src/page_iterator.rs +253 -0
data/vendor/kreuzberg-tesseract/src/result_iterator.rs +286 -0
data/vendor/kreuzberg-tesseract/src/result_renderer.rs +183 -0
data/vendor/kreuzberg-tesseract/tests/integration_test.rs +211 -0
metadata +196 -45
data/vendor/kreuzberg/benches/otel_overhead.rs +0 -48
data/vendor/kreuzberg/src/extractors/fictionbook.rs.backup2 +0 -738
data/vendor/rb-sys/.cargo-ok +0 -1
data/vendor/rb-sys/.cargo_vcs_info.json +0 -6
data/vendor/rb-sys/Cargo.lock +0 -393
data/vendor/rb-sys/Cargo.toml +0 -70
data/vendor/rb-sys/Cargo.toml.orig +0 -57
data/vendor/rb-sys/LICENSE-APACHE +0 -190
data/vendor/rb-sys/bin/release.sh +0 -21
data/vendor/rb-sys/build/features.rs +0 -108
data/vendor/rb-sys/build/main.rs +0 -246
data/vendor/rb-sys/build/stable_api_config.rs +0 -153
data/vendor/rb-sys/build/version.rs +0 -48
data/vendor/rb-sys/readme.md +0 -36
data/vendor/rb-sys/src/bindings.rs +0 -21
data/vendor/rb-sys/src/hidden.rs +0 -11
data/vendor/rb-sys/src/lib.rs +0 -34
data/vendor/rb-sys/src/macros.rs +0 -371
data/vendor/rb-sys/src/memory.rs +0 -53
data/vendor/rb-sys/src/ruby_abi_version.rs +0 -38
data/vendor/rb-sys/src/special_consts.rs +0 -31
data/vendor/rb-sys/src/stable_api/compiled.c +0 -179
data/vendor/rb-sys/src/stable_api/compiled.rs +0 -257
data/vendor/rb-sys/src/stable_api/ruby_2_6.rs +0 -316
data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +0 -316
data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +0 -324
data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +0 -317
data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +0 -315
data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +0 -326
data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +0 -327
data/vendor/rb-sys/src/stable_api.rs +0 -261
data/vendor/rb-sys/src/symbol.rs +0 -31
data/vendor/rb-sys/src/tracking_allocator.rs +0 -332
data/vendor/rb-sys/src/utils.rs +0 -89
data/vendor/rb-sys/src/value_type.rs +0 -7

data/vendor/kreuzberg/stopwords/ko_stopwords.json CHANGED Viewed

@@ -1,681 +1,681 @@
-[
-	"!",
-	"\"",
-	"$",
-	"%",
-	"&",
-	"'",
-	"(",
-	")",
-	"*",
-	"+",
-	",",
-	"-",
-	".",
-	"...",
-	"0",
-	"1",
-	"2",
-	"3",
-	"4",
-	"5",
-	"6",
-	"7",
-	"8",
-	"9",
-	";",
-	"<",
-	"=",
-	">",
-	"?",
-	"@",
-	"\\",
-	"^",
-	"_",
-	"`",
-	"|",
-	"~",
-	"·",
-	"—",
-	"——",
-	"‘",
-	"’",
-	"“",
-	"”",
-	"…",
-	"、",
-	"。",
-	"〈",
-	"〉",
-	"《",
-	"》",
-	"가",
-	"가까스로",
-	"가령",
-	"각",
-	"각각",
-	"각자",
-	"각종",
-	"갖고말하자면",
-	"같다",
-	"같이",
-	"개의치않고",
-	"거니와",
-	"거바",
-	"거의",
-	"것",
-	"것과 같이",
-	"것들",
-	"게다가",
-	"게우다",
-	"겨우",
-	"견지에서",
-	"결과에 이르다",
-	"결국",
-	"결론을 낼 수 있다",
-	"겸사겸사",
-	"고려하면",
-	"고로",
-	"곧",
-	"공동으로",
-	"과",
-	"과연",
-	"관계가 있다",
-	"관계없이",
-	"관련이 있다",
-	"관하여",
-	"관한",
-	"관해서는",
-	"구",
-	"구체적으로",
-	"구토하다",
-	"그",
-	"그들",
-	"그때",
-	"그래",
-	"그래도",
-	"그래서",
-	"그러나",
-	"그러니",
-	"그러니까",
-	"그러면",
-	"그러므로",
-	"그러한즉",
-	"그런 까닭에",
-	"그런데",
-	"그런즉",
-	"그럼",
-	"그럼에도 불구하고",
-	"그렇게 함으로써",
-	"그렇지",
-	"그렇지 않다면",
-	"그렇지 않으면",
-	"그렇지만",
-	"그렇지않으면",
-	"그리고",
-	"그리하여",
-	"그만이다",
-	"그에 따르는",
-	"그위에",
-	"그저",
-	"그중에서",
-	"그치지 않다",
-	"근거로",
-	"근거하여",
-	"기대여",
-	"기점으로",
-	"기준으로",
-	"기타",
-	"까닭으로",
-	"까악",
-	"까지",
-	"까지 미치다",
-	"까지도",
-	"꽈당",
-	"끙끙",
-	"끼익",
-	"나",
-	"나머지는",
-	"남들",
-	"남짓",
-	"너",
-	"너희",
-	"너희들",
-	"네",
-	"넷",
-	"년",
-	"논하지 않다",
-	"놀라다",
-	"누가 알겠는가",
-	"누구",
-	"다른",
-	"다른 방면으로",
-	"다만",
-	"다섯",
-	"다소",
-	"다수",
-	"다시 말하자면",
-	"다시말하면",
-	"다음",
-	"다음에",
-	"다음으로",
-	"단지",
-	"답다",
-	"당신",
-	"당장",
-	"대로 하다",
-	"대하면",
-	"대하여",
-	"대해 말하자면",
-	"대해서",
-	"댕그",
-	"더구나",
-	"더군다나",
-	"더라도",
-	"더불어",
-	"더욱더",
-	"더욱이는",
-	"도달하다",
-	"도착하다",
-	"동시에",
-	"동안",
-	"된바에야",
-	"된이상",
-	"두번째로",
-	"둘",
-	"둥둥",
-	"뒤따라",
-	"뒤이어",
-	"든간에",
-	"들",
-	"등",
-	"등등",
-	"딩동",
-	"따라",
-	"따라서",
-	"따위",
-	"따지지 않다",
-	"딱",
-	"때",
-	"때가 되어",
-	"때문에",
-	"또",
-	"또한",
-	"뚝뚝",
-	"라 해도",
-	"령",
-	"로",
-	"로 인하여",
-	"로부터",
-	"로써",
-	"륙",
-	"를",
-	"마음대로",
-	"마저",
-	"마저도",
-	"마치",
-	"막론하고",
-	"만 못하다",
-	"만약",
-	"만약에",
-	"만은 아니다",
-	"만이 아니다",
-	"만일",
-	"만큼",
-	"말하자면",
-	"말할것도 없고",
-	"매",
-	"매번",
-	"메쓰겁다",
-	"몇",
-	"모",
-	"모두",
-	"무렵",
-	"무릎쓰고",
-	"무슨",
-	"무엇",
-	"무엇때문에",
-	"물론",
-	"및",
-	"바꾸어말하면",
-	"바꾸어말하자면",
-	"바꾸어서 말하면",
-	"바꾸어서 한다면",
-	"바꿔 말하면",
-	"바로",
-	"바와같이",
-	"밖에 안된다",
-	"반대로",
-	"반대로 말하자면",
-	"반드시",
-	"버금",
-	"보는데서",
-	"보다더",
-	"보드득",
-	"본대로",
-	"봐",
-	"봐라",
-	"부류의 사람들",
-	"부터",
-	"불구하고",
-	"불문하고",
-	"붕붕",
-	"비걱거리다",
-	"비교적",
-	"비길수 없다",
-	"비로소",
-	"비록",
-	"비슷하다",
-	"비추어 보아",
-	"비하면",
-	"뿐만 아니라",
-	"뿐만아니라",
-	"뿐이다",
-	"삐걱",
-	"삐걱거리다",
-	"사",
-	"삼",
-	"상대적으로 말하자면",
-	"생각한대로",
-	"설령",
-	"설마",
-	"설사",
-	"셋",
-	"소생",
-	"소인",
-	"솨",
-	"쉿",
-	"습니까",
-	"습니다",
-	"시각",
-	"시간",
-	"시작하여",
-	"시초에",
-	"시키다",
-	"실로",
-	"심지어",
-	"아",
-	"아니",
-	"아니나다를가",
-	"아니라면",
-	"아니면",
-	"아니었다면",
-	"아래윗",
-	"아무거나",
-	"아무도",
-	"아야",
-	"아울러",
-	"아이",
-	"아이고",
-	"아이구",
-	"아이야",
-	"아이쿠",
-	"아하",
-	"아홉",
-	"안 그러면",
-	"않기 위하여",
-	"않기 위해서",
-	"알 수 있다",
-	"알았어",
-	"앗",
-	"앞에서",
-	"앞의것",
-	"야",
-	"약간",
-	"양자",
-	"어",
-	"어기여차",
-	"어느",
-	"어느 년도",
-	"어느것",
-	"어느곳",
-	"어느때",
-	"어느쪽",
-	"어느해",
-	"어디",
-	"어때",
-	"어떠한",
-	"어떤",
-	"어떤것",
-	"어떤것들",
-	"어떻게",
-	"어떻해",
-	"어이",
-	"어째서",
-	"어쨋든",
-	"어쩔수 없다",
-	"어찌",
-	"어찌됏든",
-	"어찌됏어",
-	"어찌하든지",
-	"어찌하여",
-	"언제",
-	"언젠가",
-	"얼마",
-	"얼마 안 되는 것",
-	"얼마간",
-	"얼마나",
-	"얼마든지",
-	"얼마만큼",
-	"얼마큼",
-	"엉엉",
-	"에",
-	"에 가서",
-	"에 달려 있다",
-	"에 대해",
-	"에 있다",
-	"에 한하다",
-	"에게",
-	"에서",
-	"여",
-	"여기",
-	"여덟",
-	"여러분",
-	"여보시오",
-	"여부",
-	"여섯",
-	"여전히",
-	"여차",
-	"연관되다",
-	"연이서",
-	"영",
-	"영차",
-	"옆사람",
-	"예",
-	"예를 들면",
-	"예를 들자면",
-	"예컨대",
-	"예하면",
-	"오",
-	"오로지",
-	"오르다",
-	"오자마자",
-	"오직",
-	"오호",
-	"오히려",
-	"와",
-	"와 같은 사람들",
-	"와르르",
-	"와아",
-	"왜",
-	"왜냐하면",
-	"외에도",
-	"요만큼",
-	"요만한 것",
-	"요만한걸",
-	"요컨대",
-	"우르르",
-	"우리",
-	"우리들",
-	"우선",
-	"우에 종합한것과같이",
-	"운운",
-	"월",
-	"위에서 서술한바와같이",
-	"위하여",
-	"위해서",
-	"윙윙",
-	"육",
-	"으로",
-	"으로 인하여",
-	"으로서",
-	"으로써",
-	"을",
-	"응",
-	"응당",
-	"의",
-	"의거하여",
-	"의지하여",
-	"의해",
-	"의해되다",
-	"의해서",
-	"이",
-	"이 되다",
-	"이 때문에",
-	"이 밖에",
-	"이 외에",
-	"이 정도의",
-	"이것",
-	"이곳",
-	"이때",
-	"이라면",
-	"이래",
-	"이러이러하다",
-	"이러한",
-	"이런",
-	"이럴정도로",
-	"이렇게 많은 것",
-	"이렇게되면",
-	"이렇게말하자면",
-	"이렇구나",
-	"이로 인하여",
-	"이르기까지",
-	"이리하여",
-	"이만큼",
-	"이번",
-	"이봐",
-	"이상",
-	"이어서",
-	"이었다",
-	"이와 같다",
-	"이와 같은",
-	"이와 반대로",
-	"이와같다면",
-	"이외에도",
-	"이용하여",
-	"이유만으로",
-	"이젠",
-	"이지만",
-	"이쪽",
-	"이천구",
-	"이천육",
-	"이천칠",
-	"이천팔",
-	"인 듯하다",
-	"인젠",
-	"일",
-	"일것이다",
-	"일곱",
-	"일단",
-	"일때",
-	"일반적으로",
-	"일지라도",
-	"임에 틀림없다",
-	"입각하여",
-	"입장에서",
-	"잇따라",
-	"있다",
-	"자",
-	"자기",
-	"자기집",
-	"자마자",
-	"자신",
-	"잠깐",
-	"잠시",
-	"저",
-	"저것",
-	"저것만큼",
-	"저기",
-	"저쪽",
-	"저희",
-	"전부",
-	"전자",
-	"전후",
-	"점에서 보아",
-	"정도에 이르다",
-	"제",
-	"제각기",
-	"제외하고",
-	"조금",
-	"조차",
-	"조차도",
-	"졸졸",
-	"좀",
-	"좋아",
-	"좍좍",
-	"주룩주룩",
-	"주저하지 않고",
-	"줄은 몰랏다",
-	"줄은모른다",
-	"중에서",
-	"중의하나",
-	"즈음하여",
-	"즉",
-	"즉시",
-	"지든지",
-	"지만",
-	"지말고",
-	"진짜로",
-	"쪽으로",
-	"차라리",
-	"참",
-	"참나",
-	"첫번째로",
-	"쳇",
-	"총적으로",
-	"총적으로 말하면",
-	"총적으로 보면",
-	"칠",
-	"콸콸",
-	"쾅쾅",
-	"쿵",
-	"타다",
-	"타인",
-	"탕탕",
-	"토하다",
-	"통하여",
-	"툭",
-	"퉤",
-	"틈타",
-	"팍",
-	"팔",
-	"퍽",
-	"펄렁",
-	"하",
-	"하게될것이다",
-	"하게하다",
-	"하겠는가",
-	"하고 있다",
-	"하고있었다",
-	"하곤하였다",
-	"하구나",
-	"하기 때문에",
-	"하기 위하여",
-	"하기는한데",
-	"하기만 하면",
-	"하기보다는",
-	"하기에",
-	"하나",
-	"하느니",
-	"하는 김에",
-	"하는 편이 낫다",
-	"하는것도",
-	"하는것만 못하다",
-	"하는것이 낫다",
-	"하는바",
-	"하더라도",
-	"하도다",
-	"하도록시키다",
-	"하도록하다",
-	"하든지",
-	"하려고하다",
-	"하마터면",
-	"하면 할수록",
-	"하면된다",
-	"하면서",
-	"하물며",
-	"하여금",
-	"하여야",
-	"하자마자",
-	"하지 않는다면",
-	"하지 않도록",
-	"하지마",
-	"하지마라",
-	"하지만",
-	"하하",
-	"한 까닭에",
-	"한 이유는",
-	"한 후",
-	"한다면",
-	"한다면 몰라도",
-	"한데",
-	"한마디",
-	"한적이있다",
-	"한켠으로는",
-	"한항목",
-	"할 따름이다",
-	"할 생각이다",
-	"할 줄 안다",
-	"할 지경이다",
-	"할 힘이 있다",
-	"할때",
-	"할만하다",
-	"할망정",
-	"할뿐",
-	"할수있다",
-	"할수있어",
-	"할줄알다",
-	"할지라도",
-	"할지언정",
-	"함께",
-	"해도된다",
-	"해도좋다",
-	"해봐요",
-	"해서는 안된다",
-	"해야한다",
-	"해요",
-	"했어요",
-	"향하다",
-	"향하여",
-	"향해서",
-	"허",
-	"허걱",
-	"허허",
-	"헉",
-	"헉헉",
-	"헐떡헐떡",
-	"형식으로 쓰여",
-	"혹시",
-	"혹은",
-	"혼자",
-	"훨씬",
-	"휘익",
-	"휴",
-	"흐흐",
-	"흥",
-	"힘입어",
-	"︿",
-	"！",
-	"＃",
-	"＄",
-	"％",
-	"＆",
-	"（",
-	"）",
-	"＊",
-	"＋",
-	"，",
-	"０",
-	"１",
-	"２",
-	"３",
-	"４",
-	"５",
-	"６",
-	"７",
-	"８",
-	"９",
-	"：",
-	"；",
-	"＜",
-	"＞",
-	"？",
-	"＠",
-	"［",
-	"］",
-	"｛",
-	"｜",
-	"｝",
-	"～",
-	"￥"
-]
+[
+	"!",
+	"\"",
+	"$",
+	"%",
+	"&",
+	"'",
+	"(",
+	")",
+	"*",
+	"+",
+	",",
+	"-",
+	".",
+	"...",
+	"0",
+	"1",
+	"2",
+	"3",
+	"4",
+	"5",
+	"6",
+	"7",
+	"8",
+	"9",
+	";",
+	"<",
+	"=",
+	">",
+	"?",
+	"@",
+	"\\",
+	"^",
+	"_",
+	"`",
+	"|",
+	"~",
+	"·",
+	"—",
+	"——",
+	"‘",
+	"’",
+	"“",
+	"”",
+	"…",
+	"、",
+	"。",
+	"〈",
+	"〉",
+	"《",
+	"》",
+	"가",
+	"가까스로",
+	"가령",
+	"각",
+	"각각",
+	"각자",
+	"각종",
+	"갖고말하자면",
+	"같다",
+	"같이",
+	"개의치않고",
+	"거니와",
+	"거바",
+	"거의",
+	"것",
+	"것과 같이",
+	"것들",
+	"게다가",
+	"게우다",
+	"겨우",
+	"견지에서",
+	"결과에 이르다",
+	"결국",
+	"결론을 낼 수 있다",
+	"겸사겸사",
+	"고려하면",
+	"고로",
+	"곧",
+	"공동으로",
+	"과",
+	"과연",
+	"관계가 있다",
+	"관계없이",
+	"관련이 있다",
+	"관하여",
+	"관한",
+	"관해서는",
+	"구",
+	"구체적으로",
+	"구토하다",
+	"그",
+	"그들",
+	"그때",
+	"그래",
+	"그래도",
+	"그래서",
+	"그러나",
+	"그러니",
+	"그러니까",
+	"그러면",
+	"그러므로",
+	"그러한즉",
+	"그런 까닭에",
+	"그런데",
+	"그런즉",
+	"그럼",
+	"그럼에도 불구하고",
+	"그렇게 함으로써",
+	"그렇지",
+	"그렇지 않다면",
+	"그렇지 않으면",
+	"그렇지만",
+	"그렇지않으면",
+	"그리고",
+	"그리하여",
+	"그만이다",
+	"그에 따르는",
+	"그위에",
+	"그저",
+	"그중에서",
+	"그치지 않다",
+	"근거로",
+	"근거하여",
+	"기대여",
+	"기점으로",
+	"기준으로",
+	"기타",
+	"까닭으로",
+	"까악",
+	"까지",
+	"까지 미치다",
+	"까지도",
+	"꽈당",
+	"끙끙",
+	"끼익",
+	"나",
+	"나머지는",
+	"남들",
+	"남짓",
+	"너",
+	"너희",
+	"너희들",
+	"네",
+	"넷",
+	"년",
+	"논하지 않다",
+	"놀라다",
+	"누가 알겠는가",
+	"누구",
+	"다른",
+	"다른 방면으로",
+	"다만",
+	"다섯",
+	"다소",
+	"다수",
+	"다시 말하자면",
+	"다시말하면",
+	"다음",
+	"다음에",
+	"다음으로",
+	"단지",
+	"답다",
+	"당신",
+	"당장",
+	"대로 하다",
+	"대하면",
+	"대하여",
+	"대해 말하자면",
+	"대해서",
+	"댕그",
+	"더구나",
+	"더군다나",
+	"더라도",
+	"더불어",
+	"더욱더",
+	"더욱이는",
+	"도달하다",
+	"도착하다",
+	"동시에",
+	"동안",
+	"된바에야",
+	"된이상",
+	"두번째로",
+	"둘",
+	"둥둥",
+	"뒤따라",
+	"뒤이어",
+	"든간에",
+	"들",
+	"등",
+	"등등",
+	"딩동",
+	"따라",
+	"따라서",
+	"따위",
+	"따지지 않다",
+	"딱",
+	"때",
+	"때가 되어",
+	"때문에",
+	"또",
+	"또한",
+	"뚝뚝",
+	"라 해도",
+	"령",
+	"로",
+	"로 인하여",
+	"로부터",
+	"로써",
+	"륙",
+	"를",
+	"마음대로",
+	"마저",
+	"마저도",
+	"마치",
+	"막론하고",
+	"만 못하다",
+	"만약",
+	"만약에",
+	"만은 아니다",
+	"만이 아니다",
+	"만일",
+	"만큼",
+	"말하자면",
+	"말할것도 없고",
+	"매",
+	"매번",
+	"메쓰겁다",
+	"몇",
+	"모",
+	"모두",
+	"무렵",
+	"무릎쓰고",
+	"무슨",
+	"무엇",
+	"무엇때문에",
+	"물론",
+	"및",
+	"바꾸어말하면",
+	"바꾸어말하자면",
+	"바꾸어서 말하면",
+	"바꾸어서 한다면",
+	"바꿔 말하면",
+	"바로",
+	"바와같이",
+	"밖에 안된다",
+	"반대로",
+	"반대로 말하자면",
+	"반드시",
+	"버금",
+	"보는데서",
+	"보다더",
+	"보드득",
+	"본대로",
+	"봐",
+	"봐라",
+	"부류의 사람들",
+	"부터",
+	"불구하고",
+	"불문하고",
+	"붕붕",
+	"비걱거리다",
+	"비교적",
+	"비길수 없다",
+	"비로소",
+	"비록",
+	"비슷하다",
+	"비추어 보아",
+	"비하면",
+	"뿐만 아니라",
+	"뿐만아니라",
+	"뿐이다",
+	"삐걱",
+	"삐걱거리다",
+	"사",
+	"삼",
+	"상대적으로 말하자면",
+	"생각한대로",
+	"설령",
+	"설마",
+	"설사",
+	"셋",
+	"소생",
+	"소인",
+	"솨",
+	"쉿",
+	"습니까",
+	"습니다",
+	"시각",
+	"시간",
+	"시작하여",
+	"시초에",
+	"시키다",
+	"실로",
+	"심지어",
+	"아",
+	"아니",
+	"아니나다를가",
+	"아니라면",
+	"아니면",
+	"아니었다면",
+	"아래윗",
+	"아무거나",
+	"아무도",
+	"아야",
+	"아울러",
+	"아이",
+	"아이고",
+	"아이구",
+	"아이야",
+	"아이쿠",
+	"아하",
+	"아홉",
+	"안 그러면",
+	"않기 위하여",
+	"않기 위해서",
+	"알 수 있다",
+	"알았어",
+	"앗",
+	"앞에서",
+	"앞의것",
+	"야",
+	"약간",
+	"양자",
+	"어",
+	"어기여차",
+	"어느",
+	"어느 년도",
+	"어느것",
+	"어느곳",
+	"어느때",
+	"어느쪽",
+	"어느해",
+	"어디",
+	"어때",
+	"어떠한",
+	"어떤",
+	"어떤것",
+	"어떤것들",
+	"어떻게",
+	"어떻해",
+	"어이",
+	"어째서",
+	"어쨋든",
+	"어쩔수 없다",
+	"어찌",
+	"어찌됏든",
+	"어찌됏어",
+	"어찌하든지",
+	"어찌하여",
+	"언제",
+	"언젠가",
+	"얼마",
+	"얼마 안 되는 것",
+	"얼마간",
+	"얼마나",
+	"얼마든지",
+	"얼마만큼",
+	"얼마큼",
+	"엉엉",
+	"에",
+	"에 가서",
+	"에 달려 있다",
+	"에 대해",
+	"에 있다",
+	"에 한하다",
+	"에게",
+	"에서",
+	"여",
+	"여기",
+	"여덟",
+	"여러분",
+	"여보시오",
+	"여부",
+	"여섯",
+	"여전히",
+	"여차",
+	"연관되다",
+	"연이서",
+	"영",
+	"영차",
+	"옆사람",
+	"예",
+	"예를 들면",
+	"예를 들자면",
+	"예컨대",
+	"예하면",
+	"오",
+	"오로지",
+	"오르다",
+	"오자마자",
+	"오직",
+	"오호",
+	"오히려",
+	"와",
+	"와 같은 사람들",
+	"와르르",
+	"와아",
+	"왜",
+	"왜냐하면",
+	"외에도",
+	"요만큼",
+	"요만한 것",
+	"요만한걸",
+	"요컨대",
+	"우르르",
+	"우리",
+	"우리들",
+	"우선",
+	"우에 종합한것과같이",
+	"운운",
+	"월",
+	"위에서 서술한바와같이",
+	"위하여",
+	"위해서",
+	"윙윙",
+	"육",
+	"으로",
+	"으로 인하여",
+	"으로서",
+	"으로써",
+	"을",
+	"응",
+	"응당",
+	"의",
+	"의거하여",
+	"의지하여",
+	"의해",
+	"의해되다",
+	"의해서",
+	"이",
+	"이 되다",
+	"이 때문에",
+	"이 밖에",
+	"이 외에",
+	"이 정도의",
+	"이것",
+	"이곳",
+	"이때",
+	"이라면",
+	"이래",
+	"이러이러하다",
+	"이러한",
+	"이런",
+	"이럴정도로",
+	"이렇게 많은 것",
+	"이렇게되면",
+	"이렇게말하자면",
+	"이렇구나",
+	"이로 인하여",
+	"이르기까지",
+	"이리하여",
+	"이만큼",
+	"이번",
+	"이봐",
+	"이상",
+	"이어서",
+	"이었다",
+	"이와 같다",
+	"이와 같은",
+	"이와 반대로",
+	"이와같다면",
+	"이외에도",
+	"이용하여",
+	"이유만으로",
+	"이젠",
+	"이지만",
+	"이쪽",
+	"이천구",
+	"이천육",
+	"이천칠",
+	"이천팔",
+	"인 듯하다",
+	"인젠",
+	"일",
+	"일것이다",
+	"일곱",
+	"일단",
+	"일때",
+	"일반적으로",
+	"일지라도",
+	"임에 틀림없다",
+	"입각하여",
+	"입장에서",
+	"잇따라",
+	"있다",
+	"자",
+	"자기",
+	"자기집",
+	"자마자",
+	"자신",
+	"잠깐",
+	"잠시",
+	"저",
+	"저것",
+	"저것만큼",
+	"저기",
+	"저쪽",
+	"저희",
+	"전부",
+	"전자",
+	"전후",
+	"점에서 보아",
+	"정도에 이르다",
+	"제",
+	"제각기",
+	"제외하고",
+	"조금",
+	"조차",
+	"조차도",
+	"졸졸",
+	"좀",
+	"좋아",
+	"좍좍",
+	"주룩주룩",
+	"주저하지 않고",
+	"줄은 몰랏다",
+	"줄은모른다",
+	"중에서",
+	"중의하나",
+	"즈음하여",
+	"즉",
+	"즉시",
+	"지든지",
+	"지만",
+	"지말고",
+	"진짜로",
+	"쪽으로",
+	"차라리",
+	"참",
+	"참나",
+	"첫번째로",
+	"쳇",
+	"총적으로",
+	"총적으로 말하면",
+	"총적으로 보면",
+	"칠",
+	"콸콸",
+	"쾅쾅",
+	"쿵",
+	"타다",
+	"타인",
+	"탕탕",
+	"토하다",
+	"통하여",
+	"툭",
+	"퉤",
+	"틈타",
+	"팍",
+	"팔",
+	"퍽",
+	"펄렁",
+	"하",
+	"하게될것이다",
+	"하게하다",
+	"하겠는가",
+	"하고 있다",
+	"하고있었다",
+	"하곤하였다",
+	"하구나",
+	"하기 때문에",
+	"하기 위하여",
+	"하기는한데",
+	"하기만 하면",
+	"하기보다는",
+	"하기에",
+	"하나",
+	"하느니",
+	"하는 김에",
+	"하는 편이 낫다",
+	"하는것도",
+	"하는것만 못하다",
+	"하는것이 낫다",
+	"하는바",
+	"하더라도",
+	"하도다",
+	"하도록시키다",
+	"하도록하다",
+	"하든지",
+	"하려고하다",
+	"하마터면",
+	"하면 할수록",
+	"하면된다",
+	"하면서",
+	"하물며",
+	"하여금",
+	"하여야",
+	"하자마자",
+	"하지 않는다면",
+	"하지 않도록",
+	"하지마",
+	"하지마라",
+	"하지만",
+	"하하",
+	"한 까닭에",
+	"한 이유는",
+	"한 후",
+	"한다면",
+	"한다면 몰라도",
+	"한데",
+	"한마디",
+	"한적이있다",
+	"한켠으로는",
+	"한항목",
+	"할 따름이다",
+	"할 생각이다",
+	"할 줄 안다",
+	"할 지경이다",
+	"할 힘이 있다",
+	"할때",
+	"할만하다",
+	"할망정",
+	"할뿐",
+	"할수있다",
+	"할수있어",
+	"할줄알다",
+	"할지라도",
+	"할지언정",
+	"함께",
+	"해도된다",
+	"해도좋다",
+	"해봐요",
+	"해서는 안된다",
+	"해야한다",
+	"해요",
+	"했어요",
+	"향하다",
+	"향하여",
+	"향해서",
+	"허",
+	"허걱",
+	"허허",
+	"헉",
+	"헉헉",
+	"헐떡헐떡",
+	"형식으로 쓰여",
+	"혹시",
+	"혹은",
+	"혼자",
+	"훨씬",
+	"휘익",
+	"휴",
+	"흐흐",
+	"흥",
+	"힘입어",
+	"︿",
+	"！",
+	"＃",
+	"＄",
+	"％",
+	"＆",
+	"（",
+	"）",
+	"＊",
+	"＋",
+	"，",
+	"０",
+	"１",
+	"２",
+	"３",
+	"４",
+	"５",
+	"６",
+	"７",
+	"８",
+	"９",
+	"：",
+	"；",
+	"＜",
+	"＞",
+	"？",
+	"＠",
+	"［",
+	"］",
+	"｛",
+	"｜",
+	"｝",
+	"～",
+	"￥"
+]