kotoshu 1.0.5 → 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/.kotoshu-baseline.json +1 -1
  3. data/CHANGELOG.md +217 -8
  4. data/Cargo.lock +6 -6
  5. data/TODO.impl/00-vision.md +46 -0
  6. data/TODO.impl/140-reader-ttl-freedom.md +55 -0
  7. data/TODO.impl/141-changelog-union-errorclass.md +51 -0
  8. data/TODO.impl/142-arm-time-pairing.md +46 -0
  9. data/TODO.impl/143-bounded-jobs-timeouts.md +33 -0
  10. data/TODO.impl/144-rbs-examples-catalog.md +47 -0
  11. data/TODO.impl/145-no-dead-urls.md +37 -0
  12. data/TODO.impl/146-realword-confusion-detection.md +44 -0
  13. data/TODO.impl/147-bcp47-gem-half.md +42 -0
  14. data/TODO.impl/148-checks-framework.md +47 -0
  15. data/conformance/vectors.jsonl +207 -207
  16. data/data/ATTRIBUTION.md +78 -0
  17. data/data/README.md +135 -0
  18. data/docs/ARCHITECTURE_IMPROVEMENTS.md +801 -0
  19. data/docs/ARCHITECTURE_IMPROVEMENTS_PLAN.md +505 -0
  20. data/docs/KOTOSHU_FULL_PLAN.md +970 -0
  21. data/docs/KOTOSHU_LANGUAGETOOL_GAPS.md +917 -0
  22. data/docs/KOTOSHU_SOLIDIFICATION_PLAN.md +1650 -0
  23. data/docs/ONNX_FUNCTIONAL_VERIFICATION.md +149 -0
  24. data/docs/RELEASING.md +78 -0
  25. data/docs/TDD_ITERATION_STRATEGY.md +1123 -0
  26. data/docs/architecture-overview.md +473 -0
  27. data/docs/architecture.md +821 -0
  28. data/docs/embeddings_architecture_v2_plan.md +861 -0
  29. data/docs/getting-started.md +393 -0
  30. data/docs/language-modules.md +361 -0
  31. data/docs/onnx-implementation.md +166 -0
  32. data/docs/performance.md +488 -0
  33. data/docs/plugins.md +750 -0
  34. data/examples/08_debug_mode.rb +27 -0
  35. data/examples/09_metrics.rb +29 -0
  36. data/examples/10_keyboard_layout_demo.rb +153 -0
  37. data/examples/11_onnx_spell_checking.rb +170 -0
  38. data/examples/12_kelly_frequency_integration.rb +242 -0
  39. data/examples/12_multi_language_integration.rb +200 -0
  40. data/examples/13_fasttext_onnx_semantic.rb +323 -0
  41. data/examples/14_embeddings_v2_demo.rb +160 -0
  42. data/examples/README_LANGUAGE_EXAMPLES.md +103 -0
  43. data/examples/SPELLS_EXAMPLES.md +199 -0
  44. data/examples/de_german_example.rb +119 -0
  45. data/examples/en_english_example.rb +88 -0
  46. data/examples/es_spanish_example.rb +156 -0
  47. data/examples/fast_benchmark.rb +201 -0
  48. data/examples/fr_french_example.rb +110 -0
  49. data/examples/ja_japanese_example.rb +118 -0
  50. data/examples/pt_portuguese_example.rb +128 -0
  51. data/examples/ru_russian_example.rb +121 -0
  52. data/examples/spell_grammar_check_yaml.rb +293 -0
  53. data/examples/spylls_basic.rb +38 -0
  54. data/examples/spylls_dic.rb +36 -0
  55. data/examples/spylls_lookup.rb +45 -0
  56. data/examples/spylls_suggest.rb +39 -0
  57. data/examples/spylls_unmunch.rb +260 -0
  58. data/examples/spylls_utils.rb +76 -0
  59. data/examples/suggestion_benchmark.rb +266 -0
  60. data/examples/tokenization_examples.rb +179 -0
  61. data/ext/kotoshu_native/Cargo.toml +2 -2
  62. data/lib/kotoshu/cache/base_cache.rb +1 -0
  63. data/lib/kotoshu/cache/frequency_cache.rb +9 -2
  64. data/lib/kotoshu/cache/language_cache.rb +63 -6
  65. data/lib/kotoshu/cache/model_cache.rb +20 -2
  66. data/lib/kotoshu/cache/model_registry.rb +4 -0
  67. data/lib/kotoshu/cache/staged_languages.rb +3 -12
  68. data/lib/kotoshu/checks/base.rb +56 -0
  69. data/lib/kotoshu/checks/finding.rb +19 -0
  70. data/lib/kotoshu/checks/grammar_check.rb +43 -0
  71. data/lib/kotoshu/checks/registry.rb +59 -0
  72. data/lib/kotoshu/checks/spelling_check.rb +30 -0
  73. data/lib/kotoshu/checks.rb +16 -0
  74. data/lib/kotoshu/configuration.rb +1 -1
  75. data/lib/kotoshu/core/exceptions.rb +19 -0
  76. data/lib/kotoshu/data/common_words_loader.rb +37 -7
  77. data/lib/kotoshu/data/confusion/pinyin.json +1 -0
  78. data/lib/kotoshu/dictionaries/catalog.rb +1 -1
  79. data/lib/kotoshu/dictionary/plain_text.rb +1 -1
  80. data/lib/kotoshu/integrity/net_http.rb +1 -0
  81. data/lib/kotoshu/keyboard/layout.rb +6 -6
  82. data/lib/kotoshu/keyboard/layouts/chinese_ime.rb +69 -0
  83. data/lib/kotoshu/keyboard/registry.rb +26 -0
  84. data/lib/kotoshu/keyboard.rb +6 -0
  85. data/lib/kotoshu/multi_language_checker.rb +11 -2
  86. data/lib/kotoshu/resource_bundle.rb +5 -1
  87. data/lib/kotoshu/resource_manager.rb +61 -16
  88. data/lib/kotoshu/source_registry.rb +3 -0
  89. data/lib/kotoshu/spellchecker.rb +11 -7
  90. data/lib/kotoshu/suggestions/confusion_set.rb +68 -0
  91. data/lib/kotoshu/suggestions/frequency_provider.rb +89 -19
  92. data/lib/kotoshu/suggestions/generator.rb +11 -0
  93. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +10 -0
  94. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +32 -3
  95. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +45 -11
  96. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +289 -35
  97. data/lib/kotoshu/suggestions.rb +18 -1
  98. data/lib/kotoshu/typo/engine.rb +4 -1
  99. data/lib/kotoshu/version.rb +1 -1
  100. data/lib/kotoshu.rb +2 -0
  101. data/scripts/check_russian_cefr.rb +34 -0
  102. data/scripts/convert_fasttext_to_onnx.py +317 -0
  103. data/scripts/diagnose_cefr.rb +41 -0
  104. data/scripts/download_and_convert_models.sh +262 -0
  105. data/scripts/examine_kelly_excel.rb +45 -0
  106. data/scripts/extract_vocabularies.rb +327 -0
  107. data/scripts/parse_kelly.rb +313 -0
  108. data/scripts/requirements.txt +12 -0
  109. data/scripts/setup_python.sh +76 -0
  110. data/scripts/test_cefr_cleaning.rb +27 -0
  111. data/scripts/test_excel_regex.rb +36 -0
  112. data/scripts/test_new_regex.rb +24 -0
  113. data/scripts/test_quotes.rb +11 -0
  114. data/scripts/test_unicode_regex.rb +36 -0
  115. data/sig/kotoshu.rbs +71 -3
  116. metadata +79 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e247c0934704839d352ba2f01044bc3fe696f8806dd397451896e3bf13c67e0e
4
- data.tar.gz: 9aeedc6f0b68331f8ded8c1ebd1f2b4351c588acc076a9dd6ba650c6007b7a34
3
+ metadata.gz: b10c4a6e93c03192123ee21a205b9849b96fa2e25ddff6fb42700ec6e6eac68f
4
+ data.tar.gz: 74731b6b25982128ac06f7f0b212922f60702677605ca0805ced57b900aa100e
5
5
  SHA512:
6
- metadata.gz: f94d4bfff7e7c160f7cf79e1f278d233ef47aab18400d1b71ae1a07260976c7bd4f8b56f5b611c1845478982fc91a3e6799c9b84deba69484fc8694f1109b093
7
- data.tar.gz: 4ea43cec8c39463ee911592ef0d3ba2ed6fe5b74f600b60a58b279298b6d19afcbc8e7279716f72985800e633986f6908c72c361eceffaccc01997436f9647ab
6
+ metadata.gz: ee233628af8159f5d5bbb2ec9c6a1d80aa487ab5d3b93e0928e3ebb8121c111b5a4875f7a0f2c9a95903d5f35702189c84739cb39ced6d8b8c4a4aadc610cd2f
7
+ data.tar.gz: 8cf82e5e7a0677790b4c432c5e0acd28e51a86835c4b9be366665d361c4cf3a93681683753d7cb67335b6e6846cf773fd3c90df7e8a6f96abad1822ac5404f79