kotoshu 1.0.6 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/.kotoshu-baseline.json +1 -1
  3. data/CHANGELOG.md +32 -0
  4. data/Cargo.lock +6 -6
  5. data/TODO.impl/00-vision.md +46 -0
  6. data/TODO.impl/146-realword-confusion-detection.md +44 -0
  7. data/TODO.impl/147-bcp47-gem-half.md +42 -0
  8. data/TODO.impl/148-checks-framework.md +47 -0
  9. data/conformance/vectors.jsonl +207 -207
  10. data/data/ATTRIBUTION.md +78 -0
  11. data/data/README.md +135 -0
  12. data/docs/ARCHITECTURE_IMPROVEMENTS.md +801 -0
  13. data/docs/ARCHITECTURE_IMPROVEMENTS_PLAN.md +505 -0
  14. data/docs/KOTOSHU_FULL_PLAN.md +970 -0
  15. data/docs/KOTOSHU_LANGUAGETOOL_GAPS.md +917 -0
  16. data/docs/KOTOSHU_SOLIDIFICATION_PLAN.md +1650 -0
  17. data/docs/ONNX_FUNCTIONAL_VERIFICATION.md +149 -0
  18. data/docs/RELEASING.md +78 -0
  19. data/docs/TDD_ITERATION_STRATEGY.md +1123 -0
  20. data/docs/architecture-overview.md +473 -0
  21. data/docs/architecture.md +821 -0
  22. data/docs/embeddings_architecture_v2_plan.md +861 -0
  23. data/docs/getting-started.md +393 -0
  24. data/docs/language-modules.md +361 -0
  25. data/docs/onnx-implementation.md +166 -0
  26. data/docs/performance.md +488 -0
  27. data/docs/plugins.md +750 -0
  28. data/examples/ru_russian_example.rb +1 -2
  29. data/ext/kotoshu_native/Cargo.toml +2 -2
  30. data/lib/kotoshu/cache/frequency_cache.rb +29 -3
  31. data/lib/kotoshu/cache/language_cache.rb +63 -6
  32. data/lib/kotoshu/cache/staged_languages.rb +3 -12
  33. data/lib/kotoshu/checks/base.rb +56 -0
  34. data/lib/kotoshu/checks/finding.rb +19 -0
  35. data/lib/kotoshu/checks/grammar_check.rb +43 -0
  36. data/lib/kotoshu/checks/registry.rb +59 -0
  37. data/lib/kotoshu/checks/spelling_check.rb +30 -0
  38. data/lib/kotoshu/checks.rb +16 -0
  39. data/lib/kotoshu/data/common_words_loader.rb +37 -7
  40. data/lib/kotoshu/data/confusion/pinyin.json +1 -0
  41. data/lib/kotoshu/keyboard/layout.rb +6 -6
  42. data/lib/kotoshu/keyboard/layouts/chinese_ime.rb +69 -0
  43. data/lib/kotoshu/keyboard/registry.rb +26 -0
  44. data/lib/kotoshu/keyboard.rb +6 -0
  45. data/lib/kotoshu/multi_language_checker.rb +11 -2
  46. data/lib/kotoshu/resource_bundle.rb +5 -1
  47. data/lib/kotoshu/resource_manager.rb +27 -2
  48. data/lib/kotoshu/source_registry.rb +3 -0
  49. data/lib/kotoshu/spellchecker.rb +9 -5
  50. data/lib/kotoshu/suggestions/confusion_set.rb +68 -0
  51. data/lib/kotoshu/suggestions/frequency_provider.rb +96 -19
  52. data/lib/kotoshu/suggestions/generator.rb +11 -0
  53. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +10 -0
  54. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +32 -3
  55. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +45 -11
  56. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +310 -34
  57. data/lib/kotoshu/suggestions.rb +18 -1
  58. data/lib/kotoshu/version.rb +1 -1
  59. data/lib/kotoshu.rb +1 -0
  60. data/scripts/check_russian_cefr.rb +34 -0
  61. data/scripts/convert_fasttext_to_onnx.py +317 -0
  62. data/scripts/diagnose_cefr.rb +41 -0
  63. data/scripts/download_and_convert_models.sh +262 -0
  64. data/scripts/examine_kelly_excel.rb +45 -0
  65. data/scripts/extract_vocabularies.rb +327 -0
  66. data/scripts/parse_kelly.rb +313 -0
  67. data/scripts/requirements.txt +12 -0
  68. data/scripts/setup_python.sh +76 -0
  69. data/scripts/test_cefr_cleaning.rb +27 -0
  70. data/scripts/test_excel_regex.rb +36 -0
  71. data/scripts/test_new_regex.rb +24 -0
  72. data/scripts/test_quotes.rb +11 -0
  73. data/scripts/test_unicode_regex.rb +36 -0
  74. metadata +46 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 739d346be14f0f6d49ea7084f1ee7fb14f3c14450546d4c8e73b568818d75db5
4
- data.tar.gz: 71b99b4af5f814a8e25736fac52f12243a120517d087bb3c250f0c52691a75e3
3
+ metadata.gz: 2e0e6dc04a08737e455f97e405334cc8f4f620b94241fffd902c6b6e3c5ca695
4
+ data.tar.gz: 413bf5bde3121c709fb94ab345bafcef58880ae2c1b408275ba4a6e8d3d206f8
5
5
  SHA512:
6
- metadata.gz: 9f832600a2d7b627688cd863ace72a54dd08cdc0a5cd0b182645e00c26c66d0fa39686effeee50bc2d2220c8e01cb0cf369f697b1b4fdc919dffcf46fbe52939
7
- data.tar.gz: b7a554c59a8c4497bf53b899a112463d28414ae8773f9f3fe4d2c19f9535c942eefcc6812a38a130089a06ee24126b18e1e76e61f75d1eb51e0079360b519074
6
+ metadata.gz: 0b329177f96ed12abe9c34f4f2186a9c98511aa67e6d9904bba16f66ae321620a91fd93580e2267cc0c17fa2ea3dddcb48f467aaf0e645107aa6137963b11213
7
+ data.tar.gz: 7c32da93203d90e90eda84b62900f7d2b1cb3f21feda9a8b8d89b7f8ab561e119e046054f25b27a02fc6775896efaf1e5b550cc51d50efe46ee6dc80608996e7