kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,420 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Russian language implementation.
6
+ #
7
+ # Supports multiple dialects: ru-RU, ru-BY, ru-KZ, ru-KG, ru-MD
8
+ #
9
+ # Full Hunspell integration with spell checking, POS tagging, and grammar rules
10
+ # specifically handling Russian Cyrillic script and case system.
11
+ class Russian < Language::Base
12
+ # Russian spell checker with Hunspell integration.
13
+ class SpellChecker < Components::SpellChecker
14
+ attr_reader :aff_path, :dic_path, :script
15
+
16
+ def initialize(aff_path:, dic_path:, script: :cyrillic, encoding: 'UTF-8')
17
+ @aff_path = aff_path
18
+ @dic_path = dic_path
19
+ @script = script
20
+ @encoding = encoding
21
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
22
+ end
23
+
24
+ def check(word)
25
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
26
+
27
+ first_form = @lookuper.good_forms(word).first
28
+ if first_form
29
+ { found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
30
+ else
31
+ { found: false, stem: nil, flags: [] }
32
+ end
33
+ end
34
+
35
+ def suggest(word, max_suggestions: 10)
36
+ return [] if word.nil? || word.empty?
37
+
38
+ first_form = @lookuper.good_forms(word).first
39
+ return [] if first_form
40
+
41
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
42
+ end
43
+
44
+ def correct?(word)
45
+ check(word)[:found]
46
+ end
47
+
48
+ def lookuper
49
+ @lookuper
50
+ end
51
+
52
+ private
53
+
54
+ def calculate_distance(a, b)
55
+ return a.length if b.empty?
56
+ return b.length if a.empty?
57
+
58
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
59
+ (1..b.length).each { |j| matrix[0][j] = j }
60
+ (1..a.length).each do |i|
61
+ (1..b.length).each do |j|
62
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
63
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
64
+ end
65
+ end
66
+ matrix[a.length][b.length]
67
+ end
68
+
69
+ def calculate_score(original, suggestion, rank)
70
+ distance = calculate_distance(original, suggestion)
71
+ max_len = [original.length, suggestion.length].max
72
+ distance_score = 1.0 - (distance.to_f / max_len)
73
+ rank_penalty = rank * 0.05
74
+ [distance_score - rank_penalty, 0.0].max
75
+ end
76
+
77
+ def generate_suggestions(word, _max_suggestions)
78
+ variations = []
79
+
80
+ # Russian character substitutions (common Cyrillic errors)
81
+ cyrillic_substitutions = {
82
+ 'а' => %w[о и е я],
83
+ 'о' => %w[а е и],
84
+ 'е' => %w[и э а],
85
+ 'и' => %w[е е],
86
+ 'п' => %w[т к],
87
+ 'т' => %w[п д],
88
+ 'к' => %w[г х],
89
+ 'н' => %w[т п],
90
+ 'с' => %w[з ш],
91
+ 'ш' => %w[с щ],
92
+ 'щ' => %w[ш],
93
+ 'б' => %w[п в],
94
+ 'в' => %w[б ф],
95
+ 'ф' => %w[в в],
96
+ 'д' => %w[т],
97
+ 'г' => %w[к х],
98
+ 'х' => %w[г к],
99
+ 'я' => %w[а е],
100
+ 'ю' => %w[у],
101
+ 'ё' => %w[е],
102
+ 'ж' => %w[з ш],
103
+ 'з' => %w[с ж],
104
+ 'ь' => %w[ъ],
105
+ 'ъ' => %w[ь],
106
+ }
107
+
108
+ word.chars.each_with_index do |char, i|
109
+ next unless cyrillic_substitutions.key?(char.downcase)
110
+
111
+ cyrillic_substitutions[char.downcase].each do |sub|
112
+ substituted = word.dup
113
+ substituted[i] = sub
114
+ variations << substituted if @lookuper.good_forms(substituted).first
115
+ end
116
+ end
117
+
118
+ # Doubled and deleted letters
119
+ word.chars.each_with_index do |char, i|
120
+ next if i == 0
121
+
122
+ doubled = word.dup
123
+ doubled.insert(i, char)
124
+ variations << doubled if @lookuper.good_forms(doubled).first
125
+ end
126
+
127
+ (0...word.length).each do |i|
128
+ deleted = word.dup
129
+ deleted.slice!(i)
130
+ next if deleted.empty?
131
+
132
+ variations << deleted if @lookuper.good_forms(deleted).first
133
+ end
134
+
135
+ variations.uniq!
136
+ variations.map do |suggestion|
137
+ { word: suggestion, distance: calculate_distance(word, suggestion),
138
+ score: calculate_score(word, suggestion, 0) }
139
+ end.sort_by { |s| s[:distance] }
140
+ end
141
+ end
142
+
143
+ # Russian tokenizer with abbreviation handling.
144
+ class Tokenizer < Language::Tokenizer::RussianTokenizer
145
+ end
146
+
147
+ # Russian POS tagger.
148
+ class POSTagger < Components::PosTagger
149
+ FLAG_TO_POS = {
150
+ 'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
151
+ 'S' => 'NOUN', 'Sub' => 'NOUN',
152
+ 'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
153
+ 'VBP' => 'VERB', 'VBZ' => 'VERB',
154
+ 'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
155
+ 'Adj' => 'ADJ',
156
+ 'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
157
+ 'Adv' => 'ADV',
158
+ 'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
159
+ 'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
160
+ 'WP' => 'PRON', 'WP$' => 'PRON_POSS',
161
+ 'Pro' => 'PRON',
162
+ 'I' => 'PREP', 'IN' => 'PREP',
163
+ 'Präp' => 'PREP',
164
+ 'C' => 'CONJ', 'CC' => 'CONJ',
165
+ 'Conj' => 'CONJ',
166
+ 'U' => 'PART', 'RP' => 'PART',
167
+ 'Pt' => 'PART',
168
+ 'INTJ' => 'INTJ', 'UH' => 'INTJ',
169
+ 'Int' => 'INTJ',
170
+ 'CD' => 'NUM',
171
+ 'FW' => 'X',
172
+ 'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
173
+ '?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
174
+ }.freeze
175
+
176
+ attr_reader :aff_path, :dic_path, :script
177
+
178
+ def initialize(aff_path:, dic_path:, script: :cyrillic, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
179
+ @aff_path = aff_path
180
+ @dic_path = dic_path
181
+ @script = script
182
+ @encoding = encoding
183
+ @flag_mapping = flag_mapping
184
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
185
+ @lookup_cache = {}
186
+ end
187
+
188
+ def tag(tokens)
189
+ return [] if tokens.nil? || tokens.empty?
190
+
191
+ tokens.map do |token|
192
+ word = token[:token]
193
+ if word.nil? || word.empty?
194
+ token.merge(pos_tag: nil, lemma: nil)
195
+ else
196
+ lookup_result = lookup_with_pos(word)
197
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
198
+ end
199
+ end
200
+ end
201
+
202
+ def flag_mapping
203
+ @flag_mapping
204
+ end
205
+
206
+ def flag_mapping=(mapping)
207
+ @flag_mapping = mapping
208
+ end
209
+
210
+ def clear_cache
211
+ @lookup_cache.clear
212
+ end
213
+
214
+ def cache_size
215
+ @lookup_cache.size
216
+ end
217
+
218
+ private
219
+
220
+ def lookup_with_pos(word)
221
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
222
+ return @lookup_cache[word] if @lookup_cache.key?(word)
223
+
224
+ first_form = @lookuper.good_forms(word).first
225
+ pos_tag = derive_pos_tag(first_form)
226
+ cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
227
+ @lookup_cache[word] = cache_result
228
+ cache_result
229
+ end
230
+
231
+ def derive_pos_tag(result)
232
+ return nil unless result
233
+
234
+ flags = result.flags&.to_a || []
235
+ return guess_pos_from_affix(result) if flags.empty?
236
+
237
+ flags.each do |flag|
238
+ pos_tag = flag_to_pos(flag)
239
+ return pos_tag if pos_tag
240
+ end
241
+ guess_pos_from_affix(result)
242
+ end
243
+
244
+ def flag_to_pos(flag)
245
+ return @flag_mapping[flag] if @flag_mapping.key?(flag)
246
+
247
+ first_char = flag[0]
248
+ @flag_mapping[first_char]
249
+ end
250
+
251
+ def guess_pos_from_affix(result)
252
+ suffix = result.suffix
253
+ return guess_pos_from_suffix(suffix) if suffix
254
+
255
+ nil
256
+ end
257
+
258
+ def guess_pos_from_suffix(suffix)
259
+ # Russian suffix patterns
260
+ return 'VERB' if suffix.match?(/^(ть|ти|чь|л|ла|ло|ли|ют|ют|ешь|ишь|им|ите|ат|ят)$/)
261
+ return 'ADV' if suffix.match?(/^(о|е|и)$/)
262
+ return 'NOUN' if suffix.match?(/^(ость|ение|ание|ка|ник|чик|щик|ство|тель|ение|ство)$/)
263
+ return 'ADJ' if suffix.match?(/^(ый|ий|ой|ое|ая|ое|ые|их|ем|им|ом|ого|ому)$/)
264
+
265
+ nil
266
+ end
267
+ end
268
+
269
+ # Russian grammar rules module.
270
+ module GrammarRules
271
+ class Rule
272
+ attr_reader :id, :name, :description
273
+
274
+ def initialize(id, name, description)
275
+ @id = id
276
+ @name = name
277
+ @description = description
278
+ end
279
+
280
+ def check(tokens)
281
+ raise NotImplementedError, "#{self.class} must implement #check"
282
+ end
283
+ end
284
+
285
+ # Rule: Verbal aspect consistency
286
+ class VerbalAspectRule < Rule
287
+ IMPERFECTIVE_SUFFIXES = %w[ать ять].freeze
288
+ PERFECTIVE_SUFFIXES = %w[ить по].freeze
289
+
290
+ def initialize
291
+ super('RU_VERBAL_ASPECT', 'Verbal Aspect', 'Russian verbs should use consistent aspect (imperfective/perfective).')
292
+ end
293
+
294
+ def check(_tokens)
295
+ # Simplified implementation
296
+ []
297
+ end
298
+ end
299
+
300
+ # Rule: Case agreement
301
+ class CaseAgreementRule < Rule
302
+ def initialize
303
+ super('RU_CASE_AGREEMENT', 'Case Agreement', 'Nouns, adjectives, and verbs must agree in case.')
304
+ end
305
+
306
+ def check(_tokens)
307
+ # Simplified implementation
308
+ []
309
+ end
310
+ end
311
+
312
+ class RuleRegistry
313
+ class << self
314
+ def default_rules
315
+ [VerbalAspectRule.new, CaseAgreementRule.new]
316
+ end
317
+
318
+ def get_rule(id)
319
+ default_rules.find { |rule| rule.id == id }
320
+ end
321
+ end
322
+ end
323
+ end
324
+
325
+ # Registration
326
+ register "ru"
327
+ register "ru-RU"
328
+ register "ru-BY"
329
+ register "ru-KZ"
330
+ register "ru-KG"
331
+ register "ru-MD"
332
+
333
+ HUNSPELL_DICTIONARIES = {
334
+ 'ru-RU' => {
335
+ aff: 'spec/integrational/fixtures/ru_RU.aff',
336
+ dic: 'spec/integrational/fixtures/ru_RU.dic'
337
+ }
338
+ }.freeze
339
+
340
+ VARIANT_NAMES = {
341
+ 'RU' => 'Russian',
342
+ 'BY' => 'Belarusian',
343
+ 'KZ' => 'Kazakh',
344
+ 'KG' => 'Kyrgyz',
345
+ 'MD' => 'Moldovan'
346
+ }.freeze
347
+
348
+ def initialize(code: "ru", name: "Russian", variant: nil)
349
+ variant ||= extract_region_code(code)
350
+ super
351
+ @hunspell_paths = resolve_hunspell_paths(code)
352
+ end
353
+
354
+ def description
355
+ return name unless variant
356
+
357
+ variant_name = VARIANT_NAMES[variant] || variant
358
+ "#{name} (#{variant_name})"
359
+ end
360
+
361
+ def tokenizer
362
+ @tokenizer ||= Tokenizer.new
363
+ end
364
+
365
+ def normalizer
366
+ @normalizer ||= Language::Normalizer::Base.new
367
+ end
368
+
369
+ def dictionary_class
370
+ Dictionary::UnixWords
371
+ end
372
+
373
+ def default_dictionary_paths
374
+ case code
375
+ when "ru-RU"
376
+ ["/usr/share/dict/russian"]
377
+ else
378
+ ["/usr/share/dict/words"]
379
+ end
380
+ end
381
+
382
+ def script_type
383
+ :cyrillic
384
+ end
385
+
386
+ def create_spell_checker
387
+ SpellChecker.new(
388
+ aff_path: @hunspell_paths[:aff],
389
+ dic_path: @hunspell_paths[:dic],
390
+ script: :cyrillic
391
+ )
392
+ end
393
+
394
+ def create_tokenizer
395
+ Tokenizer.new
396
+ end
397
+
398
+ def create_pos_tagger
399
+ POSTagger.new(
400
+ aff_path: @hunspell_paths[:aff],
401
+ dic_path: @hunspell_paths[:dic],
402
+ script: :cyrillic,
403
+ flag_mapping: POSTagger::FLAG_TO_POS
404
+ )
405
+ end
406
+
407
+ private
408
+
409
+ def extract_region_code(code)
410
+ return nil unless code.include?("-")
411
+
412
+ code.split("-", 2).last.upcase
413
+ end
414
+
415
+ def resolve_hunspell_paths(code)
416
+ HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['ru-RU']
417
+ end
418
+ end
419
+ end
420
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Slovak language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Slovak diacritics (á ä č ď é í ľ ĺ ň ó ô ŕ š ť
9
+ # ú ý ž) are dead keys over the US QWERTY grid
10
+ # (Keyboard::Layouts::Latin::Slovak), the Polish programmers'
11
+ # pattern the eval harness also models.
12
+ class Slovak < LatinBase
13
+ register "sk"
14
+ register "sk-SK"
15
+
16
+ def initialize(code: "sk", name: "Slovak")
17
+ super
18
+ end
19
+
20
+ def default_dictionary_paths
21
+ ["/usr/share/dict/slovak"]
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Slovenian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. The Slovenian letters č š ž (plus the Croatian
9
+ # đ ć the grid also carries) are real keys on the South-Slavic
10
+ # QWERTZ physical layout (Keyboard::Layouts::Slovenian), shared
11
+ # with Croatian.
12
+ class Slovenian < LatinBase
13
+ register "sl"
14
+ register "sl-SI"
15
+
16
+ def initialize(code: "sl", name: "Slovenian")
17
+ super
18
+ end
19
+
20
+ def default_dictionary_paths
21
+ ["/usr/share/dict/slovenian"]
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Serbian language implementation (plan 100, batch 3).
6
+ #
7
+ # Cyrillic composition like Bulgarian/Ukrainian: Serbian is
8
+ # written in both scripts, but Cyrillic is the constitutional
9
+ # primary and the script the staged dictionary carries — the
10
+ # module registers sr for the Cyrillic grid
11
+ # (Keyboard::Layouts::SerbianCyrillic); Serbian Latin (sr-Latn)
12
+ # has its own module over the distinct staged Latin dictionary
13
+ # (plan 110, Kotoshu::Languages::SerbianLatin). The Serbian
14
+ # letters љ њ ђ ћ џ ј are real keys on that grid.
15
+ class Serbian < LatinBase
16
+ register "sr"
17
+ register "sr-RS"
18
+
19
+ def initialize(code: "sr", name: "Serbian")
20
+ super
21
+ end
22
+
23
+ # Cyrillic-script tokenizer.
24
+ #
25
+ # @return [Language::Tokenizer::CyrillicTokenizer]
26
+ def tokenizer
27
+ @tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
28
+ end
29
+
30
+ def script_type
31
+ :cyrillic
32
+ end
33
+
34
+ def default_dictionary_paths
35
+ ["/usr/share/dict/serbian"]
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Serbian Latin implementation (plan 110).
6
+ #
7
+ # The Latin script variant of Serbian, the last named gap in the
8
+ # corpus after plan 107 opened the staged manifest. The data
9
+ # decided the wiring: the dictionaries manifest stages a distinct
10
+ # sr-Latn dictionary (LibreOffice, wooorm/dictionaries), so
11
+ # sr-Latn resolves its own download rather than riding the
12
+ # Cyrillic sr dictionary — and because sr and sr-Latn differ by
13
+ # script, not vocabulary, a script-subtag code must survive
14
+ # ResourceManager language normalization (plan 110) for the
15
+ # staged Latin files to be reachable at all.
16
+ #
17
+ # Thin composition like its Croatian sibling: shared Latin
18
+ # tokenizer (\p{Latin} covers š đ č ć ž), base normalizer,
19
+ # registry entry. The five Latin digraph letters are real keys on
20
+ # the South-Slavic QWERTZ grid
21
+ # (Keyboard::Layouts::SerbianLatin, the physical arrangement the
22
+ # hr and sl modules share).
23
+ class SerbianLatin < LatinBase
24
+ register "sr-Latn"
25
+
26
+ def initialize(code: "sr-Latn", name: "Serbian (Latin)")
27
+ super
28
+ end
29
+
30
+ def default_dictionary_paths
31
+ ["/usr/share/dict/serbian"]
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Swedish language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Swedish å ä ö are real keys on the Nordic physical layout (Keyboard::Layouts::Latin::Swedish).
9
+ class Swedish < LatinBase
10
+ register "sv"
11
+ register "sv-SE"
12
+
13
+ def initialize(code: "sv", name: "Swedish")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/swedish"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Turkish language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition plus one piece of real care: Turkish case
8
+ # folding. Turkish has a separate dotted/dotless i pair (I/ı and
9
+ # i/İ), so a Unicode-default downcase maps uppercase words onto
10
+ # letter sequences no Turkish dictionary contains. The Turkish
11
+ # normalizer uses Ruby's :turkic case mapping (I -> ı, İ -> i),
12
+ # so İZMİR looks up as izmir and ISTANBUL as ıstanbul.
13
+ #
14
+ # The ı ğ ü ş i ö ç letters are real keys on the Turkish-Q
15
+ # physical layout (Keyboard::Layouts::TurkishQ).
16
+ class Turkish < LatinBase
17
+ register "tr"
18
+ register "tr-TR"
19
+
20
+ def initialize(code: "tr", name: "Turkish")
21
+ super
22
+ end
23
+
24
+ # Turkish-aware normalizer (dotless-i case folding).
25
+ #
26
+ # @return [Language::Normalizer::Turkish]
27
+ def normalizer
28
+ @normalizer ||= Language::Normalizer::Turkish.new
29
+ end
30
+
31
+ def default_dictionary_paths
32
+ ["/usr/share/dict/turkish"]
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Ukrainian language implementation (plan 84, wave 1).
6
+ #
7
+ # Cyrillic composition: the Cyrillic tokenizer keeps
8
+ # Ukrainian-script words (і ї є ґ included; the apostrophe stays a
9
+ # word character for names like Мар'яна). The base normalizer
10
+ # suffices — Ukrainian case folding has no special pairs.
11
+ #
12
+ # Keyboard proximity uses the Ukrainian JCUKEN grid
13
+ # (Keyboard::Layouts::Ukrainian).
14
+ class Ukrainian < LatinBase
15
+ register "uk"
16
+ register "uk-UA"
17
+
18
+ def initialize(code: "uk", name: "Ukrainian")
19
+ super
20
+ end
21
+
22
+ # Cyrillic-script tokenizer.
23
+ #
24
+ # @return [Language::Tokenizer::CyrillicTokenizer]
25
+ def tokenizer
26
+ @tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
27
+ end
28
+
29
+ def script_type
30
+ :cyrillic
31
+ end
32
+
33
+ def default_dictionary_paths
34
+ ["/usr/share/dict/ukrainian"]
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Vietnamese language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Vietnamese tone and vowel marks are dead-key sequences over ASCII letters (VIQR-style typing).
9
+ class Vietnamese < LatinBase
10
+ register "vi"
11
+ register "vi-VN"
12
+
13
+ def initialize(code: "vi", name: "Vietnamese")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/vietnamese"]
19
+ end
20
+ end
21
+ end
22
+ end