kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "digest"
5
+
6
+ module Kotoshu
7
+ module Integrity
8
+ # Parsed view of a content repo's `manifest.json`.
9
+ #
10
+ # Format (per TODO.impl/09-integrity-security.md task 1):
11
+ #
12
+ # {
13
+ # "version": 1,
14
+ # "generated_at": "2026-06-25T10:00:00Z",
15
+ # "resources": {
16
+ # "en/spelling/index.dic": {
17
+ # "size": 49568,
18
+ # "sha256": "ab12...",
19
+ # "language": "en",
20
+ # "type": "spelling",
21
+ # "license": "LGPL/MPL/GPL",
22
+ # "source": "SCROLL"
23
+ # }
24
+ # }
25
+ # }
26
+ #
27
+ # Construction:
28
+ #
29
+ # manifest = Manifest.parse(json_string)
30
+ # manifest.fetch("en/spelling/index.dic") # => Entry or nil
31
+ # manifest.verify_content!("en/spelling/index.dic", bytes) # raises on mismatch
32
+ #
33
+ # `Manifest.load(url, http:)`, returns nil when the manifest 404s (graceful
34
+ # degradation — see module docs).
35
+ class Manifest
36
+ Entry = Struct.new(:path, :sha256, :size, :language, :type, :license, :source,
37
+ keyword_init: true) do
38
+ def verify?(content)
39
+ Digest::SHA256.hexdigest(content) == sha256
40
+ end
41
+ end
42
+
43
+ # Parse a manifest JSON string. Returns an empty Manifest if the JSON
44
+ # is parseable but has no resources (caller treats as "no constraints").
45
+ def self.parse(json)
46
+ data = JSON.parse(json)
47
+ entries = {}
48
+ (data["resources"] || {}).each do |path, fields|
49
+ entries[path] = Entry.new(
50
+ path: path,
51
+ sha256: fields["sha256"],
52
+ size: fields["size"],
53
+ language: fields["language"],
54
+ type: fields["type"],
55
+ license: fields["license"],
56
+ source: fields["source"]
57
+ )
58
+ end
59
+ new(entries, version: data["version"], generated_at: data["generated_at"])
60
+ rescue JSON::ParserError => e
61
+ raise Kotoshu::IntegrityError.new(
62
+ "manifest",
63
+ expected: "<valid JSON>",
64
+ actual: "<parse error: #{e.message}>"
65
+ )
66
+ end
67
+
68
+ # Fetch and parse a manifest from a URL. Returns nil when the
69
+ # manifest is absent (HTTP 404/410) so callers can fall back to
70
+ # unverified downloads — see module docs. Any other failure
71
+ # (5xx, network error, parse error) raises.
72
+ def self.load(url, http: Kotoshu::Integrity::NetHTTP)
73
+ body = http.get(url)
74
+ return nil if body.nil?
75
+
76
+ parse(body)
77
+ end
78
+
79
+ attr_reader :entries, :version, :generated_at
80
+
81
+ def initialize(entries, version: nil, generated_at: nil)
82
+ @entries = entries
83
+ @version = version
84
+ @generated_at = generated_at
85
+ end
86
+
87
+ def fetch(path)
88
+ @entries[path]
89
+ end
90
+
91
+ def empty?
92
+ @entries.empty?
93
+ end
94
+
95
+ # Verify that content for `path` matches the manifest entry.
96
+ # Raises {Kotoshu::IntegrityError} on mismatch. No-op when the
97
+ # manifest has no entry for `path` (returns nil — caller decides
98
+ # whether to treat absence as failure in strict mode).
99
+ def verify_content!(path, content, url: nil)
100
+ entry = @entries[path]
101
+ return nil unless entry
102
+
103
+ actual = Digest::SHA256.hexdigest(content)
104
+ unless actual == entry.sha256
105
+ raise Kotoshu::IntegrityError.new(
106
+ path,
107
+ expected: entry.sha256,
108
+ actual: actual,
109
+ url: url
110
+ )
111
+ end
112
+ true
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "net/http"
4
+ require "uri"
5
+
6
+ module Kotoshu
7
+ module Integrity
8
+ # Thin wrapper around Net::HTTP so manifest fetches are testable
9
+ # without the network. Returns response body as a String on 2xx,
10
+ # nil on 404/410 (so callers can treat "manifest not published yet"
11
+ # as graceful degradation), and raises on other errors.
12
+ module NetHTTP
13
+ class << self
14
+ def get(url, redirect_limit: 3)
15
+ if Kotoshu.configuration.offline
16
+ raise HttpError,
17
+ "GET #{url} refused: offline mode is enabled (KOTOSHU_OFFLINE=1)"
18
+ end
19
+
20
+ uri = URI(url)
21
+ raise ArgumentError, "Only http/https supported: #{url}" unless
22
+ ["http", "https"].include?(uri.scheme)
23
+
24
+ http = Net::HTTP.new(uri.host, uri.port)
25
+ http.use_ssl = (uri.scheme == "https")
26
+ http.open_timeout = 10
27
+ http.read_timeout = 30
28
+
29
+ request = Net::HTTP::Get.new(uri.request_uri)
30
+ response = http.request(request)
31
+
32
+ case response
33
+ when Net::HTTPSuccess
34
+ response.body
35
+ when Net::HTTPNotFound, Net::HTTPGone
36
+ nil
37
+ when Net::HTTPRedirection
38
+ raise TooManyRedirects if redirect_limit.zero?
39
+
40
+ get(response["location"], redirect_limit: redirect_limit - 1)
41
+ else
42
+ raise HttpError, "GET #{url} failed: #{response.code} #{response.message}"
43
+ end
44
+ end
45
+ end
46
+
47
+ class HttpError < StandardError; end
48
+ class TooManyRedirects < StandardError; end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Integrity
5
+ # Pure value object that decides when an {AuditLog} should rotate
6
+ # and produces the rename plan that realises the rotation.
7
+ #
8
+ # The policy is intentionally IO-free so it can be unit-tested in
9
+ # isolation. {AuditLog#record} consults the policy on every write
10
+ # and executes the returned plan via +FileUtils+.
11
+ #
12
+ # Rotation scheme: keep +rotations+ historical files alongside the
13
+ # current one. On rotation:
14
+ #
15
+ # 1. Drop the oldest rotation slot (<path>.<rotations>).
16
+ # 2. Shift each remaining rotation up by one suffix.
17
+ # 3. Promote the current log to <path>.1.
18
+ #
19
+ # Net effect: current + N rotated files, total bounded at
20
+ # +max_bytes * (rotations + 1)+.
21
+ class RotationPolicy
22
+ DEFAULT_MAX_BYTES = 10 * 1024 * 1024 # 10 MB
23
+ DEFAULT_ROTATIONS = 5
24
+
25
+ attr_reader :max_bytes, :rotations
26
+
27
+ def initialize(max_bytes: DEFAULT_MAX_BYTES, rotations: DEFAULT_ROTATIONS)
28
+ @max_bytes = max_bytes.to_i
29
+ @rotations = rotations.to_i
30
+ raise ArgumentError, "max_bytes must be >= 0" if @max_bytes.negative?
31
+ raise ArgumentError, "rotations must be >= 0" if @rotations.negative?
32
+ end
33
+
34
+ # True when the current file size exceeds the configured ceiling.
35
+ #
36
+ # @param current_size [Integer] Bytes of the current log file.
37
+ def rotate?(current_size)
38
+ current_size > max_bytes
39
+ end
40
+
41
+ # Plan the file operations required to rotate +path+.
42
+ #
43
+ # Returns a hash with two keys:
44
+ #
45
+ # +:deletes+ — Array<String> of paths to remove (+FileUtils.rm_f+).
46
+ # +:moves+ — Array<[String, String]> of (source, dest) pairs
47
+ # (+FileUtils.mv+).
48
+ #
49
+ # The plan is ordered so that a caller executing the deletes first
50
+ # and then the moves in array order will not clobber any still-live
51
+ # rotation. Sources that happen not to exist are silently skipped
52
+ # by +FileUtils.mv+, so the plan can be applied verbatim.
53
+ #
54
+ # When +rotations+ is zero, the plan deletes the current path; the
55
+ # caller's next append recreates it as an empty file (effectively a
56
+ # truncate).
57
+ #
58
+ # @param path [String] The current log path (e.g. "/x/audit.log").
59
+ def plan_for(path)
60
+ if rotations.positive?
61
+ {
62
+ deletes: [rotated_path(path, rotations)],
63
+ moves: shift_moves(path).push([path, rotated_path(path, 1)])
64
+ }
65
+ else
66
+ { deletes: [path], moves: [] }
67
+ end
68
+ end
69
+
70
+ private
71
+
72
+ # Pairs that shift every existing rotation up by one suffix.
73
+ # Ordered oldest-first so the rename chain doesn't overwrite
74
+ # in-flight files. For +rotations = 5+ this yields:
75
+ #
76
+ # [[p.4, p.5], [p.3, p.4], [p.2, p.3], [p.1, p.2]]
77
+ def shift_moves(path)
78
+ (rotations - 1).downto(1).map do |n|
79
+ [rotated_path(path, n), rotated_path(path, n + 1)]
80
+ end
81
+ end
82
+
83
+ def rotated_path(path, suffix)
84
+ "#{path}.#{suffix}"
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Integrity verification for downloaded resources.
5
+ #
6
+ # Two cooperating pieces:
7
+ #
8
+ # - {Manifest} — parsed view of a content repo's `manifest.json`. Lookup
9
+ # by relative path yields the expected SHA-256 and size.
10
+ # - {AuditLog} — append-only JSON log at `$XDG_DATA_HOME/kotoshu/audit.log` recording
11
+ # every download's URL, size, computed SHA-256, manifest hash (when
12
+ # available), and status.
13
+ #
14
+ # Caches call `Manifest.verify_content!` after each download. If the
15
+ # manifest is absent (the upstream repo hasn't shipped one yet), the
16
+ # caller logs the download with status `"unverified"` and proceeds —
17
+ # graceful degradation. When a manifest IS present and the SHA-256
18
+ # doesn't match, {Kotoshu::IntegrityError} is raised and the corrupt
19
+ # bytes are removed before they reach the cache.
20
+ module Integrity
21
+ autoload :Manifest, "kotoshu/integrity/manifest"
22
+ autoload :AuditLog, "kotoshu/integrity/audit_log"
23
+ autoload :NetHTTP, "kotoshu/integrity/net_http"
24
+ autoload :RotationPolicy, "kotoshu/integrity/rotation_policy"
25
+ end
26
+ end
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "kotoshu"
4
+
5
+ module Kotoshu
6
+ # Jekyll integration (plan 89, item 4). Soft dependency: jekyll is
7
+ # NOT in kotoshu.gemspec; the generator soft-requires it exactly
8
+ # like Models::OnnxModel soft-requires onnxruntime.
9
+ module Jekyll
10
+ # True when jekyll was requireable at load time.
11
+ JEKYLL_LOADED = begin
12
+ require "jekyll"
13
+ true
14
+ rescue LoadError
15
+ false
16
+ end
17
+
18
+ # Raised when the generator is used without jekyll.
19
+ class JekyllUnavailable < Kotoshu::Error
20
+ def initialize
21
+ super("Kotoshu's Jekyll generator needs the jekyll gem. " \
22
+ "Add gem 'jekyll' to your Gemfile.")
23
+ end
24
+ end
25
+
26
+ if JEKYLL_LOADED
27
+ # Jekyll generator checking posts and drafts; a new spelling
28
+ # error fails the build. Baselines are respected when
29
+ # .kotoshu-baseline.json sits in the site source (the same file
30
+ # `kotoshu baseline init` writes), so existing errors do not
31
+ # block the build until they are fixed.
32
+ #
33
+ # @example _config.yml
34
+ # plugins:
35
+ # - kotoshu
36
+ #
37
+ # Users register this class in a plugins file:
38
+ # require "kotoshu/jekyll"
39
+ class Generator < ::Jekyll::Generator
40
+ safe true
41
+ priority :low
42
+
43
+ def generate(site)
44
+ documents = site.posts.docs
45
+ if documents.empty?
46
+ ::Jekyll.logger.info "kotoshu:", "no posts or drafts to check"
47
+ return
48
+ end
49
+
50
+ baseline = load_baseline(site)
51
+ failures = []
52
+ documents.each do |document|
53
+ # Same plan-116 budget as the CLI/rake paths: covered
54
+ # occurrences skip the suggestion sweep.
55
+ filter = baseline&.suggestions_filter_for(document.relative_path.to_s)
56
+ result = Kotoshu.check(document.content, suggestions_filter: filter)
57
+ application = baseline&.apply(result, file: document.relative_path.to_s)
58
+ result = application.result if application
59
+ failures.concat(failure_lines(document, result))
60
+ end
61
+
62
+ return if failures.empty?
63
+
64
+ raise "kotoshu: spelling errors in #{failures.size} location(s):\n" +
65
+ failures.join("\n")
66
+ end
67
+
68
+ private
69
+
70
+ # Load the baseline from the site source when present.
71
+ #
72
+ # @param site [Jekyll::Site] the site being built
73
+ # @return [Baseline::Store, nil]
74
+ def load_baseline(site)
75
+ path = File.join(site.source, Baseline::Store::DEFAULT_FILENAME)
76
+ File.exist?(path) ? Baseline::Store.load(path) : nil
77
+ end
78
+
79
+ # One failure line per misspelling.
80
+ #
81
+ # @param document [Jekyll::Document] the checked document
82
+ # @param result [Models::Result::DocumentResult] check result
83
+ # @return [Array<String>]
84
+ def failure_lines(document, result)
85
+ result.each_error.map do |error|
86
+ suggestions = error.has_suggestions? ? " -> #{error.top_suggestions(3).join(', ')}" : ""
87
+ " #{document.relative_path}: #{error.word}#{suggestions}"
88
+ end
89
+ end
90
+ end
91
+ else
92
+ # Placeholder raising a friendly error when jekyll is missing;
93
+ # keeps the constant resolvable for tooling.
94
+ class Generator
95
+ def generate(*)
96
+ raise JekyllUnavailable
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ # Base class for keyboard layouts
6
+ #
7
+ # Each layout defines key positions and provides distance calculations
8
+ # for typo detection and suggestion ranking in spell checking.
9
+ #
10
+ # @example Using a keyboard layout
11
+ # layout = Keyboard::Layouts::QWERTY.new
12
+ # layout.distance('q', 'w') # => 1 (adjacent keys)
13
+ # layout.distance('q', 'p') # => 8 (far apart)
14
+ # layout.adjacent_keys('q') # => ['w', 'a', 's']
15
+ #
16
+ # @example Checking language support
17
+ # qwerty = Keyboard::Layouts::QWERTY.new
18
+ # qwerty.supports_language?('en') # => true
19
+ # qwerty.supports_language?('de') # => false
20
+ #
21
+ class Layout
22
+ # @return [String] the name of this keyboard layout
23
+ attr_reader :name
24
+
25
+ # @return [Array<String>] list of language codes this layout supports
26
+ attr_reader :language_codes
27
+
28
+ # @return [Hash] mapping of key characters to [row, col] positions
29
+ attr_reader :key_positions
30
+
31
+ # Initialize a keyboard layout
32
+ #
33
+ # @param name [String] the name of the layout
34
+ # @param language_codes [Array<String>] list of language codes this layout supports
35
+ # @param key_positions [Hash] mapping of key characters to [row, col] positions
36
+ def initialize(name:, language_codes:, key_positions:)
37
+ @name = name
38
+ @language_codes = Array(language_codes).freeze
39
+ @key_positions = key_positions.freeze
40
+ end
41
+
42
+ # Get position [row, col] for a key
43
+ #
44
+ # @param key [String] the key character to look up
45
+ # @return [Array<Integer>, nil] the [row, col] position, or nil if key not found
46
+ def position(key)
47
+ @key_positions[key.downcase]
48
+ end
49
+
50
+ # Calculate Manhattan distance between two keys
51
+ #
52
+ # Manhattan distance is the sum of absolute differences of row and column:
53
+ # distance = abs(row1 - row2) + abs(col1 - col2)
54
+ #
55
+ # @param key1 [String] first key character
56
+ # @param key2 [String] second key character
57
+ # @return [Integer] Manhattan distance (0 if same key, Float::INFINITY if either key not found)
58
+ def distance(key1, key2)
59
+ pos1 = position(key1)
60
+ pos2 = position(key2)
61
+
62
+ return Float::INFINITY unless pos1 && pos2
63
+
64
+ (pos1[0] - pos2[0]).abs + (pos1[1] - pos2[1]).abs
65
+ end
66
+
67
+ # Check if layout supports a language
68
+ #
69
+ # Supports both exact matching and language variant matching.
70
+ # For example, if 'en' is supported, then 'en-US', 'en-GB', etc. are also supported.
71
+ #
72
+ # @param language_code [String] the language code to check (e.g., 'en', 'en-US', 'de')
73
+ # @return [Boolean] true if this layout supports the language
74
+ def supports_language?(language_code)
75
+ # Try exact match first
76
+ return true if @language_codes.include?(language_code)
77
+
78
+ # Try base language match (e.g., 'en' for 'en-US')
79
+ base_lang = language_code.to_s.split('-').first
80
+ @language_codes.include?(base_lang)
81
+ end
82
+
83
+ # Get adjacent keys for a given key (within 1 unit distance)
84
+ #
85
+ # Adjacent keys are those that are directly next to the given key
86
+ # horizontally or vertically (not diagonal).
87
+ #
88
+ # @param key [String] the key character to find adjacent keys for
89
+ # @return [Array<String>] list of adjacent key characters
90
+ def adjacent_keys(key)
91
+ pos = position(key)
92
+ return [] unless pos
93
+
94
+ @key_positions.select do |k, p|
95
+ next if k == key
96
+
97
+ ((p[0] - pos[0]).abs + (p[1] - pos[1]).abs) == 1
98
+ end.keys
99
+ end
100
+
101
+ # String representation of the layout
102
+ #
103
+ # @return [String] layout name
104
+ def to_s
105
+ "Keyboard::#{@name}"
106
+ end
107
+
108
+ # Inspect method for debugging
109
+ #
110
+ # @return [String] detailed inspection string
111
+ def inspect
112
+ "#<#{self.class} name=#{@name} languages=#{@language_codes.join(',')}>"
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Arabic 101 keyboard layout (standard IBM PC Arabic layout).
7
+ #
8
+ # The Arabic letters sit on the QWERTY key faces: the number row
9
+ # is shared with ASCII, and ض ص ث ق ف غ ع ه خ ح ج د / ش س ي ب ل ا
10
+ # ت ن م ك ط / ئ ء ؤ ر لا ى ة و ز ظ fill the three letter rows.
11
+ # Diacritics (tashkeel) are dead keys and are not in the grid;
12
+ # the hamza carriers أ إ آ and the ه/ة ي/ى spelling slips are
13
+ # handled at the normalizer level, not as physical keys.
14
+ #
15
+ # The key grid is mirrored from the models repo eval harness
16
+ # (kotoshu/models-fasttext-onnx eval/noise.py _AR_101, plan 83
17
+ # batch 2 of the model coverage expansion, verified there
18
+ # against the kbdlayout.info Arabic 101 driver table) so
19
+ # typo-adjacency used for suggestion ranking matches the
20
+ # adjacency used to evaluate the models.
21
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
22
+ # the sync.
23
+ #
24
+ # Languages: ar, ar-SA
25
+ class Arabic101 < Layout
26
+ # Key positions for the Arabic 101 layout.
27
+ # Each key maps to [row, column] coordinates.
28
+ KEY_POSITIONS = {
29
+ # Number row
30
+ 'ذ' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
31
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
32
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
33
+ '=' => [0, 12],
34
+ # Top row (ض ص ث ق ف غ ع ه خ ح ج د)
35
+ 'ض' => [1, 0], 'ص' => [1, 1], 'ث' => [1, 2], 'ق' => [1, 3],
36
+ 'ف' => [1, 4], 'غ' => [1, 5], 'ع' => [1, 6], 'ه' => [1, 7],
37
+ 'خ' => [1, 8], 'ح' => [1, 9], 'ج' => [1, 10], 'د' => [1, 11],
38
+ # Home row (ش س ي ب ل ا ت ن م ك ط)
39
+ 'ش' => [2, 0], 'س' => [2, 1], 'ي' => [2, 2], 'ب' => [2, 3],
40
+ 'ل' => [2, 4], 'ا' => [2, 5], 'ت' => [2, 6], 'ن' => [2, 7],
41
+ 'م' => [2, 8], 'ك' => [2, 9], 'ط' => [2, 10],
42
+ # Bottom row (ئ ء ؤ ر لا ى ة و ز ظ)
43
+ 'ئ' => [3, 0], 'ء' => [3, 1], 'ؤ' => [3, 2], 'ر' => [3, 3],
44
+ 'لا' => [3, 4], 'ى' => [3, 5], 'ة' => [3, 6], 'و' => [3, 7],
45
+ 'ز' => [3, 8], 'ظ' => [3, 9]
46
+ }.freeze
47
+
48
+ # Initialize the Arabic 101 layout.
49
+ def initialize
50
+ super(
51
+ name: 'Arabic-101',
52
+ language_codes: %w[ar ar-SA],
53
+ key_positions: KEY_POSITIONS
54
+ )
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # AZERTY keyboard layout
7
+ #
8
+ # Standard AZERTY layout used for:
9
+ # - French (fr, fr-FR)
10
+ # - Belgium (be)
11
+ #
12
+ # Key differences from QWERTY:
13
+ # - a and q are swapped (a/q → q/a)
14
+ # - z and w are swapped (z/w → w/z)
15
+ # - Number row is shifted (requires Shift for numbers)
16
+ # - Has accent keys: é, à, ç, è, ù
17
+ #
18
+ # Key positions use [row, col] coordinates where:
19
+ # - row 0: number/symbol row (²&é"'(-...)
20
+ # - row 1: top row (azerty...)
21
+ # - row 2: home row (qsdfg...)
22
+ # - row 3: bottom row (wxcvb...) - note: w is here
23
+ class AZERTY < Layout
24
+ # Key positions for AZERTY layout
25
+ # Each key maps to [row, column] coordinates
26
+ KEY_POSITIONS = {
27
+ # Top row (number/symbol row - numbers require Shift)
28
+ '`' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3], '4' => [0, 4],
29
+ '5' => [0, 5], '6' => [0, 6], '7' => [0, 7], '8' => [0, 8], '9' => [0, 9],
30
+ '0' => [0, 10], ')' => [0, 11], '=' => [0, 12],
31
+ # Top row (AZERTY - note a and q swapped, z and w swapped)
32
+ 'a' => [1, 0], 'z' => [1, 1], 'e' => [1, 2], 'r' => [1, 3], 't' => [1, 4],
33
+ 'y' => [1, 5], 'u' => [1, 6], 'i' => [1, 7], 'o' => [1, 8], 'p' => [1, 9],
34
+ '^' => [1, 10], '$' => [1, 11],
35
+ # Home row (QSDFG - note q is here)
36
+ 'q' => [2, 0], 's' => [2, 1], 'd' => [2, 2], 'f' => [2, 3], 'g' => [2, 4],
37
+ 'h' => [2, 5], 'j' => [2, 6], 'k' => [2, 7], 'l' => [2, 8], 'm' => [2, 9],
38
+ 'ù' => [2, 10],
39
+ # Bottom row (WXCVB - note w is here)
40
+ 'w' => [3, 0], 'x' => [3, 1], 'c' => [3, 2], 'v' => [3, 3], 'b' => [3, 4],
41
+ 'n' => [3, 5], ',' => [3, 6], ';' => [3, 7], ':' => [3, 8], '!' => [3, 9]
42
+ }.freeze
43
+
44
+ # Initialize AZERTY layout
45
+ def initialize
46
+ super(
47
+ name: 'AZERTY',
48
+ language_codes: %w[fr be fr-FR],
49
+ key_positions: KEY_POSITIONS
50
+ )
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Bulgarian BDS keyboard layout (БДС 5237:1978, KBDBUL.DLL).
7
+ #
8
+ # The national Bulgarian phonetic-adjacent ordering: ы у е и ш щ
9
+ # к с д з ц / ь я а о ж г т н в м ч / ю й ъ э ф х п р л б. The
10
+ # Bulgarian-only letters ъ ь are real keys (unlike JCUKEN, where
11
+ # ъ sits top-right and ы is the home-row opener).
12
+ #
13
+ # The key grid is mirrored from the models repo eval harness
14
+ # (kotoshu/models-fasttext-onnx eval/noise.py _BG_BDS, plan 83
15
+ # batch 2 of the model coverage expansion, verified there
16
+ # against the kbdlayout.info KBDBUL driver table) so
17
+ # typo-adjacency used for suggestion ranking matches the
18
+ # adjacency used to evaluate the models.
19
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
20
+ # the sync.
21
+ #
22
+ # Languages: bg, bg-BG
23
+ class BulgarianBds < Layout
24
+ # Key positions for the Bulgarian BDS layout.
25
+ # Each key maps to [row, column] coordinates.
26
+ KEY_POSITIONS = {
27
+ # Number row
28
+ ')' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
29
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
30
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '$' => [0, 11],
31
+ '€' => [0, 12],
32
+ # Top row (ы у е и ш щ к с д з ц)
33
+ 'ы' => [1, 0], 'у' => [1, 1], 'е' => [1, 2], 'и' => [1, 3],
34
+ 'ш' => [1, 4], 'щ' => [1, 5], 'к' => [1, 6], 'с' => [1, 7],
35
+ 'д' => [1, 8], 'з' => [1, 9], 'ц' => [1, 10],
36
+ # Home row (ь я а о ж г т н в м ч)
37
+ 'ь' => [2, 0], 'я' => [2, 1], 'а' => [2, 2], 'о' => [2, 3],
38
+ 'ж' => [2, 4], 'г' => [2, 5], 'т' => [2, 6], 'н' => [2, 7],
39
+ 'в' => [2, 8], 'м' => [2, 9], 'ч' => [2, 10],
40
+ # Bottom row (ю й ъ э ф х п р л б)
41
+ 'ю' => [3, 0], 'й' => [3, 1], 'ъ' => [3, 2], 'э' => [3, 3],
42
+ 'ф' => [3, 4], 'х' => [3, 5], 'п' => [3, 6], 'р' => [3, 7],
43
+ 'л' => [3, 8], 'б' => [3, 9]
44
+ }.freeze
45
+
46
+ # Initialize the Bulgarian BDS layout.
47
+ def initialize
48
+ super(
49
+ name: 'Bulgarian-BDS',
50
+ language_codes: %w[bg bg-BG],
51
+ key_positions: KEY_POSITIONS
52
+ )
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end