kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,631 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Kotoshu
6
+ module Suggestions
7
+ module Strategies
8
+ # Edit distance suggestion strategy with enhanced ranking.
9
+ # Generates suggestions by finding words with small edit distance,
10
+ # ranked by word frequency, keyboard proximity, and common typo patterns.
11
+ #
12
+ # Multi-language support:
13
+ # - Automatically selects keyboard layout based on language_code
14
+ # - Loads frequency data from YAML files (Phase 1) or GitHub (Phase 2)
15
+ # - Supports language-specific typo patterns
16
+ #
17
+ # This is MORE OOP than Spylls which uses standalone functions
18
+ # for edit distance operations.
19
+ #
20
+ # Follows Open-Closed Principle: Extend by adding YAML files,
21
+ # NOT by modifying this class.
22
+ class EditDistanceStrategy < BaseStrategy
23
+ attr_reader :language_code, :keyboard_layout
24
+
25
+ # @param name [String, Symbol] Name of the strategy
26
+ # @param config [Hash] Configuration options
27
+ # @option config [String] :language_code Language code for keyboard layout (default: 'en')
28
+ # @option config [Keyboard::Layout] :keyboard_layout Custom keyboard layout (optional)
29
+ # @option config [Hash] :frequency_tiers Custom frequency tiers (optional)
30
+ # @option config [Integer] :max_distance Maximum edit distance (default: 2)
31
+ # @option config [Integer] :max_results Maximum results to return (default: 10)
32
+ def initialize(name: :edit_distance, language_code: 'en', keyboard_layout: nil,
33
+ frequency_tiers: nil, frequency_provider: nil, **config)
34
+ super(name: name, **config)
35
+ @language_code = language_code
36
+
37
+ # Use OOP registry for keyboard layout lookup
38
+ @keyboard_layout = resolve_keyboard_layout(keyboard_layout)
39
+
40
+ # Frequency data comes from a provider (extracted in TODO 56
41
+ # T5.1 step 3 Phase A) so the strategy constructor no longer
42
+ # performs disk IO or network access. Callers can inject a
43
+ # custom provider for testing.
44
+ @frequency_provider = frequency_provider || FrequencyProvider.new
45
+
46
+ # Use custom frequency tiers if provided, otherwise resolve
47
+ # lazily through the provider on first use.
48
+ if frequency_tiers
49
+ @frequency_tiers = frequency_tiers
50
+ @common_words = Set.new
51
+ else
52
+ @frequency_tiers = nil
53
+ end
54
+ end
55
+
56
+ # Public method to get current keyboard being used
57
+ #
58
+ # @return [Keyboard::Layout] The keyboard layout instance
59
+ def keyboard
60
+ @keyboard_layout
61
+ end
62
+
63
+ # Public method to get keyboard name
64
+ #
65
+ # @return [String] Keyboard layout name
66
+ def keyboard_name
67
+ @keyboard_layout.name
68
+ end
69
+
70
+ # Check if a substitution is a keyboard-adjacent typo
71
+ #
72
+ # @param char1 [String] First character
73
+ # @param char2 [String] Second character
74
+ # @return [Boolean] True if keys are adjacent
75
+ def adjacent_key_typo?(char1, char2)
76
+ @keyboard_layout.adjacent_keys(char1).include?(char2)
77
+ end
78
+
79
+ # Get adjacent keys for a given key
80
+ #
81
+ # @param key [String] The key to find adjacent keys for
82
+ # @return [Array<String>] List of adjacent key characters
83
+ def adjacent_keys(key)
84
+ @keyboard_layout.adjacent_keys(key)
85
+ end
86
+
87
+ # Get frequency bonus for a word
88
+ #
89
+ # @param word [String] The word to check
90
+ # @return [Integer] Frequency bonus (0-200)
91
+ def frequency_bonus(word)
92
+ tiers = frequency_tiers
93
+ return 0 unless tiers
94
+
95
+ word_downcase = word.downcase
96
+
97
+ # Top 50: 200 bonus
98
+ return 200 if tiers[:top_50]&.include?(word_downcase)
99
+
100
+ # Top 200: 100 bonus
101
+ return 100 if tiers[:top_200]&.include?(word_downcase)
102
+
103
+ # Top 1000: 50 bonus
104
+ return 50 if tiers[:top_1000]&.include?(word_downcase)
105
+
106
+ # Not in common words: no bonus
107
+ 0
108
+ end
109
+
110
+ # Frequency tiers used for ranking. Lazy-loads from the
111
+ # provider on first access when not set at construction.
112
+ def frequency_tiers
113
+ @frequency_tiers ||= @frequency_provider.tiers_for(@language_code)
114
+ end
115
+
116
+ # Generate suggestions based on enhanced edit distance scoring.
117
+ #
118
+ # Scoring factors:
119
+ # - Edit distance (primary factor)
120
+ # - Word frequency (common words rank higher)
121
+ # - Keyboard proximity (adjacent key typos rank higher)
122
+ # - Common typo patterns (missing double letters, etc.)
123
+ #
124
+ # @param context [Context] The suggestion context
125
+ # @return [SuggestionSet] Suggestions within max_distance
126
+ def generate(context)
127
+ word = context.word
128
+ max_dist = get_config(:max_distance, 2)
129
+ min_confidence = get_config(:min_confidence, 0.75) # Higher threshold for quality
130
+ min_similarity = get_config(:min_jaro_similarity, 0.70) # Minimum Jaro-Winkler similarity (0.0-1.0)
131
+ min_results = get_config(:min_results, 3) # Always return at least 3 suggestions if available
132
+
133
+ # When the dictionary is case-insensitive, normalize case before
134
+ # edit-distance comparison — otherwise "HELO" can never match
135
+ # "Hello" within distance 2 (case differences alone cost 4).
136
+ # The original dictionary casing is preserved on the returned
137
+ # suggestion (we only normalize for the comparison).
138
+ case_insensitive = dictionary_case_insensitive?(context)
139
+ compare_word = case_insensitive ? word.downcase : word
140
+
141
+ # Calculate enhanced scores for all candidates
142
+ candidates = []
143
+ # Length filter: edit distance cannot be less than the length
144
+ # difference. Skip dictionary words whose length differs from
145
+ # the input by more than max_dist — they can't possibly match.
146
+ # This is the single biggest performance win for this strategy:
147
+ # without it, we pay the full O(m*n) DP cost on every word in
148
+ # the dictionary. When the dictionary is an IndexedDictionary,
149
+ # the lookup is O(buckets_in_range) instead of O(n) scan.
150
+ target_length = compare_word.length
151
+ length_min = target_length - max_dist
152
+ length_max = target_length + max_dist
153
+
154
+ words_to_check = dictionary_words_in_length_range(
155
+ context, min: length_min, max: length_max
156
+ )
157
+
158
+ words_to_check.each do |dict_word|
159
+ next if dict_word == word
160
+
161
+ compare_dict = case_insensitive ? dict_word.downcase : dict_word
162
+ # distance_with_threshold bails out early when the row
163
+ # minimum exceeds max_dist — avoids computing the full
164
+ # DP for clearly-different pairs of similar-length words.
165
+ dist = edit_distance_with_threshold(compare_word, compare_dict, max_dist)
166
+ next unless dist && dist > 0
167
+
168
+ # Calculate enhanced score (lower is better)
169
+ score = calculate_enhanced_score(compare_word, compare_dict, dist)
170
+ candidates << [dict_word, dist, score]
171
+ end
172
+
173
+ # Single-edit sweep: enumerate the distance-1 edits of the
174
+ # misspelling itself (adjacent transposition, substitution,
175
+ # insertion, deletion) and keep the ones the dictionary
176
+ # accepts. The headword loop above can only ever surface raw
177
+ # dictionary stems — without this sweep, valid dictionary
178
+ # FORMS (affixed like "definately" -> "definitely", or
179
+ # capitalization variants like "Teh" -> "The") can never be
180
+ # suggested, no matter how close they are. Transposition is
181
+ # enumerated as a single operation (restricted Damerau), so
182
+ # a swapped pair costs 1, not 2.
183
+ edit_sweep_candidates(context, word, case_insensitive: case_insensitive,
184
+ max_dist: max_dist) do |candidate, dist|
185
+ compare_candidate = case_insensitive ? candidate.downcase : candidate
186
+ score = calculate_enhanced_score(compare_word, compare_candidate, dist)
187
+ candidates << [candidate, dist, score]
188
+ end
189
+
190
+ # Sort by enhanced score (lower is better)
191
+ sorted_candidates = candidates.sort_by { |_, _, score| score }
192
+
193
+ # Drop case variants of the same word, keeping the
194
+ # best-scoring form (first after the sort). The headword
195
+ # loop and the edit sweep can both propose a word — e.g.
196
+ # "the" (stem) and "The" (lookup-valid INITCAP form of the
197
+ # same stem) — and the final SuggestionSet would deduplicate
198
+ # them anyway; deduplicating here keeps the strategy output
199
+ # free of case duplicates and the score range used for
200
+ # confidence normalization honest.
201
+ seen_words = Set.new
202
+ sorted_candidates = sorted_candidates.select { |dict_word, _, _| seen_words.add?(dict_word.downcase) }
203
+
204
+ # Calculate confidence scores with threshold filtering
205
+ if sorted_candidates.empty?
206
+ return SuggestionSet.empty
207
+ end
208
+
209
+ max_score = sorted_candidates.map { |_, _, s| s.to_f }.max
210
+ min_score = sorted_candidates.map { |_, _, s| s.to_f }.min
211
+ score_range = (max_score - min_score).abs
212
+
213
+ # Create suggestions with confidence-based filtering
214
+ suggestions = []
215
+ sorted_candidates.each do |dict_word, dist, score|
216
+ # Normalize score to confidence (0.0 to 1.0)
217
+ # Lower score = higher confidence
218
+ if score_range > 0
219
+ normalized = (score.to_f - min_score) / score_range # 0 to 1
220
+ confidence = 1.0 - normalized # Invert: lower score = higher confidence
221
+ else
222
+ confidence = 1.0
223
+ end
224
+
225
+ # Calculate Jaro-Winkler similarity for additional filtering.
226
+ # Use the same case normalization as the edit distance so the
227
+ # similarity score is consistent with the distance threshold.
228
+ compare_dict = case_insensitive ? dict_word.downcase : dict_word
229
+ jaro_similarity = calculate_ngram_similarity(compare_word, compare_dict)
230
+
231
+ # Skip low-confidence or low-similarity suggestions (unless we need more for min_results)
232
+ if (confidence < min_confidence || jaro_similarity < min_similarity) && (suggestions.size >= min_results)
233
+ next
234
+ end
235
+
236
+ suggestions << Suggestion.new(
237
+ word: dict_word,
238
+ distance: dist,
239
+ confidence: confidence,
240
+ source: @name,
241
+ original_length: word.length,
242
+ ngram_score: jaro_similarity, # Now stores Jaro-Winkler similarity (0.0-1.0)
243
+ enhanced_score: score
244
+ )
245
+
246
+ # Stop when we have enough high-quality suggestions
247
+ break if suggestions.size >= max_results
248
+ end
249
+
250
+ SuggestionSet.new(suggestions, max_size: max_results)
251
+ end
252
+
253
+ # Check if this strategy should handle the context.
254
+ #
255
+ # @param context [Context] The suggestion context
256
+ # @return [Boolean] True if the word needs correction
257
+ def handles?(context)
258
+ return false unless enabled?
259
+
260
+ # Only handle if the word is not in the dictionary
261
+ !dictionary_lookup(context, context.word)
262
+ end
263
+
264
+ private
265
+
266
+ # Get all words from the dictionary.
267
+ #
268
+ # @param context [Context] The suggestion context
269
+ # @return [Array<String>] All dictionary words
270
+ def dictionary_words(context)
271
+ dictionary = context.dictionary
272
+
273
+ if indexed_dictionary?(dictionary)
274
+ dictionary.all_words
275
+ else
276
+ case dictionary
277
+ when Kotoshu::Dictionary::Base then dictionary.words
278
+ when Hash then dictionary.keys
279
+ when Set then dictionary.to_a
280
+ when Array then dictionary.dup
281
+ else Array(dictionary).flat_map(&:to_a)
282
+ end
283
+ end
284
+ end
285
+
286
+ # Get dictionary words whose length is in [min, max].
287
+ #
288
+ # Uses the dictionary's find_by_length_range when the backend
289
+ # is one that supports it (Dictionary::Base subclasses,
290
+ # IndexedDictionary) — both expose an index-backed
291
+ # implementation. Falls back to manual filtering for ad-hoc
292
+ # dictionary shapes (Hash, Array, Set) used in tests.
293
+ #
294
+ # @param context [Context] The suggestion context
295
+ # @param min [Integer] Minimum length (inclusive)
296
+ # @param max [Integer] Maximum length (inclusive)
297
+ # @return [Array<String>] Words in the length range
298
+ def dictionary_words_in_length_range(context, min:, max:)
299
+ dictionary = context.dictionary
300
+
301
+ case dictionary
302
+ when Kotoshu::Dictionary::Base, ::Kotoshu::Core::IndexedDictionary
303
+ dictionary.find_by_length_range(min_length: min, max_length: max)
304
+ else
305
+ dictionary_words(context).select { |w| w.length >= min && w.length <= max }
306
+ end
307
+ end
308
+
309
+ def indexed_dictionary?(dictionary)
310
+ defined?(::Kotoshu::Core::IndexedDictionary) && dictionary.is_a?(::Kotoshu::Core::IndexedDictionary)
311
+ end
312
+
313
+ # Check if a word exists in the dictionary.
314
+ #
315
+ # @param context [Context] The suggestion context
316
+ # @param word [String] The word to check
317
+ # @return [Boolean] True if word exists
318
+ def dictionary_lookup(context, word)
319
+ dictionary = context.dictionary
320
+
321
+ if defined?(::Kotoshu::Core::IndexedDictionary) && dictionary.is_a?(::Kotoshu::Core::IndexedDictionary)
322
+ dictionary.has_word?(word)
323
+ else
324
+ case dictionary
325
+ when Kotoshu::Dictionary::Base then dictionary.lookup(word)
326
+ when Hash then dictionary.key?(word)
327
+ when Set, Array then dictionary.include?(word)
328
+ else false
329
+ end
330
+ end
331
+ end
332
+
333
+ # Determine whether the context's dictionary treats lookups
334
+ # case-insensitively. When true, the edit-distance and similarity
335
+ # calculations lowercase both sides so case differences don't
336
+ # masquerade as edit-distance penalties.
337
+ #
338
+ # @param context [Context] The suggestion context
339
+ # @return [Boolean]
340
+ def dictionary_case_insensitive?(context)
341
+ dictionary = context.dictionary
342
+ # PlainText and similar backends expose `case_sensitive` directly.
343
+ # Hunspell/IndexedDictionary route case through the casing layer,
344
+ # so we leave them case-sensitive here (their casing machinery
345
+ # already produces the right variants upstream).
346
+ return false unless dictionary.is_a?(::Kotoshu::Dictionary::PlainText)
347
+
348
+ !dictionary.case_sensitive
349
+ end
350
+
351
+ # Calculate Damerau-Levenshtein edit distance between two strings.
352
+ # Delegates to Algorithms::EditDistance.
353
+ #
354
+ # @param str1 [String] First string
355
+ # @param str2 [String] Second string
356
+ # @return [Integer] Edit distance
357
+ def edit_distance(str1, str2)
358
+ Algorithms::EditDistance.distance(str1, str2)
359
+ end
360
+
361
+ # Optimized edit distance with early termination.
362
+ # Returns early if distance exceeds threshold.
363
+ #
364
+ # @param str1 [String] First string
365
+ # @param str2 [String] Second string
366
+ # @param threshold [Integer] Maximum distance to calculate
367
+ # @return [Integer, nil] Distance or nil if exceeds threshold
368
+ def edit_distance_with_threshold(str1, str2, threshold)
369
+ Algorithms::EditDistance.distance_with_threshold(str1, str2, threshold)
370
+ end
371
+
372
+ # Enumerate the single-edit candidates of the misspelling and
373
+ # yield the ones the dictionary accepts, with their true
374
+ # (Damerau-Levenshtein) distance. Edit order is fixed and
375
+ # mirrors typo frequency: transposition, substitution,
376
+ # insertion, deletion — ties in the candidate sort resolve in
377
+ # this order, and the native engine reproduces it exactly.
378
+ #
379
+ # Substitutions and insertions use the dictionary's TRY
380
+ # string (Hunspell TRY directive); with no TRY available
381
+ # they are skipped, exactly as in Hunspell. Deletions and
382
+ # transpositions need no alphabet and always run.
383
+ #
384
+ # Candidates are validated with the full dictionary lookup,
385
+ # so affixed forms (stem + suffix/prefix, e.g. "definitely")
386
+ # and capitalization variants (e.g. "The") are accepted with
387
+ # their surface spelling preserved.
388
+ #
389
+ # @param context [Context] The suggestion context
390
+ # @param word [String] The misspelled word
391
+ # @param case_insensitive [Boolean] Comparison case handling
392
+ # @param max_dist [Integer] Maximum distance to accept
393
+ # @yield [String, Integer] Each accepted candidate and distance
394
+ def edit_sweep_candidates(context, word, case_insensitive:, max_dist:)
395
+ compare_word = case_insensitive ? word.downcase : word
396
+ try_string = try_string_for(context)
397
+ seen = Set.new
398
+
399
+ each_sweep_edit(word, try_string) do |candidate|
400
+ next if candidate == word
401
+ next unless seen.add?(candidate)
402
+
403
+ next unless dictionary_lookup(context, candidate)
404
+
405
+ compare_candidate = case_insensitive ? candidate.downcase : candidate
406
+ dist = edit_distance_with_threshold(compare_word, compare_candidate, max_dist)
407
+ next unless dist && dist > 0
408
+
409
+ yield candidate, dist
410
+ end
411
+ end
412
+
413
+ # Yield every single-edit variant of the word in canonical
414
+ # order: adjacent transpositions (Permutations.swapchar,
415
+ # restricted Damerau operation), substitutions over the TRY
416
+ # alphabet (Permutations.badchar), insertions over the TRY
417
+ # alphabet (Permutations.forgotchar), and deletions
418
+ # (Permutations.extrachar). Reuses the spylls-derived
419
+ # permutation primitives — the same generators the
420
+ # Hunspell-style suggester uses.
421
+ #
422
+ # @param word [String] The word to edit
423
+ # @param try_string [String, nil] Substitution/insertion alphabet
424
+ # @yield [String] Each edit variant
425
+ def each_sweep_edit(word, try_string, &block)
426
+ # Adjacent transposition (and, for 4/5-letter words,
427
+ # double adjacent swaps) — the most common typing error.
428
+ Algorithms::Permutations.swapchar(word, &block)
429
+
430
+ # Substitution of each character with each TRY character.
431
+ Algorithms::Permutations.badchar(word, try_string, &block)
432
+
433
+ # Insertion of each TRY character at each position.
434
+ Algorithms::Permutations.forgotchar(word, try_string, &block)
435
+
436
+ # Deletion of one character at each position.
437
+ Algorithms::Permutations.extrachar(word, &block)
438
+ end
439
+
440
+ # The substitution/insertion alphabet for the edit sweep: an
441
+ # explicit +try_string+ config wins (used by callers without
442
+ # a TRY-capable dictionary); otherwise the dictionary's TRY
443
+ # string (nil for backends without one — the sweep then runs
444
+ # transpositions and deletions only).
445
+ #
446
+ # @param context [Context] The suggestion context
447
+ # @return [String, nil] TRY characters or nil
448
+ def try_string_for(context)
449
+ configured = get_config(:try_string)
450
+ return configured if configured
451
+
452
+ dictionary = context.dictionary
453
+ case dictionary
454
+ when Kotoshu::Dictionary::Base then dictionary.try_string
455
+ else nil
456
+ end
457
+ end
458
+
459
+ public
460
+
461
+ # Calculate enhanced score combining multiple factors.
462
+ #
463
+ # Lower score = better suggestion
464
+ #
465
+ # @param original [String] The original misspelled word
466
+ # @param suggestion [String] The suggested word
467
+ # @param distance [Integer] Edit distance
468
+ # @return [Float] Enhanced score (lower is better)
469
+ def calculate_enhanced_score(original, suggestion, distance)
470
+ score = distance * 1000.0 # Base score from edit distance
471
+
472
+ # Factor 1: Word frequency bonus (common words get lower score)
473
+ score -= frequency_bonus(suggestion)
474
+
475
+ # Factor 2: Keyboard proximity penalty (typo-like patterns get lower score)
476
+ score += keyboard_penalty(original, suggestion)
477
+
478
+ # Factor 3: Common typo pattern bonus
479
+ # Transposition (swap adjacent chars) is the MOST common typo
480
+ trans_bonus = transposition_bonus(original, suggestion)
481
+ score -= trans_bonus
482
+
483
+ # Factor 4: Missing double letter bonus (helo -> hello)
484
+ score -= typo_pattern_bonus(original, suggestion)
485
+
486
+ # Factor 5: Length similarity bonus (similar length is better)
487
+ length_diff = (original.length - suggestion.length).abs
488
+ score += length_diff * 50
489
+
490
+ score
491
+ end
492
+
493
+ # Calculate bonus for transposition (swap adjacent characters).
494
+ # This is the MOST common typing error, so it gets the highest bonus.
495
+ #
496
+ # @param original [String] The original word
497
+ # @param suggestion [String] The suggested word
498
+ # @return [Float] Transposition bonus (0 or 200)
499
+ def transposition_bonus(original, suggestion)
500
+ # Transposition only makes sense for same-length words
501
+ return 0 unless original.length == suggestion.length
502
+
503
+ o = original.downcase
504
+ s = suggestion.downcase
505
+
506
+ # Count transpositions needed
507
+ transpositions = 0
508
+ (0...o.length).each do |i|
509
+ next if o[i] == s[i]
510
+
511
+ # Find matching char in suggestion
512
+ match_idx = s.index(o[i], i + 1)
513
+ if match_idx && (match_idx == i + 1 || (match_idx > i + 1 && s[i] == o[match_idx]))
514
+ # This is a simple adjacent swap
515
+ transpositions += 1
516
+ end
517
+ end
518
+
519
+ # Only give bonus for single transposition
520
+ transpositions == 1 ? 200 : (transpositions * 100)
521
+ end
522
+
523
+ # Calculate keyboard proximity penalty.
524
+ #
525
+ # Substitutions between adjacent keys get lower penalty.
526
+ # Uses OOP keyboard layout for language-aware distance calculations.
527
+ #
528
+ # @param original [String] The original word
529
+ # @param suggestion [String] The suggested word
530
+ # @return [Float] Keyboard penalty (0-200)
531
+ def keyboard_penalty(original, suggestion)
532
+ penalty = 0
533
+
534
+ # Find the edit script to see what changed
535
+ o_chars = original.chars
536
+ s_chars = suggestion.chars
537
+
538
+ # Simple comparison for equal-length words (substitutions)
539
+ if o_chars.length == s_chars.length
540
+ o_chars.each_with_index do |c1, i|
541
+ c2 = s_chars[i]
542
+ next if c1 == c2
543
+
544
+ # Use OOP keyboard layout for distance calculation
545
+ key_dist = @keyboard_layout.distance(c1, c2)
546
+
547
+ penalty += if key_dist == Float::INFINITY
548
+ # Symbol or unknown key - medium penalty
549
+ 50
550
+ elsif key_dist == 1
551
+ 10 # Very likely typo (adjacent keys)
552
+ elsif key_dist == 2
553
+ 30 # Somewhat likely
554
+ else
555
+ 100 # Unlikely to be typo (far keys)
556
+ end
557
+ end
558
+ end
559
+
560
+ penalty
561
+ end
562
+
563
+ # Calculate bonus for common typo patterns.
564
+ #
565
+ # @param original [String] The original word
566
+ # @param suggestion [String] The suggested word
567
+ # @return [Float] Pattern bonus (0-300)
568
+ def typo_pattern_bonus(original, suggestion)
569
+ bonus = 0
570
+
571
+ # Pattern 1: Missing double letter (helo -> hello)
572
+ # This is the MOST COMMON typo after transposition, give it highest bonus
573
+ if suggestion.length == original.length + 1
574
+ # Check if suggestion has a double letter that original is missing
575
+ suggestion.chars.each_cons(2).with_index do |pair, i|
576
+ if pair[0] == pair[1] # Found double letter at positions i and i+1
577
+ # Check if removing the second occurrence (at i+1) gives us the original word
578
+ # For "hello" with "ll" at position 2, remove position 3: "hel" + "o" = "helo"
579
+ expected = suggestion[0...i + 1] + suggestion[i + 2..-1]
580
+ if expected == original
581
+ bonus += 300 # Strong bonus for missing double letter (MORE than transposition!)
582
+ break
583
+ end
584
+ end
585
+ end
586
+ end
587
+
588
+ # Pattern 2: Extra double letter (helllo -> hello)
589
+ if original.length == suggestion.length + 1
590
+ # Check if original has a double letter that suggestion doesn't
591
+ original.chars.each_cons(2).with_index do |pair, i|
592
+ if pair[0] == pair[1] # Found double letter in original
593
+ # Check if removing it gives the suggestion
594
+ reconstructed = original[0...i + 1] + original[i + 1..-1]
595
+ if reconstructed == suggestion
596
+ bonus += 100 # Bonus for extra double letter
597
+ break
598
+ end
599
+ end
600
+ end
601
+ end
602
+
603
+ # Pattern 3: Common prefixes/suffixes
604
+ if original.start_with?(suggestion[0...3]) && suggestion.length > original.length
605
+ bonus += 30 # Suggestion extends common prefix
606
+ end
607
+
608
+ bonus
609
+ end
610
+
611
+ private
612
+
613
+ # Resolve keyboard layout using OOP registry pattern
614
+ #
615
+ # @param keyboard_layout [Keyboard::Layout, String, nil] Layout override
616
+ # @return [Keyboard::Layout] Resolved layout
617
+ def resolve_keyboard_layout(keyboard_layout)
618
+ if keyboard_layout.is_a?(Keyboard::Layout)
619
+ keyboard_layout
620
+ elsif keyboard_layout.is_a?(String)
621
+ Keyboard::Registry.layout_by_name(keyboard_layout)
622
+ elsif @language_code
623
+ Keyboard::Registry.layout_for(@language_code)
624
+ else
625
+ Keyboard::Registry.layout_by_name('QWERTY')
626
+ end
627
+ end
628
+ end
629
+ end
630
+ end
631
+ end