kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,207 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # Keyboard proximity suggestion strategy.
7
+ #
8
+ # Generates suggestions by finding words that can be formed by
9
+ # substituting adjacent keys on a QWERTY keyboard.
10
+ #
11
+ # @example Creating a keyboard proximity strategy
12
+ # strategy = KeyboardProximityStrategy.new
13
+ # result = strategy.generate(context)
14
+ class KeyboardProximityStrategy < BaseStrategy
15
+ # QWERTY keyboard layout (US).
16
+ #
17
+ # Each key maps to its adjacent keys.
18
+ KEYBOARD_LAYOUT = {
19
+ "`" => %w[1 tab],
20
+ "1" => ["`", "2", "q"],
21
+ "2" => %w[1 3 w q],
22
+ "3" => %w[2 4 e w],
23
+ "4" => %w[3 5 r e],
24
+ "5" => %w[4 6 t r],
25
+ "6" => %w[5 7 y t],
26
+ "7" => %w[6 8 u y],
27
+ "8" => %w[7 9 i u],
28
+ "9" => %w[8 0 o i],
29
+ "0" => %w[9 p o],
30
+ "-" => ["0", "="],
31
+ "=" => ["-"],
32
+ "q" => %w[tab w a 1],
33
+ "w" => %w[q e a s 2],
34
+ "e" => %w[w r s d 3],
35
+ "r" => %w[e t d f 4],
36
+ "t" => %w[r y f g 5],
37
+ "y" => %w[t u g h 6],
38
+ "u" => %w[y i h j 7],
39
+ "i" => %w[u o j k 8],
40
+ "o" => %w[i p k l 9],
41
+ "p" => ["o", "l", ";", "0"],
42
+ "[" => ["p", "'"],
43
+ "]" => ["enter", "\\"],
44
+ "\\" => ["enter"], # Backslash neighbors
45
+ "a" => %w[caps s z q],
46
+ "s" => %w[a d z x w],
47
+ "d" => %w[s f x c e],
48
+ "f" => %w[d g c v r],
49
+ "g" => %w[f h v b t],
50
+ "h" => %w[g j b n y],
51
+ "j" => %w[h k n m u],
52
+ "k" => ["j", "l", "m", ",", "i"],
53
+ "l" => ["k", ";", ",", ".", "o"],
54
+ ";" => ["l", "'", ".", "p"],
55
+ "'" => [";"],
56
+ "z" => %w[shift s x a],
57
+ "x" => %w[z c s d],
58
+ "c" => %w[x v d f],
59
+ "v" => %w[c b f g],
60
+ "b" => %w[v n g h],
61
+ "n" => %w[b m h j],
62
+ "m" => ["n", ",", "j", "k"],
63
+ "," => ["m", ".", "k", "l"],
64
+ "." => [",", "/", "l", ";"],
65
+ "/" => [".", "shift"],
66
+ " " => [] # Space has no neighbors
67
+ }.freeze
68
+
69
+ # Create a new keyboard proximity strategy.
70
+ #
71
+ # @param name [String, Symbol] Name of the strategy
72
+ # @param config [Hash] Configuration options
73
+ # @option config [Integer] max_distance Maximum keyboard distance
74
+ # @option config [Integer] max_results Maximum results to return
75
+ def initialize(name: :keyboard_proximity, **config)
76
+ super
77
+ end
78
+
79
+ # Generate suggestions based on keyboard proximity.
80
+ #
81
+ # @param context [Context] The suggestion context
82
+ # @return [SuggestionSet] Suggestions within keyboard distance
83
+ def generate(context)
84
+ word = context.word
85
+ max_dist = get_config(:max_distance, 2)
86
+ min_similarity = get_config(:min_similarity, 0.70) # Filter low-similarity suggestions
87
+
88
+ # One index per generate call: the variant set reaches tens
89
+ # of thousands of lookups, each of which used to scan the
90
+ # whole word list (with a downcase per candidate on the
91
+ # case-insensitive branch). WordIndex preserves find_word
92
+ # semantics exactly — exact (lowercase) match first, then
93
+ # first-in-list-order on the case-insensitive branch.
94
+ index = WordIndex.build(dictionary_words(context))
95
+
96
+ # Generate keyboard variants
97
+ variants = keyboard_variants(word, max_dist)
98
+
99
+ # Find matching dictionary words with their edit distances and similarity
100
+ results_with_distances = {}
101
+ variants.each do |variant|
102
+ dict_word = index.find(variant)
103
+ next unless dict_word && dict_word != word
104
+
105
+ # Calculate edit distance from original word
106
+ dist = edit_distance(word, dict_word)
107
+ next if dist > max_dist
108
+
109
+ # Calculate typo correction similarity
110
+ similarity = calculate_ngram_similarity(word, dict_word)
111
+ next if similarity < min_similarity # Filter by similarity threshold
112
+
113
+ # Keep the minimum distance for each word
114
+ results_with_distances[dict_word] ||= dist
115
+ results_with_distances[dict_word] = dist if dist < results_with_distances[dict_word]
116
+ end
117
+
118
+ # Sort by distance and create suggestions
119
+ sorted_words = results_with_distances.sort_by { |_, dist| dist }.map(&:first)
120
+ create_suggestion_set(sorted_words, distances: results_with_distances, original_word: word)
121
+ end
122
+
123
+ # Check if this strategy should handle the context.
124
+ #
125
+ # @param context [Context] The suggestion context
126
+ # @return [Boolean] True if the word needs correction
127
+ def handles?(context)
128
+ return false unless enabled?
129
+
130
+ !dictionary_lookup(context, context.word)
131
+ end
132
+
133
+ private
134
+
135
+ # Calculate edit distance between two strings.
136
+ # Uses Damerau-Levenshtein distance (substitution, insertion,
137
+ # deletion, adjacent transposition) via Algorithms::EditDistance,
138
+ # so a swapped pair ("teh" -> "the") costs 1, not 2.
139
+ #
140
+ # @param str1 [String] First string
141
+ # @param str2 [String] Second string
142
+ # @return [Integer] Edit distance
143
+ def edit_distance(str1, str2)
144
+ Algorithms::EditDistance.distance(str1, str2)
145
+ end
146
+
147
+ # Get neighbors for a key.
148
+ #
149
+ # @param char [String] The character
150
+ # @return [Array<String>] Neighbor keys
151
+ def neighbors(char)
152
+ KEYBOARD_LAYOUT[char.downcase] || []
153
+ end
154
+
155
+ # Generate keyboard variants of a word.
156
+ #
157
+ # Each round applies one edit operation to every current
158
+ # variant: substitution with a keyboard-adjacent key,
159
+ # insertion of an adjacent key, deletion, and adjacent
160
+ # transposition (the restricted Damerau operation — the most
161
+ # common typing error, which is why it is enumerated here as a
162
+ # single edit like the others).
163
+ #
164
+ # @param word [String] The word
165
+ # @param max_distance [Integer] Maximum edit distance
166
+ # @return [Array<String>] Keyboard variants
167
+ def keyboard_variants(word, max_distance)
168
+ return [] if word.nil? || word.empty?
169
+
170
+ word = word.downcase
171
+ variants = Set.new([word])
172
+
173
+ max_distance.times do
174
+ new_variants = Set.new
175
+
176
+ variants.each do |variant|
177
+ # Generate all single-key substitutions
178
+ variant.each_char.with_index do |char, i|
179
+ neighbors(char).each do |neighbor|
180
+ new_word = variant[0...i] + neighbor + variant[(i + 1)..]
181
+ new_variants.add(new_word)
182
+
183
+ # Also try insertions and deletions
184
+ new_variants.add(variant[0...i] + variant[(i + 1)..]) # Delete
185
+ new_variants.add(variant[0...i] + neighbor + variant[i..]) # Insert
186
+ end
187
+ end
188
+
189
+ # Adjacent transposition: swap each neighboring
190
+ # character pair (a single 1-cost edit).
191
+ (0...variant.length - 1).each do |i|
192
+ swapped = variant[0...i] + variant[i + 1] + variant[i] + variant[(i + 2)..]
193
+ new_variants.add(swapped)
194
+ end
195
+ end
196
+
197
+ variants = new_variants
198
+ end
199
+
200
+ variants.to_a
201
+ end
202
+ end
203
+ end
204
+ end
205
+ end
206
+
207
+ require "set" if RUBY_VERSION < "3.0"
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # N-gram suggestion strategy.
7
+ #
8
+ # Generates suggestions by finding words with high n-gram similarity.
9
+ # N-grams are contiguous sequences of n characters.
10
+ #
11
+ # @example Creating an n-gram strategy
12
+ # strategy = NgramStrategy.new(n: 3)
13
+ # result = strategy.generate(context)
14
+ class NgramStrategy < BaseStrategy
15
+ # Create a new n-gram strategy.
16
+ #
17
+ # @param name [String, Symbol] Name of the strategy
18
+ # @param config [Hash] Configuration options
19
+ # @option config [Integer] n N-gram size (default: 3)
20
+ # @option config [Float] min_similarity Minimum similarity threshold (0-1)
21
+ # @option config [Integer] max_results Maximum results to return
22
+ def initialize(name: :ngram, **config)
23
+ super
24
+ end
25
+
26
+ # Generate suggestions based on n-gram similarity.
27
+ #
28
+ # @param context [Context] The suggestion context
29
+ # @return [SuggestionSet] Suggestions with high n-gram similarity
30
+ def generate(context)
31
+ word = context.word
32
+ n = get_config(:n, 3)
33
+ min_sim = get_config(:min_similarity, 0.3)
34
+ min_typo_similarity = get_config(:min_typo_similarity, 0.70) # Filter by typo correction similarity
35
+
36
+ return create_suggestion_set([]) if word.length < n
37
+
38
+ # Get n-grams for input word
39
+ word_ngrams = extract_ngrams(word, n)
40
+ word_length = word.length
41
+
42
+ # Calculate n-gram similarity for each dictionary word
43
+ results = {}
44
+ each_word_with_length(context) do |dict_word, dict_length|
45
+ next if dict_word == word
46
+ next if dict_length < n
47
+
48
+ # Length pre-gate: similarity is a Jaccard coefficient
49
+ # over n-gram multisets, so it is at most
50
+ # min-gram-count / max-gram-count — each word of length L
51
+ # contributes L - n + 1 grams. A word whose length bound
52
+ # already sits under min_similarity can never pass the
53
+ # real test, so skip it before extracting a single
54
+ # n-gram. Mathematically the same set of words passes
55
+ # (same gate as ngram.rs in kotoshu-rs). The length is
56
+ # read from the dictionary's memoized sweep index before
57
+ # any per-word work when the backend maintains one.
58
+ min_grams = (dict_length < word_length ? dict_length : word_length) - (n - 1)
59
+ max_grams = (dict_length > word_length ? dict_length : word_length) - (n - 1)
60
+ next if min_grams.to_f / max_grams < min_sim
61
+
62
+ similarity = ngram_similarity(word_ngrams, dict_word, n)
63
+ next if similarity < min_sim
64
+
65
+ # Also check typo correction similarity for filtering
66
+ typo_sim = calculate_ngram_similarity(word, dict_word)
67
+ next if typo_sim < min_typo_similarity
68
+
69
+ # Convert similarity to distance (higher similarity = lower distance)
70
+ dist = ((1 - similarity) * 10).to_i
71
+ next if dist.zero?
72
+
73
+ results[dict_word] ||= dist
74
+ results[dict_word] = dist if dist < results[dict_word]
75
+ end
76
+
77
+ # Convert to suggestions sorted by similarity
78
+ sorted_words = results.sort_by { |_, dist| dist }.map(&:first)
79
+ create_suggestion_set(sorted_words, distances: results, original_word: word)
80
+ end
81
+
82
+ # Check if this strategy should handle the context.
83
+ #
84
+ # @param context [Context] The suggestion context
85
+ # @return [Boolean] True if the word needs correction
86
+ def handles?(context)
87
+ return false unless enabled?
88
+
89
+ !dictionary_lookup(context, context.word)
90
+ end
91
+
92
+ private
93
+
94
+ # Yield each dictionary word paired with its char length, in
95
+ # word-list order. Over a Dictionary::Base backend the pairs
96
+ # come from the dictionary's memoized sweep index
97
+ # ({Dictionary::Base#sweep_index}) — the length gate reads the
98
+ # indexed value before any per-word work, and the sweep never
99
+ # copies the word list. Ad-hoc Hash/Array dictionaries compute
100
+ # the length live, exactly as before.
101
+ #
102
+ # @param context [Context] The suggestion context
103
+ # @yield [String, Integer] Each word and its char length
104
+ # @return [void]
105
+ def each_word_with_length(context)
106
+ dictionary = context.dictionary
107
+ if dictionary.is_a?(Kotoshu::Dictionary::Base)
108
+ dictionary.sweep_index.each_with_length { |*pair| yield(*pair) }
109
+ else
110
+ dictionary_words(context).each { |dict_word| yield dict_word, dict_word.length }
111
+ end
112
+ end
113
+
114
+ # Extract n-grams from a word.
115
+ #
116
+ # @param word [String] The word
117
+ # @param n [Integer] N-gram size
118
+ # @return [Hash] N-gram to count mapping
119
+ def extract_ngrams(word, n)
120
+ ngrams = Hash.new(0)
121
+
122
+ (word.length - n + 1).times do |i|
123
+ ngram = word[i...i + n]
124
+ ngrams[ngram] += 1
125
+ end
126
+
127
+ ngrams
128
+ end
129
+
130
+ # Calculate n-gram similarity between two words.
131
+ #
132
+ # Uses the Jaccard similarity coefficient:
133
+ # similarity = |intersection| / |union|
134
+ #
135
+ # @param word_ngrams [Hash] N-grams for the first word
136
+ # @param other_word [String] The second word
137
+ # @param n [Integer] N-gram size
138
+ # @return [Float] Similarity score (0-1)
139
+ def ngram_similarity(word_ngrams, other_word, n)
140
+ other_ngrams = extract_ngrams(other_word, n)
141
+
142
+ # Intersection and union in one pass over both maps. The
143
+ # original built the key union as an intermediate Array
144
+ # (`word_ngrams.keys | other_ngrams.keys`) — one allocation
145
+ # per dictionary word that dominated the sweep once the
146
+ # length gate landed.
147
+ intersection = 0
148
+ union = 0
149
+ word_ngrams.each do |ngram, count|
150
+ other_count = other_ngrams[ngram] || 0
151
+ intersection += count < other_count ? count : other_count
152
+ union += count > other_count ? count : other_count
153
+ end
154
+ other_ngrams.each do |ngram, count|
155
+ union += count unless word_ngrams.key?(ngram)
156
+ end
157
+
158
+ return 0.0 if union.zero?
159
+
160
+ intersection.to_f / union
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,284 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # Phonetic suggestion strategy.
7
+ #
8
+ # Generates suggestions by finding words with similar phonetic codes
9
+ # using algorithms like Soundex and Metaphone.
10
+ #
11
+ # @example Creating a phonetic strategy
12
+ # strategy = PhoneticStrategy.new(algorithm: :soundex)
13
+ # result = strategy.generate(context)
14
+ class PhoneticStrategy < BaseStrategy
15
+ # Supported algorithms.
16
+ ALGORITHMS = %i[soundex metaphone].freeze
17
+
18
+ # Create a new phonetic strategy.
19
+ #
20
+ # @param name [String, Symbol] Name of the strategy
21
+ # @param config [Hash] Configuration options
22
+ # @option config [Symbol] algorithm The algorithm to use (:soundex or :metaphone)
23
+ # @option config [Integer] max_results Maximum results to return
24
+ def initialize(name: :phonetic, **config)
25
+ super
26
+ end
27
+
28
+ # Generate suggestions based on phonetic similarity.
29
+ #
30
+ # @param context [Context] The suggestion context
31
+ # @return [SuggestionSet] Suggestions with same phonetic code
32
+ def generate(context)
33
+ word = context.word
34
+ algorithm = get_config(:algorithm, :soundex)
35
+ max_dist = 2
36
+
37
+ # Get phonetic code for input word
38
+ word_code = phonetic_code(word, algorithm)
39
+
40
+ # Find words with same phonetic code
41
+ results = []
42
+ each_phonetic_pair(context, algorithm) do |dict_word, dict_code|
43
+ next if dict_word == word
44
+ next unless dict_code == word_code
45
+
46
+ dist = edit_distance(word, dict_word)
47
+ next if dist > max_dist || dist.zero?
48
+
49
+ results << [dict_word, dist]
50
+ end
51
+
52
+ # Sort by distance and convert to suggestions
53
+ sorted_words = results.sort_by { |_, dist| dist }.map(&:first)
54
+ create_suggestion_set(sorted_words)
55
+ end
56
+
57
+ # Check if this strategy should handle the context.
58
+ #
59
+ # @param context [Context] The suggestion context
60
+ # @return [Boolean] True if the word needs correction
61
+ def handles?(context)
62
+ return false unless enabled?
63
+
64
+ !dictionary_lookup(context, context.word)
65
+ end
66
+
67
+ private
68
+
69
+ # Yield each dictionary word paired with its phonetic code, in
70
+ # word-list order. For Soundex over a Dictionary::Base backend
71
+ # the codes come from the dictionary's memoized sweep index
72
+ # ({Dictionary::Base#sweep_index}) — one Soundex per word for
73
+ # the dictionary's lifetime instead of one per sweep. Every
74
+ # other combination (ad-hoc Hash/Array dictionaries, Metaphone)
75
+ # computes each code live, exactly as before.
76
+ #
77
+ # @param context [Context] The suggestion context
78
+ # @param algorithm [Symbol] The phonetic algorithm
79
+ # @yield [String, String] Each word and its phonetic code
80
+ # @return [void]
81
+ def each_phonetic_pair(context, algorithm)
82
+ dictionary = context.dictionary
83
+ if algorithm == :soundex && dictionary.is_a?(Kotoshu::Dictionary::Base)
84
+ dictionary.sweep_index.each_with_soundex { |*pair| yield(*pair) }
85
+ else
86
+ dictionary_words(context).each do |dict_word|
87
+ yield dict_word, phonetic_code(dict_word, algorithm)
88
+ end
89
+ end
90
+ end
91
+
92
+ # Get phonetic code for a word.
93
+ #
94
+ # @param word [String] The word
95
+ # @param algorithm [Symbol] The algorithm to use
96
+ # @return [String] The phonetic code
97
+ def phonetic_code(word, algorithm = :soundex)
98
+ case algorithm
99
+ when :soundex
100
+ soundex_code(word)
101
+ when :metaphone
102
+ metaphone_code(word)
103
+ else
104
+ soundex_code(word)
105
+ end
106
+ end
107
+
108
+ # Calculate Soundex code for a word.
109
+ #
110
+ # Soundex is a phonetic algorithm developed by Robert C. Russell
111
+ # and Margaret King Odell in the early 1900s. The implementation
112
+ # lives in {Algorithms::Soundex} and is shared with
113
+ # {Suggestions::SweepIndex}, which memoizes the code per
114
+ # dictionary word.
115
+ #
116
+ # @param word [String] The word
117
+ # @return [String] The Soundex code (letter + 3 digits)
118
+ #
119
+ # @example
120
+ # soundex_code("Robert") # => "R163"
121
+ # soundex_code("Rupert") # => "R163"
122
+ # soundex_code("Ashcraft") # => "A261"
123
+ def soundex_code(word)
124
+ Algorithms::Soundex.code(word)
125
+ end
126
+
127
+ # Calculate Metaphone code for a word.
128
+ #
129
+ # Metaphone is an improved phonetic algorithm developed by
130
+ # Lawrence Philips in 1990.
131
+ #
132
+ # @param word [String] The word
133
+ # @return [String] The Metaphone code
134
+ #
135
+ # @example
136
+ # metaphone_code("Schmidt") # => "XMT"
137
+ # metaphone_code("Smith") # => "SM0"
138
+ def metaphone_code(word)
139
+ return "" if word.nil? || word.empty?
140
+
141
+ word = word.upcase.gsub(/[^A-Z]/, "")
142
+ return "" if word.empty?
143
+
144
+ # Metaphone rules (simplified implementation)
145
+ code = ""
146
+ i = 0
147
+ length = word.length
148
+
149
+ while i < length && code.length < 4
150
+ char = word[i]
151
+ next_char = i + 1 < length ? word[i + 1] : ""
152
+
153
+ case char
154
+ when "A", "E", "I", "O", "U"
155
+ # Vowels are only encoded at the beginning
156
+ code += char if i.zero?
157
+ when "B"
158
+ code += "B"
159
+ when "C"
160
+ if next_char == "H" && i + 2 < length && %w[A E I O U].include?(word[i + 2])
161
+ # "CH" followed by vowel => "X"
162
+ code += "X"
163
+ i += 1
164
+ elsif next_char == "I" && i + 2 < length && word[i + 2] == "A"
165
+ # "CIA" => "X"
166
+ code += "X"
167
+ i += 2
168
+ elsif %w[S G].include?(next_char)
169
+ # "CS", "CG" => "X"
170
+ code += "X"
171
+ i += 1
172
+ else
173
+ code += "K"
174
+ end
175
+ when "D"
176
+ if next_char == "G" && i + 2 < length && %w[I E Y].include?(word[i + 2])
177
+ # "DG" followed by I, E, Y => "J"
178
+ code += "J"
179
+ i += 1
180
+ else
181
+ code += "T"
182
+ end
183
+ when "F"
184
+ code += "F"
185
+ when "G"
186
+ if next_char == "H"
187
+ # "GH" => silent unless at beginning or after vowel
188
+ if i.zero?
189
+ code += "K"
190
+ i += 1
191
+ end
192
+ elsif next_char == "N"
193
+ # "GN" => "N" (silent G)
194
+ i += 1
195
+ elsif next_char == "N" && i + 2 < length && word[i + 2] == "E" && i + 3 < length && word[i + 3] == "D"
196
+ # "GNED" => "N" (silent G)
197
+ i += 3
198
+ else
199
+ code += "K"
200
+ end
201
+ when "H"
202
+ # H is silent unless at beginning
203
+ code += "H" if i.zero?
204
+ when "J"
205
+ code += "J"
206
+ when "K"
207
+ code += "K"
208
+ i += 1 if next_char == "N" # "KN" => "N"
209
+ when "L"
210
+ code += "L"
211
+ when "M"
212
+ code += "M"
213
+ when "N"
214
+ code += "N"
215
+ when "P"
216
+ if next_char == "H"
217
+ # "PH" => "F"
218
+ code += "F"
219
+ i += 1
220
+ else
221
+ code += "P"
222
+ end
223
+ when "Q"
224
+ code += "K"
225
+ when "R"
226
+ code += "R"
227
+ when "S"
228
+ if next_char == "H"
229
+ # "SH" => "X"
230
+ code += "X"
231
+ i += 1
232
+ elsif next_char == "I" && i + 2 < length && word[i + 2] == "O"
233
+ # "SIO" or "SIA" => "X"
234
+ code += "X"
235
+ i += 2
236
+ else
237
+ code += "S"
238
+ end
239
+ when "T"
240
+ if next_char == "I" && i + 2 < length && %w[O A].include?(word[i + 2])
241
+ # "TIO" or "TIA" => "X"
242
+ code += "X"
243
+ i += 2
244
+ elsif next_char == "H"
245
+ # "TH" => "0"
246
+ code += "0"
247
+ i += 1
248
+ else
249
+ code += "T"
250
+ end
251
+ when "V"
252
+ code += "F"
253
+ when "W", "Y"
254
+ # W and Y are semi-vowels, only encode at beginning
255
+ code += char if i.zero?
256
+ when "X"
257
+ code += "KS"
258
+ when "Z"
259
+ code += "S"
260
+ end
261
+
262
+ i += 1
263
+ end
264
+
265
+ code[0...4] # Max 4 characters
266
+ end
267
+
268
+ # Calculate Damerau-Levenshtein edit distance.
269
+ #
270
+ # Delegates to Algorithms::EditDistance so phonetically equal
271
+ # candidates are measured with the same metric as the edit
272
+ # distance strategy — an adjacent transposition ("recieve" ->
273
+ # "receive") costs 1, not 2.
274
+ #
275
+ # @param str1 [String] First string
276
+ # @param str2 [String] Second string
277
+ # @return [Integer] Edit distance
278
+ def edit_distance(str1, str2)
279
+ Algorithms::EditDistance.distance(str1, str2)
280
+ end
281
+ end
282
+ end
283
+ end
284
+ end