kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,794 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Main suggestion orchestration for spell checking.
6
+ #
7
+ # Ported from Spylls (Python) suggest.py
8
+ #
9
+ # On a bird's-eye view level, suggest does:
10
+ # 1. Tries small word "edits" (remove letters, insert letters, swap letters)
11
+ # and checks (with the help of Lookup) if there are any valid ones
12
+ # 2. If no good suggestions found, tries "ngram-based" suggestions
13
+ # (calculating ngram-based distance to all dictionary words)
14
+ # 3. If possible, tries metaphone-based suggestions (phonetic)
15
+ #
16
+ # Note: Spylls's implementation takes one liberty vs Hunspell:
17
+ # In Hunspell, ngram suggestions and phonetic suggestions are done in the
18
+ # same cycle. Spylls does them in two separate cycles for clarity.
19
+ #
20
+ # To follow algorithm details, see Suggest.suggestions method.
21
+ module Suggest
22
+ MAXPHONSUGS = 2
23
+ MAXSUGGESTIONS = 15
24
+ GOOD_EDITS = %w[spaceword uppercase replchars].freeze
25
+
26
+ # Represents a single word suggestion.
27
+ #
28
+ # Suggestions are produced internally to store enough information to
29
+ # make sure it is a good one.
30
+ class Suggestion
31
+ # @return [String] Actual suggestion text
32
+ attr_reader :text
33
+
34
+ # @return [String] How suggestion was produced (same as method name)
35
+ attr_reader :kind
36
+
37
+ def initialize(text, kind)
38
+ @text = text
39
+ @kind = kind
40
+ end
41
+
42
+ # Create a copy with changes.
43
+ #
44
+ # @param changes [Hash] Changes to apply
45
+ # @return [Suggestion] New suggestion with changes applied
46
+ def replace(**changes)
47
+ self.class.new(
48
+ changes.fetch(:text, @text),
49
+ changes.fetch(:kind, @kind)
50
+ )
51
+ end
52
+
53
+ # String representation.
54
+ #
55
+ # @return [String]
56
+ def to_s
57
+ @text
58
+ end
59
+
60
+ # Inspect string.
61
+ #
62
+ # @return [String]
63
+ def inspect
64
+ "Suggestion[#{@kind}](#{@text.inspect})"
65
+ end
66
+ end
67
+
68
+ # Represents suggestion to split words into several.
69
+ #
70
+ # Used when the algorithm suggests that a misspelled word should be
71
+ # split into multiple dictionary words.
72
+ class MultiWordSuggestion
73
+ # @return [Array<String>] List of words
74
+ attr_reader :words
75
+
76
+ # @return [String] Same as Suggestion.kind
77
+ attr_reader :source
78
+
79
+ # @return [Boolean] Whether words can be joined by dash
80
+ attr_reader :allow_dash
81
+
82
+ def initialize(words, source, allow_dash: true)
83
+ @words = words
84
+ @source = source
85
+ @allow_dash = allow_dash
86
+ end
87
+
88
+ # Convert to string suggestion.
89
+ #
90
+ # @param separator [String] Separator to join words with
91
+ # @return [Suggestion] String suggestion with joined words
92
+ def stringify(separator = ' ')
93
+ Suggestion.new(@words.join(separator), @source)
94
+ end
95
+
96
+ # Inspect string.
97
+ #
98
+ # @return [String]
99
+ def inspect
100
+ "Suggestion[#{@source}](#{@words.inspect})"
101
+ end
102
+ end
103
+
104
+ # Main suggestion class.
105
+ #
106
+ # Typically, you would not use this directly, but you might want to for
107
+ # experiments.
108
+ #
109
+ # Example:
110
+ # dictionary = Kotoshu::Dictionary.load('en_US')
111
+ # suggester = dictionary.suggester
112
+ #
113
+ # suggester.suggestions('spylls') do |suggestion|
114
+ # puts suggestion
115
+ # end
116
+ #
117
+ # # Output:
118
+ # # Suggestion[badchar](spell)
119
+ # # Suggestion[badchar](spill)
120
+ class Suggester
121
+ # @return [Object] Aff data structure (from aff file)
122
+ attr_reader :aff
123
+
124
+ # @return [Object] Dic data structure (from dic file)
125
+ attr_reader :dic
126
+
127
+ # @return [Object] Lookup object
128
+ attr_reader :lookup
129
+
130
+ def initialize(aff, dic, lookup)
131
+ @aff = aff
132
+ @dic = dic
133
+ @lookup = lookup
134
+
135
+ # Prepare words for ngram (exclude those with bad flags)
136
+ bad_flags = [
137
+ @aff[:FORBIDDENWORD],
138
+ @aff[:NOSUGGEST],
139
+ @aff[:ONLYINCOMPOUND]
140
+ ].compact
141
+
142
+ @words_for_ngram = @dic[:words].select do |word|
143
+ flags = word[:flags] || []
144
+ (flags & bad_flags).empty?
145
+ end
146
+ end
147
+
148
+ # Outer "public" interface: returns all valid suggestions as strings.
149
+ #
150
+ # Returns an enumerator for lazy evaluation.
151
+ #
152
+ # @param word [String] Word to check
153
+ # @return [Enumerator<String>] Suggestions as strings
154
+ def call(word)
155
+ return enum_for(:call, word) unless block_given?
156
+
157
+ suggestions(word) do |suggestion|
158
+ yield suggestion.text
159
+ end
160
+ end
161
+
162
+ # Main suggestion search loop.
163
+ #
164
+ # What it does, in general:
165
+ # 1. Generates possible misspelled word cases (capitalization variants)
166
+ # 2. Produces word edits with edits, checks them with Lookup
167
+ # 3. If needed, produces ngram-based suggestions
168
+ # 4. If needed, produces phonetically similar suggestions
169
+ #
170
+ # @param word [String] Word to check
171
+ # @yield [Suggestion, MultiWordSuggestion] Each suggestion object
172
+ def suggestions(word, &block)
173
+ return enum_for(:suggestions, word) unless block
174
+
175
+ # Track all suggestions we've already yielded
176
+ handled = Set.new
177
+
178
+ # Helper: Check if suggestion is a valid word
179
+ is_good_suggestion = ->(w) do
180
+ # Check if there's any good form of this exact word
181
+ # Note: We check good_forms directly to avoid ICONV and dash-breaking
182
+ good_forms = @lookup.good_forms(w, capitalization: false, allow_nosuggest: false)
183
+ good_forms.any?
184
+ end
185
+
186
+ # Helper: same as is_good_suggestion but with capitalization variants.
187
+ # Used for CHECKSHARPS where the candidate differs in case from any
188
+ # dictionary entry (e.g. "MÜSSIG" vs stem "müßig").
189
+ is_good_suggestion_cap = ->(w) do
190
+ @lookup.good_forms(w, capitalization: true, allow_nosuggest: false).any?
191
+ end
192
+
193
+ # Helper: Check if word is forbidden
194
+ is_forbidden = ->(w) do
195
+ return false unless @aff[:FORBIDDENWORD]
196
+
197
+ @dic[:has_flag]&.call(w, @aff[:FORBIDDENWORD]) || false
198
+ end
199
+
200
+ # Get capitalization type and variants
201
+ captype, variants = @aff[:casing].corrections(word)
202
+
203
+ # Special case: CHECKSHARPS with sharp-s in word
204
+ #
205
+ # German capitalizes sharp s as SS — when CHECKSHARPS is on, an
206
+ # ALLCAPS word like "MÜßIG" is wrong even though its stem "müßig"
207
+ # carries KEEPCASE. We replace ß→SS and try the result as a
208
+ # candidate.
209
+ if @aff[:CHECKSHARPS] && captype == Capitalization::Type::ALL && word.include?('ß')
210
+ sharp_swapped = word.gsub('ß', 'SS')
211
+ if is_good_suggestion_cap.call(sharp_swapped)
212
+ yield Suggestion.new(sharp_swapped, 'checksharps')
213
+ return
214
+ end
215
+ end
216
+
217
+ # Special case: FORCEUCASE with NO capitalization
218
+ if @aff[:FORCEUCASE] && captype == Capitalization::Type::NO
219
+ @aff[:casing].capitalize(word).each do |capitalized|
220
+ if is_good_suggestion.call(capitalized)
221
+ yield Suggestion.new(capitalized.capitalize, 'forceucase')
222
+ return
223
+ end
224
+ end
225
+ end
226
+
227
+ # Dot-separated mixed-case misspelling: "something.The" is
228
+ # suggested as "something. The" (see try_dotsplit_suggestion).
229
+ try_dotsplit_suggestion(word, captype,
230
+ is_forbidden: is_forbidden, handled: handled, &block)
231
+
232
+ good_edits_found = false
233
+
234
+ # Process each capitalization variant
235
+ variants.each_with_index do |variant, idx|
236
+ collected, stop, _nocompound, good_edits =
237
+ collect_variant_suggestions(
238
+ variant, idx,
239
+ word: word,
240
+ captype: captype,
241
+ is_forbidden: is_forbidden,
242
+ handled: handled,
243
+ is_good_suggestion: is_good_suggestion
244
+ )
245
+ good_edits_found ||= good_edits
246
+
247
+ # Upstream Hunspell (hunspell.cxx suggest_internal, the
248
+ # HUHCAP/HUHINITCAP "aNew -> a New" loop) reorders the
249
+ # suggestions of the lowercased pass so that space-containing
250
+ # ones come first — in front of everything yielded so far,
251
+ # including the lowercased form itself.
252
+ if idx.positive? && [Capitalization::Type::HUH, Capitalization::Type::HUHINIT].include?(captype)
253
+ collected = move_space_splits_to_front(collected, word)
254
+ end
255
+
256
+ collected.each(&block)
257
+ return if stop
258
+ end
259
+
260
+ # Skip ngram/phonetic if we found good edits
261
+ return if good_edits_found
262
+
263
+ # Try fixing words with dashes
264
+ if word.include?('-') && handled.none? { |s| s.include?('-') }
265
+ chunks = word.split('-')
266
+ chunks.each_with_index do |chunk, idx|
267
+ next if is_good_suggestion.call(chunk)
268
+
269
+ # Try all suggestions for this chunk
270
+ call(chunk).each do |sug|
271
+ candidate = chunks[0...idx] + [sug] + chunks[(idx + 1)..]
272
+ candidate_str = candidate.join('-')
273
+
274
+ # Check if the whole word with replacement is good
275
+ if @lookup.call(candidate_str, capitalization: true, allow_nosuggest: true)
276
+ yield Suggestion.new(candidate_str, 'dashes')
277
+ end
278
+ end
279
+
280
+ # Only try one misspelled chunk
281
+ break
282
+ end
283
+ end
284
+
285
+ # Ngram-based suggestions
286
+ if @aff[:MAXNGRAMSUGS]&.positive?
287
+ limit = @aff[:MAXNGRAMSUGS]
288
+ ngrams_seen = 0
289
+ ngram_suggestions(word, captype: captype, handled: handled) do |sug|
290
+ handle_found(
291
+ Suggestion.new(sug, 'ngram'),
292
+ word: word,
293
+ captype: captype,
294
+ is_forbidden: is_forbidden,
295
+ handled: handled,
296
+ check_inclusion: true
297
+ ) do |suggestion|
298
+ yield suggestion
299
+ ngrams_seen += 1
300
+ end
301
+ # break out of ngram_suggestions, not just handle_found
302
+ break if ngrams_seen >= limit
303
+ end
304
+ end
305
+
306
+ # Phonetic suggestions
307
+ if @aff[:PHONE]
308
+ phonet_seen = 0
309
+ phonet_suggestions(word) do |sug|
310
+ handle_found(
311
+ Suggestion.new(sug, 'phonet'),
312
+ word: word,
313
+ captype: captype,
314
+ is_forbidden: is_forbidden,
315
+ handled: handled,
316
+ check_inclusion: true
317
+ ) do |suggestion|
318
+ yield suggestion
319
+ phonet_seen += 1
320
+ end
321
+ break if phonet_seen >= MAXPHONSUGS
322
+ end
323
+ end
324
+ end
325
+
326
+ # Generate all possible word edits in order of priority.
327
+ #
328
+ # Order is important - it's the order user receives suggestions.
329
+ #
330
+ # @param word [String] Word to mutate
331
+ # @yield [Suggestion, MultiWordSuggestion] Each edit suggestion
332
+ def edits(word)
333
+ # Uppercase suggestion (html -> HTML)
334
+ yield Suggestion.new(@aff[:casing].upper(word), 'uppercase')
335
+
336
+ # REP table replacements
337
+ reptable = @aff[:REP] || []
338
+ Permutations.replchars(word, reptable) do |suggestion|
339
+ if suggestion.is_a?(Array)
340
+ # Multi-word suggestion from REP with underscore
341
+ yield Suggestion.new(suggestion.join(' '), 'replchars')
342
+ yield MultiWordSuggestion.new(suggestion, 'replchars', allow_dash: false)
343
+ else
344
+ yield Suggestion.new(suggestion, 'replchars')
345
+ end
346
+ end
347
+
348
+ # Split into two words (spaceword)
349
+ Permutations.twowords(word) do |words|
350
+ yield Suggestion.new(words.join(' '), 'spaceword')
351
+ yield Suggestion.new(words.join('-'), 'spaceword') if use_dash?
352
+ end
353
+
354
+ # MAP table (related character replacements)
355
+ maptable = @aff[:MAP] || []
356
+ Permutations.mapchars(word, maptable) do |suggestion|
357
+ yield Suggestion.new(suggestion, 'mapchars')
358
+ end
359
+
360
+ # Swap adjacent characters
361
+ Permutations.swapchar(word) do |suggestion|
362
+ yield Suggestion.new(suggestion, 'swapchar')
363
+ end
364
+
365
+ # Long swaps (up to 4 chars distance)
366
+ Permutations.longswapchar(word) do |suggestion|
367
+ yield Suggestion.new(suggestion, 'longswapchar')
368
+ end
369
+
370
+ # Replace with keyboard-adjacent chars
371
+ layout = @aff[:KEY] || ''
372
+ Permutations.badcharkey(word, layout) do |suggestion|
373
+ yield Suggestion.new(suggestion, 'badcharkey')
374
+ end
375
+
376
+ # Remove one character
377
+ Permutations.extrachar(word) do |suggestion|
378
+ yield Suggestion.new(suggestion, 'extrachar')
379
+ end
380
+
381
+ # Insert one character (from TRY string)
382
+ trystring = @aff[:TRY] || ''
383
+ Permutations.forgotchar(word, trystring) do |suggestion|
384
+ yield Suggestion.new(suggestion, 'forgotchar')
385
+ end
386
+
387
+ # Move character forward/backward
388
+ Permutations.movechar(word) do |suggestion|
389
+ yield Suggestion.new(suggestion, 'movechar')
390
+ end
391
+
392
+ # Replace each character
393
+ Permutations.badchar(word, trystring) do |suggestion|
394
+ yield Suggestion.new(suggestion, 'badchar')
395
+ end
396
+
397
+ # Fix two-character doubling
398
+ Permutations.doubletwochars(word) do |suggestion|
399
+ yield Suggestion.new(suggestion, 'doubletwochars')
400
+ end
401
+
402
+ # Split by space in all positions
403
+ unless @aff[:NOSPLITSUGS]
404
+ Permutations.twowords(word) do |words|
405
+ yield MultiWordSuggestion.new(words, 'twowords', allow_dash: use_dash?)
406
+ end
407
+ end
408
+ end
409
+
410
+ # Generate edit suggestions and filter for valid words.
411
+ #
412
+ # @param word [String] Word to generate edits for
413
+ # @param compounds [Boolean] Whether to check compound words
414
+ # @param limit [Integer] Maximum number of suggestions to yield
415
+ # @yield [Suggestion, MultiWordSuggestion] Each valid edit suggestion
416
+ def edit_suggestions(word, compounds:, limit:)
417
+ count = 0
418
+
419
+ edits(word) do |suggestion|
420
+ break if count > limit
421
+
422
+ filtered = filter_suggestion(suggestion, compounds)
423
+ next unless filtered
424
+
425
+ Array(filtered).each do |sug|
426
+ yield sug
427
+ count += 1
428
+ end
429
+ end
430
+ end
431
+
432
+ # Generate ngram-based suggestions.
433
+ #
434
+ # @param word [String] Misspelled word
435
+ # @param captype [Symbol] Capitalization type of the misspelled word
436
+ # @param handled [Set<String>] Already suggested words
437
+ # @yield [String] Each ngram suggestion
438
+ def ngram_suggestions(word, captype:, handled:, &block)
439
+ return unless @aff[:MAXNGRAMSUGS]&.positive?
440
+
441
+ known_lower = handled.map(&:downcase).to_set
442
+
443
+ NgramSuggest.suggest(
444
+ word.downcase,
445
+ dictionary_words: ngram_candidate_words(captype),
446
+ prefixes: @aff[:prefixes_by_flag] || {},
447
+ suffixes: @aff[:suffixes_by_flag] || {},
448
+ known: known_lower,
449
+ maxdiff: @aff[:MAXDIFF] || 2,
450
+ onlymaxdiff: @aff[:ONLYMAXDIFF] || false,
451
+ has_phonetic: !@aff[:PHONE].nil?, &block
452
+ )
453
+ end
454
+
455
+ # Generate phonetic suggestions.
456
+ #
457
+ # @param word [String] Misspelled word
458
+ # @yield [String] Each phonetic suggestion
459
+ def phonet_suggestions(word, &)
460
+ return unless @aff[:PHONE]
461
+
462
+ PhonetSuggest.suggest(
463
+ word,
464
+ dictionary_words: @words_for_ngram,
465
+ table: @aff[:PHONE], &
466
+ )
467
+ end
468
+
469
+ # Check if dashes are allowed for joining words.
470
+ #
471
+ # Definition from Hunspell: Either dash is in TRY directive, or TRY
472
+ # indicates Latinic script (by having 'a').
473
+ #
474
+ # @return [Boolean] Whether dashes are allowed
475
+ def use_dash?
476
+ try_chars = @aff[:TRY] || ''
477
+ try_chars.include?('-') || try_chars.include?('a')
478
+ end
479
+
480
+ # Collect every valid suggestion of one capitalization variant.
481
+ #
482
+ # Buffers instead of yielding so the caller can reorder the pass
483
+ # (see move_space_splits_to_front) before emitting it. Mirrors
484
+ # the per-variant part of Hunspell's suggest flow: the lowercased
485
+ # form itself first (if valid), then the non-compound edit pass,
486
+ # then the compound edit pass — the latter skipped after a "good"
487
+ # edit (uppercase/replchars/mapchars) or a dictionary word-pair
488
+ # spaceword, which ends the whole search.
489
+ #
490
+ # @param variant [String] Capitalization variant to process
491
+ # @param idx [Integer] Index of the variant (0 = as typed)
492
+ # @param word [String] Original misspelled word
493
+ # @param captype [Symbol] Capitalization type of the original word
494
+ # @param is_forbidden [Proc] Function to check if word is forbidden
495
+ # @param handled [Set<String>] Already suggested words
496
+ # @param is_good_suggestion [Proc] Function to check a candidate
497
+ # @return [Array<Array<Suggestion>, Boolean, Boolean, Boolean>]
498
+ # Collected suggestions, stop flag, nocompound flag, good-edits
499
+ # flag — in that order
500
+ def collect_variant_suggestions(variant, idx, word:, captype:, is_forbidden:, handled:,
501
+ is_good_suggestion:)
502
+ collected = []
503
+ stop = false
504
+ nocompound = false
505
+ good_edits = false
506
+
507
+ # If different from original and is good, suggest it
508
+ if idx.positive? && is_good_suggestion.call(variant)
509
+ handle_found(
510
+ Suggestion.new(variant, 'case'),
511
+ word: word,
512
+ captype: captype,
513
+ is_forbidden: is_forbidden,
514
+ handled: handled
515
+ ) do |handled_suggestion|
516
+ collected << handled_suggestion
517
+ end
518
+ end
519
+
520
+ # Generate and check edits (non-compound first)
521
+ edit_suggestions(variant, compounds: false, limit: MAXSUGGESTIONS) do |suggestion|
522
+ handle_found(
523
+ suggestion,
524
+ word: word,
525
+ captype: captype,
526
+ is_forbidden: is_forbidden,
527
+ handled: handled,
528
+ check_inclusion: false
529
+ ) do |handled_suggestion|
530
+ next if stop
531
+
532
+ collected << handled_suggestion
533
+
534
+ kind = handled_suggestion.kind
535
+ good_edits = true if GOOD_EDITS.include?(kind)
536
+ nocompound = true if %w[uppercase replchars mapchars].include?(kind)
537
+
538
+ # If we found a spaceword that's in the dictionary as a whole,
539
+ # that's the only suggestion we need
540
+ stop = true if kind == 'spaceword'
541
+ end
542
+ end
543
+
544
+ # Generate compound suggestions if not excluded
545
+ unless nocompound || stop
546
+ limit = @aff[:MAXCPDSUGS] || MAXSUGGESTIONS
547
+ edit_suggestions(variant, compounds: true, limit: limit) do |suggestion|
548
+ handle_found(
549
+ suggestion,
550
+ word: word,
551
+ captype: captype,
552
+ is_forbidden: is_forbidden,
553
+ handled: handled,
554
+ check_inclusion: false
555
+ ) do |handled_suggestion|
556
+ collected << handled_suggestion
557
+ good_edits = true if GOOD_EDITS.include?(handled_suggestion.kind)
558
+ end
559
+ end
560
+ end
561
+
562
+ [collected, stop, nocompound, good_edits]
563
+ end
564
+
565
+ # Move space-containing suggestions to the front of the pass.
566
+ #
567
+ # Ported from Hunspell's "aNew -> a New" loop in the HUHCAP/
568
+ # HUHINITCAP branch of suggest_internal (hunspell.cxx): a
569
+ # suggestion from the lowercased pass that contains a space moves
570
+ # to the front when the part after the space differs from the
571
+ # input's corresponding suffix. Spylls does not port this; the
572
+ # opentaal_keepcase fixture (word-TV -> "word -tv, word-tv, word")
573
+ # encodes it.
574
+ #
575
+ # Hunspell erases each matching suggestion and inserts it at the
576
+ # begin position in a single forward sweep, so with several
577
+ # matches the last one ends up first — mirrored here by reversing.
578
+ #
579
+ # @param suggestions [Array<Suggestion>] Suggestions of one variant pass
580
+ # @param word [String] The original misspelled word
581
+ # @return [Array<Suggestion>] Reordered suggestions
582
+ def move_space_splits_to_front(suggestions, word)
583
+ moving = []
584
+ result = []
585
+
586
+ suggestions.each do |s|
587
+ pos = s.text.index(' ')
588
+ second = pos ? s.text[(pos + 1)..] : nil
589
+ moves = !second.nil? &&
590
+ second.length < word.length &&
591
+ word[word.length - second.length, second.length] != second
592
+
593
+ if moves
594
+ moving << s
595
+ else
596
+ result << s
597
+ end
598
+ end
599
+
600
+ moving.reverse + result
601
+ end
602
+
603
+ private
604
+
605
+ # Handle a found suggestion with proper capitalization and validation.
606
+ #
607
+ # @param suggestion [Suggestion, MultiWordSuggestion] Raw suggestion
608
+ # @param word [String] Original misspelled word (used for case preservation)
609
+ # @param captype [Symbol] Original word's capitalization type
610
+ # @param is_forbidden [Proc] Function to check if word is forbidden
611
+ # @param handled [Set<String>] Already handled suggestions
612
+ # @param check_inclusion [Boolean] Whether to check for subsumption
613
+ # @yield [Suggestion] Processed suggestion if valid
614
+ def handle_found(suggestion, word:, captype:, is_forbidden:, handled:, check_inclusion: false)
615
+ return unless block_given?
616
+
617
+ text = suggestion.text
618
+
619
+ # Apply capitalization coercion
620
+ unless @aff[:KEEPCASE] && suggestion_has_keepcase_flag?(suggestion)
621
+ text = @aff[:casing].coerce(text, captype)
622
+
623
+ # If coerced form is forbidden, revert to original
624
+ if text != suggestion.text && is_forbidden.call(text)
625
+ text = suggestion.text
626
+ end
627
+
628
+ # "aNew" coerces to "a new"; restore the original word's capital
629
+ # at the split boundary so it reads "a New" (matching the input's
630
+ # HUHINIT/HUH case pattern).
631
+ if [Capitalization::Type::HUH, Capitalization::Type::HUHINIT].include?(captype) && text.include?(' ')
632
+ pos = text.index(' ')
633
+ if pos && text[pos + 1] != word[pos] && text[pos + 1]&.upcase == word[pos]
634
+ text = text[0...pos + 1] + word[pos] + text[(pos + 2)..]
635
+ end
636
+ end
637
+ end
638
+
639
+ # Skip if forbidden
640
+ return if is_forbidden.call(text)
641
+
642
+ # Apply OCONV transformation if present
643
+ if @aff[:OCONV]
644
+ text = @aff[:OCONV].call(text)
645
+ end
646
+
647
+ # Skip if already seen
648
+ return if handled.include?(text)
649
+
650
+ # Never suggest the misspelling itself. Hunspell works on the
651
+ # lowercased variant and re-cases results at the end, so no
652
+ # candidate can round-trip back to the (invalid) input. Our
653
+ # capitalization coercion can do exactly that — e.g.
654
+ # opentaal_keepcase's "tv-word" coerced to INITCAP reproduces
655
+ # the misspelling "Tv-word" — so drop a candidate identical to
656
+ # the input, unless the input is itself correctly spelled (a
657
+ # suggest() call on a valid word may echo it).
658
+ return if text == word && !@lookup.good_forms(word, capitalization: true, allow_nosuggest: false).any?
659
+
660
+ # Skip if subsumed by existing suggestion
661
+ if check_inclusion && handled.any? { |prev| text.downcase.include?(prev.downcase) }
662
+ return
663
+ end
664
+
665
+ handled.add(text)
666
+ yield suggestion.replace(text: text)
667
+ end
668
+
669
+ # Suggest inserting a space after the first dot of a mixed-case
670
+ # misspelling: "something.The" -> "something. The".
671
+ #
672
+ # Upstream Hunspell (hunspell.cxx suggest_internal, the
673
+ # HUHCAP/HUHINITCAP branch) finds the first '.' in the word and,
674
+ # when the part after it is INITCAP-cased, inserts a space after
675
+ # the dot and pushes the result to the FRONT of the suggestion
676
+ # list (insert_sug). Spylls does not port this rule; we mirror
677
+ # Hunspell because the sug/sugutf fixtures encode it.
678
+ # (Suggestion 2C marked for kotoshu-rs re-sync.)
679
+ #
680
+ # @param word [String] Misspelled word
681
+ # @param captype [Symbol] Capitalization type of the misspelled word
682
+ # @param is_forbidden [Proc] Function to check if a word is forbidden
683
+ # @param handled [Set<String>] Already suggested words
684
+ # @yield [Suggestion] The dot-split suggestion, if valid
685
+ def try_dotsplit_suggestion(word, captype, is_forbidden:, handled:, &block)
686
+ return unless [Capitalization::Type::HUH, Capitalization::Type::HUHINIT].include?(captype)
687
+
688
+ dot_pos = word.index('.')
689
+ return unless dot_pos
690
+
691
+ postdot = word[(dot_pos + 1)..]
692
+ return unless @aff[:casing].guess(postdot) == Capitalization::Type::INIT
693
+
694
+ candidate = word.dup
695
+ candidate.insert(dot_pos + 1, ' ')
696
+ handle_found(
697
+ Suggestion.new(candidate, 'dotsplit'),
698
+ word: word,
699
+ captype: captype,
700
+ is_forbidden: is_forbidden,
701
+ handled: handled, &block
702
+ )
703
+ end
704
+
705
+ # Ngram root candidate pool for a misspelling of the given captype.
706
+ #
707
+ # Upstream Hunspell (suggestmgr.cxx ngsuggest, "skip exceptions")
708
+ # refuses capitalized dictionary words as ngram roots when the
709
+ # misspelling itself is all lowercase: a missing Shift press is the
710
+ # edit phase's job (badcharkey's case toggle), and letting INITCAP
711
+ # stems into the ngram ranking buries every other candidate behind
712
+ # their +2000 case-match bonus. Exceptions, mirrored from the same
713
+ # guard: dictionaries with a PHONE table, German dictionaries
714
+ # (regular nouns are capitalized there), and words carrying `ph:`
715
+ # phonetic hints (H_OPT_PHON).
716
+ #
717
+ # Spylls does not port this skip; the keepcase fixture (`bar` ->
718
+ # `Bar, baz.`) encodes it. (Suggestion 2C marked for kotoshu-rs
719
+ # re-sync.)
720
+ #
721
+ # @param captype [Symbol] Capitalization type of the misspelling
722
+ # @return [Array<Hash>] Dictionary words usable as ngram roots
723
+ def ngram_candidate_words(captype)
724
+ return @words_for_ngram unless captype == Capitalization::Type::NO
725
+ return @words_for_ngram if @aff[:PHONE]
726
+ return @words_for_ngram if @aff[:casing].is_a?(Capitalization::GermanCasing)
727
+
728
+ @words_for_ngram.reject { |entry| initcap_ngram_root?(entry) }
729
+ end
730
+
731
+ # Whether a dictionary entry is an INITCAP stem without `ph:` hints.
732
+ #
733
+ # @param entry [Hash] Dictionary word entry
734
+ # @return [Boolean]
735
+ def initcap_ngram_root?(entry)
736
+ return false if entry[:alt_spellings]&.any?
737
+
738
+ stem = entry[:stem].to_s
739
+ @aff[:casing].guess(stem) == Capitalization::Type::INIT
740
+ end
741
+
742
+ # Check if suggestion's stem carries the KEEPCASE flag.
743
+ #
744
+ # Mirrors Spylls suggest.py:206: when KEEPCASE is set and the
745
+ # candidate's stem has the flag, the candidate's case must NOT be
746
+ # coerced to the misspelling's captype (e.g. input "FOO" should
747
+ # suggest "foo", not "FOO"). The CHECKSHARPS exception is that for
748
+ # ß-containing words, KEEPCASE has its German-specific meaning
749
+ # (ß↔SS) and case coercion still applies.
750
+ #
751
+ # @param suggestion [Suggestion, MultiWordSuggestion]
752
+ # @return [Boolean]
753
+ def suggestion_has_keepcase_flag?(suggestion)
754
+ return false unless @aff[:KEEPCASE]
755
+ return false if @aff[:CHECKSHARPS] && suggestion.text.include?('ß')
756
+
757
+ @dic[:has_flag]&.call(suggestion.text, @aff[:KEEPCASE]) || false
758
+ end
759
+
760
+ # Filter suggestion to only valid words.
761
+ #
762
+ # For MultiWordSuggestion with allow_dash and use_dash?, returns BOTH
763
+ # the space-joined and dash-joined forms (matching Hunspell behavior).
764
+ #
765
+ # @param suggestion [Suggestion, MultiWordSuggestion]
766
+ # @param compounds [Boolean] Whether to check compound forms
767
+ # @return [Suggestion, Array<Suggestion>, nil] Filtered suggestion(s) or nil
768
+ def filter_suggestion(suggestion, compounds)
769
+ is_good = ->(word) do
770
+ if compounds
771
+ @lookup.good_forms(word, capitalization: false, allow_nosuggest: false, affix_forms: false).any?
772
+ else
773
+ @lookup.good_forms(word, capitalization: false, allow_nosuggest: false, compound_forms: false).any?
774
+ end
775
+ end
776
+
777
+ if suggestion.is_a?(MultiWordSuggestion)
778
+ return nil unless suggestion.words.all? { |w| is_good.call(w) }
779
+
780
+ if suggestion.allow_dash && use_dash?
781
+ [suggestion.stringify(' '), suggestion.stringify('-')]
782
+ else
783
+ suggestion.stringify(' ')
784
+ end
785
+ else
786
+ return nil unless is_good.call(suggestion.text)
787
+
788
+ suggestion
789
+ end
790
+ end
791
+ end
792
+ end
793
+ end
794
+ end