kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,367 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # N-gram based suggestion algorithm.
6
+ #
7
+ # Ported from Spylls (Python) ngram_suggest.py
8
+ #
9
+ # This is the core Hunspell suggestion algorithm that uses n-gram
10
+ # similarity to rank and filter spelling corrections.
11
+ #
12
+ # The algorithm works in three stages:
13
+ # 1. root_score: Quick n-gram score + left common substring
14
+ # 2. rough_affix_score: Affixed form n-gram score
15
+ # 3. precise_affix_score: Full scoring with LCS, bigrams, etc.
16
+ module NgramSuggest
17
+ # Maximum number of root words to consider in first pass
18
+ MAX_ROOTS = 100
19
+
20
+ # Maximum number of suggestions to generate
21
+ MAX_GUESSES = 200
22
+
23
+ class << self
24
+ # Main entry point for n-gram based suggestions.
25
+ #
26
+ # @param misspelling [String] The misspelled word
27
+ # @param dictionary_words [Array<Hash>] Dictionary entries with stem and flags
28
+ # @param prefixes [Hash] Prefix flags to prefix objects mapping
29
+ # @param suffixes [Hash] Suffix flags to suffix objects mapping
30
+ # @param known [Set<String>] Already suggested words (to avoid duplicates)
31
+ # @param maxdiff [Integer] MAXDIFF value from aff file (0-10)
32
+ # @param onlymaxdiff [Boolean] ONLYMAXDIFF flag
33
+ # @param has_phonetic [Boolean] Whether PHONE table exists in aff file
34
+ # @yield [String] Each suggestion
35
+ #
36
+ # This is a simplified version that works with basic dictionary structures.
37
+ # Full implementation would need affix flag parsing and Word model objects.
38
+ def suggest(misspelling,
39
+ dictionary_words:,
40
+ prefixes: {},
41
+ suffixes: {},
42
+ known: Set.new,
43
+ maxdiff: 2,
44
+ onlymaxdiff: true,
45
+ has_phonetic: false,
46
+ &block)
47
+ # Stage 1: Find best root candidates by n-gram score
48
+ root_scores = []
49
+
50
+ dictionary_words.each do |word_entry|
51
+ stem = word_entry[:stem] || word_entry
52
+
53
+ # Skip words with length difference > 4
54
+ next if (stem.length - misspelling.length).abs > 4
55
+
56
+ # Use the best score across the stem and any ph: alt-spellings,
57
+ # matching Hunspell's ngram_suggest behavior (without this,
58
+ # dictionaries that rely on `ph:` for phonetic hints never
59
+ # surface those words as candidates).
60
+ score = if word_entry[:alt_spellings]&.any?
61
+ alts = word_entry[:alt_spellings].map do |alt|
62
+ root_score(misspelling, alt)
63
+ end
64
+ [root_score(misspelling, stem), *alts].max
65
+ else
66
+ root_score(misspelling, stem)
67
+ end
68
+
69
+ # Use heap to keep only MAX_ROOTS best results
70
+ if root_scores.size >= MAX_ROOTS
71
+ # Keep only the best scores
72
+ root_scores = root_scores.sort.reverse.first(MAX_ROOTS)
73
+ end
74
+
75
+ root_scores << [score, word_entry] if score > 0
76
+ end
77
+
78
+ # Stage 2: Generate affixed forms and score them
79
+ threshold = detect_threshold(misspelling)
80
+ guess_scores = []
81
+
82
+ # Sort by score descending
83
+ root_scores.sort_by { |score, _| -score }.first(MAX_ROOTS).each do |(_, root_entry)|
84
+ root = root_entry[:stem] || root_entry
85
+
86
+ # Alt spellings (from `ph:` morph data): if the alt form passes
87
+ # the threshold, we suggest the STEM (not the alt) — this is
88
+ # how Hunspell/Spylls surface dictionary entries whose canonical
89
+ # form is unrelated but whose pronunciation matches. The alt is
90
+ # only used for scoring; the real suggestion is the stem.
91
+ if root_entry[:alt_spellings]
92
+ root_entry[:alt_spellings].each do |variant|
93
+ score = rough_affix_score(misspelling, variant.downcase)
94
+ next unless score > threshold
95
+
96
+ guess_scores << [score, variant, root]
97
+ end
98
+ end
99
+
100
+ # Generate forms with suffixes
101
+ forms = forms_for(root_entry, prefixes, suffixes, similar_to: misspelling)
102
+
103
+ forms.each do |form|
104
+ score = rough_affix_score(misspelling, form.to_s.downcase)
105
+ next unless score > threshold
106
+
107
+ guess_scores << [score, form.to_s, form.to_s]
108
+ end
109
+ end
110
+
111
+ # Limit to MAX_GUESSES. Use stable descending sort (Ruby's sort_by
112
+ # with a negated key is stable, matching Python's
113
+ # sorted(key=..., reverse=True) which preserves input order for
114
+ # ties — important for reproducibility against Hunspell fixtures
115
+ # where dictionary order matters).
116
+ guesses = guess_scores.sort_by { |score, _, _| -score }.first(MAX_GUESSES)
117
+
118
+ # Stage 3: Calculate precise scores
119
+ fact = maxdiff >= 0 ? (10.0 - maxdiff) / 5.0 : 1.0
120
+
121
+ guesses2 = guesses.map do |score, compared, real|
122
+ [precise_affix_score(misspelling, compared.to_s.downcase,
123
+ fact, base: score, has_phonetic: has_phonetic), real.to_s]
124
+ end.sort_by { |score, _| -score }
125
+
126
+ # Stage 4: Filter and yield suggestions
127
+ filter_guesses(guesses2, known: known, onlymaxdiff: onlymaxdiff, &block)
128
+ end
129
+
130
+ # Stage 1 scoring: 3-gram score + left common substring.
131
+ #
132
+ # @param word1 [String] Misspelled word
133
+ # @param word2 [String] Possible suggestion
134
+ # @return [Float] Root score
135
+ def root_score(word1, word2)
136
+ # Use lowercase for comparison as per Hunspell
137
+ word2_lower = word2.downcase
138
+
139
+ StringMetrics.ngram(3, word1, word2_lower, longer_worse: true) +
140
+ StringMetrics.leftcommonsubstring(word1, word2_lower).to_f
141
+ end
142
+
143
+ # Stage 2 scoring: N-gram score with n=len(word1) + left common substring.
144
+ #
145
+ # @param word1 [String] Misspelled word
146
+ # @param word2 [String] Possible suggestion
147
+ # @return [Float] Rough affix score
148
+ def rough_affix_score(word1, word2)
149
+ # Use lowercase for comparison as per Hunspell
150
+ word2_lower = word2.downcase
151
+
152
+ StringMetrics.ngram(word1.length, word1, word2_lower, any_mismatch: true) +
153
+ StringMetrics.leftcommonsubstring(word1, word2_lower).to_f
154
+ end
155
+
156
+ # Stage 3 scoring: Full precise scoring.
157
+ #
158
+ # Returns one of three "score groups":
159
+ # - > 1000: Very good (same word, different casing)
160
+ # - < -100: Questionable (too different)
161
+ # - -100 to 1000: Normal suggestion
162
+ #
163
+ # @param word1 [String] Misspelled word
164
+ # @param word2 [String] Possible suggestion
165
+ # @param diff_factor [Float] Factor based on MAXDIFF (0-2)
166
+ # @param base [Float] Base score from stage 2
167
+ # @param has_phonetic [Boolean] Whether PHONE table exists
168
+ # @return [Float] Precise affix score
169
+ def precise_affix_score(word1, word2, diff_factor, base:, has_phonetic: false)
170
+ # Use lowercase for LCS to catch case-only differences
171
+ word1_lower = word1.downcase
172
+ word2_lower = word2.downcase
173
+
174
+ lcs = StringMetrics.lcslen(word1_lower, word2_lower)
175
+
176
+ # Same characters with different casing -- "very good" suggestion
177
+ if word1.length == word2.length && word1.length == lcs
178
+ return base + 2000
179
+ end
180
+
181
+ # Score is: 2 * LCS - length difference
182
+ result = (2 * lcs) - (word1.length - word2.length).abs
183
+
184
+ # Add common start substring length
185
+ result += StringMetrics.leftcommonsubstring(word1_lower, word2_lower)
186
+
187
+ # Add 1 if any characters match at same positions
188
+ result += 1 if StringMetrics.commoncharacters(word1_lower, word2_lower) > 0
189
+
190
+ # Add regular 4-gram score
191
+ result += StringMetrics.ngram(4, word1_lower, word2_lower, any_mismatch: true)
192
+
193
+ # Add weighted bigrams (both directions)
194
+ bigrams = (
195
+ StringMetrics.ngram(2, word1_lower, word2_lower, any_mismatch: true, weighted: true) +
196
+ StringMetrics.ngram(2, word2_lower, word1_lower, any_mismatch: true, weighted: true)
197
+ )
198
+ result += bigrams
199
+
200
+ # Apply "questionable" threshold based on diff_factor and has_phonetic
201
+ questionable_limit = if has_phonetic
202
+ word2.length * diff_factor
203
+ else
204
+ (word1.length + word2.length) * diff_factor
205
+ end
206
+
207
+ result -= 1000 if bigrams < questionable_limit
208
+
209
+ result
210
+ end
211
+
212
+ # Calculate minimum threshold for passable suggestions.
213
+ #
214
+ # Mangles the word in 3 different ways (replacing each 4th char with '*')
215
+ # and scores them to generate a minimum acceptable score.
216
+ #
217
+ # @param word [String] The misspelled word
218
+ # @return [Float] Minimum threshold score
219
+ def detect_threshold(word)
220
+ thresh = 0.0
221
+
222
+ (1..3).each do |start_pos|
223
+ mangled = word.chars.map.with_index do |char, pos|
224
+ ((pos - start_pos) % 4).zero? && pos >= start_pos ? "*" : char
225
+ end.join
226
+
227
+ thresh += StringMetrics.ngram(word.length, word, mangled, any_mismatch: true)
228
+ end
229
+
230
+ # Take average of the three scores and subtract 1
231
+ (thresh / 3.0) - 1
232
+ end
233
+
234
+ # Generate all possible affixed forms for a dictionary word.
235
+ #
236
+ # For each flag the word carries, the corresponding affix entries are
237
+ # considered. A suffix/prefix is only kept when (a) its condition
238
+ # matches the stem (e.g. `[^ey]$` for the `-ed` suffix on "look"),
239
+ # and (b) its `add` is a suffix/prefix of the misspelling. This
240
+ # two-clause filter is what keeps the candidate space bounded — the
241
+ # condition check rejects entries that simply can't apply to this
242
+ # stem, and the `similar_to` check rejects entries that can't
243
+ # produce the misspelling we're trying to fix.
244
+ #
245
+ # Then, for every valid suffix we produce `stem + add` (with the
246
+ # suffix's `strip` length removed from the stem end). For every
247
+ # valid prefix we produce `add + stem` (with the prefix's `strip`
248
+ # length removed from the stem start). For every (prefix, suffix)
249
+ # cross-product pair, we produce `prefix.add + (stripped stem) +
250
+ # suffix.add`.
251
+ #
252
+ # The base stem is always the first form in the result.
253
+ #
254
+ # @param word_entry [Hash] Dictionary word with stem and flags
255
+ # @param all_prefixes [Hash] Flag → list of prefix hashes
256
+ # @param all_suffixes [Hash] Flag → list of suffix hashes
257
+ # @param similar_to [String] Misspelling being corrected (used as
258
+ # the suffix/prefix filter)
259
+ # @return [Array<String>] Generated affixed forms
260
+ def forms_for(word_entry, all_prefixes, all_suffixes, similar_to:)
261
+ stem = word_entry[:stem] || word_entry
262
+ flags = word_entry[:flags] || []
263
+
264
+ res = [stem]
265
+
266
+ similar = similar_to.to_s
267
+
268
+ suffixes = flags.flat_map { |f| all_suffixes[f] || [] }
269
+ prefixes = flags.flat_map { |f| all_prefixes[f] || [] }
270
+
271
+ applicable_suffixes = suffixes.select do |suffix|
272
+ add = suffix[:affix]
273
+ next false if add.nil? || add.empty?
274
+ next false if similar.length < add.length
275
+ next false unless similar.end_with?(add)
276
+
277
+ checker = suffix[:condition_checker]
278
+ checker.nil? || checker.matches?(stem)
279
+ end
280
+
281
+ applicable_prefixes = prefixes.select do |prefix|
282
+ add = prefix[:affix]
283
+ next false if add.nil? || add.empty?
284
+ next false if similar.length < add.length
285
+ next false unless similar.start_with?(add)
286
+
287
+ checker = prefix[:condition_checker]
288
+ checker.nil? || checker.matches?(stem)
289
+ end
290
+
291
+ cross = applicable_prefixes.product(applicable_suffixes).select do |prefix, suffix|
292
+ prefix[:crossproduct] && suffix[:crossproduct]
293
+ end
294
+
295
+ applicable_suffixes.each do |suffix|
296
+ strip = suffix[:strip] || ''
297
+ add = suffix[:affix]
298
+ root = strip.empty? ? stem : stem[0...(stem.length - strip.length)]
299
+ res << (root + add)
300
+ end
301
+
302
+ applicable_prefixes.each do |prefix|
303
+ strip = prefix[:strip] || ''
304
+ add = prefix[:affix]
305
+ root = strip.empty? ? stem : stem[strip.length..]
306
+ res << (add + root)
307
+ end
308
+
309
+ cross.each do |prefix, suffix|
310
+ pstrip = prefix[:strip] || ''
311
+ sstrip = suffix[:strip] || ''
312
+ pad = prefix[:affix]
313
+ sad = suffix[:affix]
314
+ base = stem.dup
315
+ base = base[pstrip.length..] if !pstrip.empty? && base.start_with?(pstrip)
316
+ base = base[0...(base.length - sstrip.length)] if !sstrip.empty? && base.end_with?(sstrip)
317
+ res << (pad + base + sad)
318
+ end
319
+
320
+ res.uniq
321
+ end
322
+
323
+ # Filter guesses by score into quality buckets.
324
+ #
325
+ # Score buckets:
326
+ # - > 1000: Very good (same word, different casing)
327
+ # - 1000 to -100: Normal suggestions
328
+ # - < -100: Questionable (too different)
329
+ #
330
+ # Stops yielding when:
331
+ # - A very good suggestion was found and then a normal one
332
+ # - A questionable suggestion was found (only yields one)
333
+ #
334
+ # @param guesses [Array<Array>] Array of [score, value] pairs
335
+ # @param known [Set<String>] Already suggested words
336
+ # @param onlymaxdiff [Boolean] Whether to exclude questionable
337
+ # @yield [String] Each filtered suggestion
338
+ def filter_guesses(guesses, known:, onlymaxdiff: true)
339
+ seen = false
340
+ found = 0
341
+
342
+ guesses.each do |score, value|
343
+ # Stop if we saw very good and now have normal suggestions
344
+ return if seen && score <= 1000
345
+
346
+ if score > 1000
347
+ # Very good suggestion - set flag to only accept other very good ones
348
+ seen = true
349
+ elsif score < -100
350
+ # Questionable suggestion
351
+ # Stop if we already found good ones, or if we're excluding questionable
352
+ return if found.positive? || onlymaxdiff
353
+
354
+ seen = true
355
+ end
356
+
357
+ # Skip if this word was already suggested
358
+ next if known.any? { |known_word| value.include?(known_word) }
359
+
360
+ found += 1
361
+ yield value
362
+ end
363
+ end
364
+ end
365
+ end
366
+ end
367
+ end
@@ -0,0 +1,283 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Word permutation algorithms for generating spelling variations.
6
+ #
7
+ # Ported from Spylls (Python) permutations.py
8
+ #
9
+ # These functions generate various word edits that are used by the
10
+ # suggestion system to find possible corrections for misspelled words.
11
+ #
12
+ # Method names match Hunspell's suggest.cxx to maintain compatibility.
13
+ module Permutations
14
+ MAX_CHAR_DISTANCE = 4
15
+
16
+ module_function
17
+
18
+ # Uses REP table (typical misspellings) to replace patterns in word.
19
+ #
20
+ # If the pattern's replacement contains "_", it means replacing to " "
21
+ # and yielding two different hypotheses:
22
+ # 1. It was one (dictionary) word "foo bar" (checked as such)
23
+ # 2. It was words ["foo", "bar"] (checked separately)
24
+ #
25
+ # @param word [String] The word to process
26
+ # @param reptable [Array<Hash>] Array of replacement pattern hashes with :regexp and :replacement
27
+ # @yield [String, Array<String>] Each suggestion (string or array of words)
28
+ #
29
+ # @example
30
+ # Kotoshu::Algorithms::Permutations.replchars("acces", [{regexp: /ac/, replacement: "ex"}]) do |sug|
31
+ # puts sug
32
+ # end
33
+ def replchars(word, reptable)
34
+ return if word.length < 2 || reptable.nil? || reptable.empty?
35
+
36
+ reptable.each do |pattern|
37
+ str = word.to_s
38
+ pos = 0
39
+
40
+ while (match_data = pattern[:regexp].match(str, pos))
41
+ suggestion = str[0...match_data.begin(0)] +
42
+ pattern[:replacement].gsub('_', ' ') +
43
+ str[match_data.end(0)..]
44
+
45
+ yield suggestion
46
+ yield suggestion.split(' ') if suggestion.include?(' ')
47
+
48
+ # Move past this match to find next occurrence
49
+ pos = match_data.end(0)
50
+ break if pos >= str.length
51
+ end
52
+ end
53
+ end
54
+
55
+ # Uses MAP table (sets of potentially similar chars) and tries to replace them recursively.
56
+ #
57
+ # Example: Assuming MAP has entry "aáã", and we have misspelling "anarchia":
58
+ # mapchars will produce: "ánarchia", "ánárchia", "ánárchiá", etc.
59
+ #
60
+ # @param word [String] The word to process
61
+ # @param maptable [Array<Set<String>>] Array of character sets for mapping
62
+ # @yield [String] Each variant with mapped characters
63
+ #
64
+ # @example
65
+ # Kotoshu::Algorithms::Permutations.mapchars("anarchia", [Set.new(['a', 'á', 'ã'])]) do |variant|
66
+ # puts variant
67
+ # end
68
+ def mapchars(word, maptable, &)
69
+ return if word.length < 2 || maptable.nil? || maptable.empty?
70
+
71
+ mapchars_internal(word, 0, maptable, &)
72
+ end
73
+
74
+ # Produces permutations with adjacent chars swapped.
75
+ #
76
+ # For short (4 or 5 letters) words also produces double swaps: ahev -> have
77
+ #
78
+ # @param word [String] The word to process
79
+ # @yield [String] Each swap variant
80
+ def swapchar(word)
81
+ return if word.length < 2
82
+
83
+ chars = word.chars
84
+ (0...chars.length - 1).each do |i|
85
+ swapped = chars[0...i] + [chars[i + 1], chars[i]] + chars[(i + 2)..]
86
+ yield swapped.join
87
+ end
88
+
89
+ # Try double swaps for short words
90
+ # ahev -> have, owudl -> would
91
+ if [4, 5].include?(word.length)
92
+ yield word[1] + word[0] + (word.length == 5 ? word[2] : '') + word[-1] + word[-2]
93
+ if word.length == 5
94
+ yield word[0] + word[2] + word[1] + word[-1] + word[-2]
95
+ end
96
+ end
97
+ end
98
+
99
+ # Produces permutations with non-adjacent chars swapped (up to 4 chars distance).
100
+ #
101
+ # @param word [String] The word to process
102
+ # @yield [String] Each long swap variant
103
+ def longswapchar(word)
104
+ chars = word.chars
105
+ (0...chars.length - 2).each do |first|
106
+ ((first + 2)...[first + MAX_CHAR_DISTANCE, chars.length].min).each do |second|
107
+ swapped = chars[0...first] +
108
+ [chars[second]] +
109
+ chars[(first + 1)...second] +
110
+ [chars[first]] +
111
+ chars[(second + 1)..]
112
+ yield swapped.join
113
+ end
114
+ end
115
+ end
116
+
117
+ # Produces permutations with chars replaced by adjacent chars on keyboard layout
118
+ # ("vat -> cat") or downcased (if it was accidental uppercase).
119
+ #
120
+ # @param word [String] The word to process
121
+ # @param layout [String] Keyboard layout string (KEY from aff file)
122
+ # @yield [String] Each variant with replaced chars
123
+ def badcharkey(word, layout)
124
+ chars = word.chars
125
+ chars.each_with_index do |c, i|
126
+ before = word[0...i]
127
+ after = word[(i + 1)..]
128
+
129
+ # Try uppercasing if not already uppercase
130
+ unless c == c.upcase
131
+ yield before + c.upcase + after.to_s
132
+ end
133
+
134
+ next if layout.nil? || layout.empty?
135
+
136
+ # Try adjacent keys on keyboard
137
+ pos = layout.index(c)
138
+ next unless pos
139
+
140
+ while pos
141
+ if pos.positive? && layout[pos - 1] != '|'
142
+ yield before + layout[pos - 1] + after.to_s
143
+ end
144
+ if pos + 1 < layout.length && layout[pos + 1] != '|'
145
+ yield before + layout[pos + 1] + after.to_s
146
+ end
147
+ pos = layout.index(c, pos + 1)
148
+ end
149
+ end
150
+ end
151
+
152
+ # Produces permutations with one char removed in all possible positions.
153
+ #
154
+ # @param word [String] The word to process
155
+ # @yield [String] Each variant with one char removed
156
+ def extrachar(word)
157
+ return if word.length < 2
158
+
159
+ word.length.times do |i|
160
+ yield word[0...i] + word[(i + 1)..]
161
+ end
162
+ end
163
+
164
+ # Produces permutations with one char inserted in all possible positions.
165
+ #
166
+ # List of chars is taken from TRY string -- if absent, tries nothing.
167
+ # Chars are expected to be sorted in order of usage in language.
168
+ #
169
+ # @param word [String] The word to process
170
+ # @param trystring [String] Characters to try inserting (from aff TRY directive)
171
+ # @yield [String] Each variant with one char inserted
172
+ def forgotchar(word, trystring)
173
+ return if trystring.nil? || trystring.empty?
174
+
175
+ trystring.each_char do |c|
176
+ (0..word.length).each do |i|
177
+ yield word[0...i] + c + word[i..]
178
+ end
179
+ end
180
+ end
181
+
182
+ # Produces permutations with one character moved by 2, 3 or 4 places forward or backward
183
+ # (not 1, because adjacent swaps are already handled by swapchar).
184
+ #
185
+ # @param word [String] The word to process
186
+ # @yield [String] Each variant with moved character
187
+ def movechar(word)
188
+ return if word.length < 2
189
+
190
+ chars = word.chars
191
+
192
+ # Move characters forward
193
+ chars.each_with_index do |char, frompos|
194
+ ((frompos + 3)...[chars.length, frompos + MAX_CHAR_DISTANCE + 1].min).each do |topos|
195
+ moved = chars[0...frompos] + chars[(frompos + 1)...topos] + [char] + chars[topos..]
196
+ yield moved.join
197
+ end
198
+ end
199
+
200
+ # Move characters backward
201
+ (chars.length - 1).downto(0) do |frompos|
202
+ [[0, frompos - MAX_CHAR_DISTANCE + 1].max, frompos - 1].min.downto(0) do |topos|
203
+ moved = chars[0...topos] + [chars[frompos]] + chars[topos...frompos] + chars[(frompos + 1)..]
204
+ yield moved.join
205
+ end
206
+ end
207
+ end
208
+
209
+ # Produces permutations with chars replaced by chars in TRY set.
210
+ #
211
+ # @param word [String] The word to process
212
+ # @param trystring [String] Characters to try replacing with (from aff TRY directive)
213
+ # @yield [String] Each variant with replaced char
214
+ def badchar(word, trystring)
215
+ return if trystring.nil? || trystring.empty?
216
+
217
+ trystring.each_char do |c|
218
+ (word.length - 1).downto(0) do |i|
219
+ next if word[i] == c
220
+
221
+ yield word[0...i] + c + word[(i + 1)..]
222
+ end
223
+ end
224
+ end
225
+
226
+ # Produces permutations with accidental two-letter-doubling fixed.
227
+ # Example: "vacacation" -> "vacation"
228
+ #
229
+ # @param word [String] The word to process
230
+ # @yield [String] Each variant with fixed doubling
231
+ def doubletwochars(word)
232
+ return if word.length < 5
233
+
234
+ (2...word.length).each do |i|
235
+ # Check if word[i-2] == word[i] and word[i-3] == word[i-1]
236
+ # Example: vacacation -> "ca" at positions 3-4, so "vac" at 2-4
237
+ if word[i - 2] == word[i] && word[i - 3] == word[i - 1]
238
+ yield word[0...(i - 1)] + word[(i + 1)..]
239
+ end
240
+ end
241
+ end
242
+
243
+ # Produces permutations of splitting word into two in all possible positions.
244
+ #
245
+ # @param word [String] The word to process
246
+ # @yield [Array<String>] Each two-word split
247
+ def twowords(word)
248
+ (1...word.length).each do |i|
249
+ yield [word[0...i], word[i..]]
250
+ end
251
+ end
252
+
253
+ # Internal recursive method for mapchars.
254
+ #
255
+ # @param word [String] Current word state
256
+ # @param start [Integer] Starting position for search
257
+ # @param maptable [Array<Set<String>>] Character mapping table
258
+ # @yield [String] Each variant
259
+ def mapchars_internal(word, start, maptable, &block)
260
+ return if start >= word.length
261
+
262
+ maptable.each do |options|
263
+ options.each do |option|
264
+ pos = word.index(option, start)
265
+ next unless pos
266
+
267
+ options.each do |other|
268
+ next if other == option
269
+
270
+ replaced = word[0...pos] + other + word[(pos + option.length)..]
271
+ yield replaced
272
+
273
+ # Recursively continue from this position
274
+ mapchars_internal(replaced, pos + 1, maptable, &block)
275
+ end
276
+ end
277
+ end
278
+ end
279
+
280
+ private_class_method :mapchars_internal
281
+ end
282
+ end
283
+ end