kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,317 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # Semantic strategy using FastText ONNX embeddings.
7
+ #
8
+ # Provides embedding-based spell correction for:
9
+ # - Typos: Re-ranks edit-distance candidates by semantic similarity
10
+ # - Real-word errors: Detects when valid words are used incorrectly in context
11
+ #
12
+ # This strategy works alongside other strategies (EditDistance, Phonetic, etc.)
13
+ # to provide comprehensive spell checking with semantic awareness.
14
+ #
15
+ # @example Basic usage
16
+ # strategy = SemanticStrategy.new(language_code: 'en')
17
+ # suggestions = strategy.generate(context)
18
+ #
19
+ # @example With preloaded embeddings (faster)
20
+ # strategy = SemanticStrategy.new(
21
+ # language_code: 'en',
22
+ # preload_embeddings: true
23
+ # )
24
+ # suggestions = strategy.generate(context)
25
+ class SemanticStrategy < BaseStrategy
26
+ # @return [String] Language code (ISO 639-1)
27
+ attr_reader :language_code
28
+
29
+ # @return [Embeddings::Vocabulary] The vocabulary
30
+ attr_reader :vocabulary
31
+
32
+ # @return [Embeddings::OnnxRuntimeModel] The ONNX model
33
+ attr_reader :model
34
+
35
+ # @return [Embeddings::SimilaritySearch] The similarity search
36
+ attr_reader :search
37
+
38
+ # Create a new semantic strategy.
39
+ #
40
+ # @param language_code [String] ISO 639-1 language code
41
+ # @param cache [Cache::ModelCache, nil] Optional cache instance
42
+ # @param preload_embeddings [Boolean] Whether to preload embeddings
43
+ # @param max_context_window [Integer] Words to consider for context
44
+ # @param min_semantic_similarity [Float] Minimum similarity for semantic suggestions
45
+ # @param semantic_boost_weight [Float] Weight for semantic similarity in re-ranking
46
+ # @param config [Hash] Additional configuration
47
+ def initialize(language_code:, cache: nil, preload_embeddings: false,
48
+ max_context_window: 5, min_semantic_similarity: 0.5,
49
+ semantic_boost_weight: 0.3, **config)
50
+ super(
51
+ name: :semantic,
52
+ min_semantic_similarity: min_semantic_similarity,
53
+ semantic_boost_weight: semantic_boost_weight,
54
+ max_context_window: max_context_window,
55
+ **config
56
+ )
57
+ @language_code = language_code
58
+ @max_context_window = max_context_window
59
+ @min_semantic_similarity = min_semantic_similarity
60
+ @semantic_boost_weight = semantic_boost_weight
61
+
62
+ # Initialize embedding components
63
+ initialize_embeddings(cache, preload_embeddings)
64
+ end
65
+
66
+ # Generate suggestions using semantic similarity.
67
+ #
68
+ # Handles two cases:
69
+ # 1. Word not in vocabulary (typo): Re-ranks edit-distance candidates
70
+ # 2. Word in vocabulary (real-word error): Finds semantically similar alternatives
71
+ #
72
+ # @param context [Context] The suggestion context
73
+ # @return [SuggestionSet] Generated suggestions
74
+ def generate(context)
75
+ word = context.word
76
+
77
+ # Ensure embeddings are loaded
78
+ return SuggestionSet.empty unless @search
79
+
80
+ # Case 1: Word not in vocabulary (typo)
81
+ unless @vocabulary.include?(word)
82
+ return generate_for_typo(context)
83
+ end
84
+
85
+ # Case 2: Real-word error detection
86
+ # Find semantically similar words that might be correct in context
87
+ generate_for_real_word_error(context)
88
+ end
89
+
90
+ # Check if this strategy should handle the context.
91
+ #
92
+ # Semantic strategy handles:
93
+ # - Words not in vocabulary (for typo re-ranking)
94
+ # - Words in vocabulary (for real-word error detection)
95
+ #
96
+ # @param _context [Context] The suggestion context
97
+ # @return [Boolean] True if the strategy should handle this context
98
+ def handles?(_context)
99
+ return false unless enabled?
100
+ return false unless @search && @vocabulary
101
+
102
+ # Handle all words - we filter in generate()
103
+ true
104
+ end
105
+
106
+ # The confidence cascade may skip this strategy: the ONNX
107
+ # rerank runs only when the traditional strategies are not
108
+ # already confident (see {SemanticCascade}).
109
+ #
110
+ # @return [Boolean]
111
+ def skip_when_confident?
112
+ true
113
+ end
114
+
115
+ # Compute semantic similarity between two words.
116
+ #
117
+ # @param word1 [String] First word
118
+ # @param word2 [String] Second word
119
+ # @return [Float, nil] Cosine similarity or nil if either word not found
120
+ def semantic_similarity(word1, word2)
121
+ return nil unless @search
122
+
123
+ @search.similarity(word1, word2)
124
+ end
125
+
126
+ # Find semantically similar words.
127
+ #
128
+ # @param word [String] The query word
129
+ # @param k [Integer] Number of neighbors
130
+ # @return [Array<Hash>] Array of {word, similarity} hashes
131
+ def find_similar_words(word, k: 10)
132
+ return [] unless @search
133
+
134
+ @search.find_nearest(word, k: k, exclude_self: false)
135
+ end
136
+
137
+ # String representation.
138
+ #
139
+ # @return [String] String representation
140
+ def to_s
141
+ "SemanticStrategy(language: #{@language_code}, vocab_size: #{@vocabulary&.size || 0}, loaded: #{@search && true})"
142
+ end
143
+ alias inspect to_s
144
+
145
+ private
146
+
147
+ # Initialize embedding components.
148
+ #
149
+ # @param cache [Cache::ModelCache, nil] Cache instance
150
+ # @param preload [Boolean] Whether to preload embeddings
151
+ def initialize_embeddings(cache, preload)
152
+ # Try to load from cache
153
+ @search = Embeddings::SimilaritySearch.from_cache(
154
+ @language_code,
155
+ cache: cache,
156
+ preload: preload,
157
+ )
158
+
159
+ # Extract vocabulary and model from search
160
+ if @search
161
+ @vocabulary = @search.vocabulary
162
+ @model = @search.model
163
+ else
164
+ @vocabulary = nil
165
+ @model = nil
166
+
167
+ warn "Warning: Could not load ONNX model for language '#{@language_code}'. Semantic strategy will be disabled." if $VERBOSE
168
+ end
169
+ end
170
+
171
+ # Generate suggestions for a typo (word not in vocabulary).
172
+ #
173
+ # Uses semantic similarity to re-rank candidates from other strategies.
174
+ #
175
+ # @param context [Context] The suggestion context
176
+ # @return [SuggestionSet] Re-ranked suggestions
177
+ def generate_for_typo(context)
178
+ word = context.word
179
+ max_results = context.max_results || max_results
180
+
181
+ # For typos, we find semantically similar words in vocabulary
182
+ # that are also close in spelling (handled by edit distance strategy)
183
+ neighbors = @search.find_nearest(
184
+ word,
185
+ k: max_results * 2, # Get more candidates for filtering
186
+ exclude_self: true,
187
+ min_similarity: @min_semantic_similarity,
188
+ )
189
+
190
+ return SuggestionSet.empty if neighbors.empty?
191
+
192
+ # Convert to suggestions
193
+ # Confidence is based on semantic similarity
194
+ suggestions = neighbors.map do |neighbor|
195
+ similarity = neighbor[:similarity]
196
+ confidence = normalize_similarity(similarity)
197
+
198
+ # Calculate "distance" as inverse of similarity
199
+ # High similarity = low distance
200
+ distance = similarity_to_distance(similarity)
201
+
202
+ create_suggestion(
203
+ neighbor[:word],
204
+ distance: distance,
205
+ confidence: confidence,
206
+ semantic_similarity: similarity,
207
+ )
208
+ end
209
+
210
+ # Sort and limit
211
+ SuggestionSet.new(suggestions, max_size: max_results)
212
+ end
213
+
214
+ # Generate suggestions for a real-word error.
215
+ #
216
+ # Finds semantically similar words that might be correct in context.
217
+ #
218
+ # @param context [Context] The suggestion context
219
+ # @return [SuggestionSet] Alternative suggestions
220
+ def generate_for_real_word_error(context)
221
+ word = context.word
222
+ max_results = context.max_results || max_results
223
+
224
+ # Get context words from the surrounding text
225
+ context_words = get_context_words(context)
226
+
227
+ # Find semantically similar words
228
+ neighbors = @search.find_nearest(
229
+ word,
230
+ k: max_results * 3,
231
+ exclude_self: true,
232
+ min_similarity: @min_semantic_similarity,
233
+ )
234
+
235
+ return SuggestionSet.empty if neighbors.empty?
236
+
237
+ # Re-rank by context similarity
238
+ suggestions = neighbors.map do |neighbor|
239
+ candidate_word = neighbor[:word]
240
+ similarity = neighbor[:similarity]
241
+
242
+ # Check if candidate makes more sense in context
243
+ context_score = compute_context_fit(candidate_word, context_words)
244
+
245
+ # Combine semantic similarity with context fit
246
+ combined_score = (similarity * 0.7) + (context_score * 0.3)
247
+
248
+ confidence = normalize_similarity(combined_score)
249
+ distance = similarity_to_distance(combined_score)
250
+
251
+ create_suggestion(
252
+ candidate_word,
253
+ distance: distance,
254
+ confidence: confidence,
255
+ semantic_similarity: similarity,
256
+ context_score: context_score,
257
+ )
258
+ end
259
+
260
+ # Sort by combined score and limit
261
+ SuggestionSet.new(suggestions.sort_by do |s|
262
+ -s.metadata[:context_score]
263
+ end, max_size: max_results)
264
+ end
265
+
266
+ # Get context words for semantic analysis.
267
+ #
268
+ # @param _context [Context] The suggestion context (unused)
269
+ # @return [Array<String>] Context words
270
+ def get_context_words(_context)
271
+ # For now, return empty - context analysis would need full text
272
+ # This could be extended in the future
273
+ []
274
+ end
275
+
276
+ # Compute how well a word fits in context.
277
+ #
278
+ # @param candidate [String] Candidate word
279
+ # @param context_words [Array<String>] Context words
280
+ # @return [Float] Context fit score (0.0 to 1.0)
281
+ def compute_context_fit(candidate, context_words)
282
+ return 0.5 if context_words.empty?
283
+
284
+ # Compute average similarity between candidate and context words
285
+ similarities = context_words.filter_map do |ctx_word|
286
+ @search.similarity(candidate, ctx_word)
287
+ end
288
+
289
+ return 0.5 if similarities.empty?
290
+
291
+ similarities.sum / similarities.size
292
+ end
293
+
294
+ # Normalize similarity to confidence (0.0 to 1.0).
295
+ #
296
+ # @param similarity [Float] Cosine similarity (-1.0 to 1.0)
297
+ # @return [Float] Normalized confidence (0.0 to 1.0)
298
+ def normalize_similarity(similarity)
299
+ # Map from [-1, 1] to [0, 1]
300
+ ((similarity + 1) / 2.0).clamp(0.0, 1.0)
301
+ end
302
+
303
+ # Convert similarity to "distance" for ranking.
304
+ #
305
+ # @param similarity [Float] Cosine similarity (-1.0 to 1.0)
306
+ # @return [Integer] Pseudo-distance (lower = better)
307
+ def similarity_to_distance(similarity)
308
+ # Map similarity to distance: higher similarity = lower distance
309
+ # Similarity 1.0 -> distance 0
310
+ # Similarity 0.0 -> distance 2
311
+ # Similarity -1.0 -> distance 4
312
+ ((1.0 - similarity) * 2).to_i.clamp(0, 5)
313
+ end
314
+ end
315
+ end
316
+ end
317
+ end
@@ -0,0 +1,270 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # SymSpell suggestion strategy.
7
+ #
8
+ # Uses deletion distance algorithm for fast approximate string matching.
9
+ # Pre-computes deletion variants for all dictionary words, enabling O(1)
10
+ # lookup for common misspellings.
11
+ #
12
+ # This is 10-100x faster than EditDistanceStrategy for large dictionaries.
13
+ #
14
+ # The algorithm works by:
15
+ # 1. Pre-computing single deletion variants for each dictionary word
16
+ # 2. Looking up input word's deletion variants in the pre-computed map
17
+ # 3. Distance is inferred from the deletion level
18
+ #
19
+ # @see https://github.com/wolfgarbe/SymSpell Original SymSpell paper
20
+ class SymSpellStrategy < BaseStrategy
21
+ # Maximum deletion distance to consider
22
+ DEFAULT_MAX_DELETION_DISTANCE = 2
23
+ # Maximum dictionary words to process (increased for better coverage)
24
+ DEFAULT_MAX_DICTIONARY_SIZE = 500_000
25
+ # Enable transposition handling (slower pre-computation, better accuracy)
26
+ DEFAULT_HANDLE_TRANSPOSITIONS = true
27
+
28
+ # Create a new SymSpell strategy.
29
+ #
30
+ # @param dictionary [Object] Dictionary to use for suggestions
31
+ # @param name [String, Symbol] Strategy name
32
+ # @param config [Hash] Configuration options
33
+ # @option config [Integer] max_deletion_distance Maximum deletion distance (default: 2)
34
+ # @option config [Integer] max_results Maximum results to return (default: 10)
35
+ # @option config [Integer] max_dictionary_size Maximum words to process (default: 500_000)
36
+ # @option config [Boolean] handle_transpositions Generate transposition variants (default: true)
37
+ def initialize(dictionary:, name: :symspell, **config)
38
+ super(name: name, **config)
39
+ @dictionary = dictionary
40
+ @max_deletion_distance = config.fetch(:max_deletion_distance, DEFAULT_MAX_DELETION_DISTANCE)
41
+ @max_dictionary_size = config.fetch(:max_dictionary_size, DEFAULT_MAX_DICTIONARY_SIZE)
42
+ @handle_transpositions = config.fetch(:handle_transpositions, DEFAULT_HANDLE_TRANSPOSITIONS)
43
+ @deletes = Hash.new { |h, k| h[k] = [] } # deletion_variant -> [original_words]
44
+ @words = Set.new
45
+ precompute!
46
+ end
47
+
48
+ # Generate suggestions using deletion distance.
49
+ #
50
+ # @param context [Context] The suggestion context
51
+ # @return [SuggestionSet] Generated suggestions
52
+ def generate(context)
53
+ word = context.word
54
+ max_dist = get_config(:max_deletion_distance, @max_deletion_distance)
55
+
56
+ # Normalize to lowercase for case-insensitive matching
57
+ word_lower = word.downcase
58
+
59
+ # Check if word is in dictionary
60
+ return SuggestionSet.empty if @words.include?(word_lower)
61
+
62
+ # Collect candidates with their distances
63
+ candidates = {}
64
+ checked = Set.new([word_lower])
65
+
66
+ # First, check if the input word is a deletion variant of any dictionary word
67
+ @deletes[word_lower].each do |dict_word|
68
+ candidates[dict_word] ||= 1
69
+ end
70
+
71
+ # If transpositions are enabled, check them too
72
+ if @handle_transpositions
73
+ generate_transpositions(word_lower).each do |transposed|
74
+ @deletes[transposed].each do |dict_word|
75
+ candidates[dict_word] ||= 1
76
+ end
77
+ end
78
+ end
79
+
80
+ # Generate deletion variants and check for matches
81
+ max_dist.times do |dist|
82
+ generate_deletions_from_set(checked).each do |variant|
83
+ next if checked.include?(variant)
84
+
85
+ checked.add(variant)
86
+
87
+ # Check if variant is directly in dictionary
88
+ candidates[variant] = dist + 1 if @words.include?(variant)
89
+
90
+ # Check if variant maps to dictionary words
91
+ @deletes[variant].each do |dict_word|
92
+ # Distance = deletions from input + deletions from dict_word
93
+ # Both reach the same variant
94
+ candidates[dict_word] ||= dist + 2
95
+ end
96
+ end
97
+ end
98
+
99
+ # Sort by distance and create suggestions
100
+ sorted_words = candidates.sort_by { |_, dist| dist }.map(&:first)
101
+ create_suggestion_set(sorted_words, distances: candidates, original_word: context.word)
102
+ end
103
+
104
+ # Pre-compute deletion variants for all dictionary words.
105
+ #
106
+ # This is called during initialization and builds the index.
107
+ def precompute!
108
+ words = dictionary_words(@dictionary)
109
+
110
+ words.first(@max_dictionary_size).each do |word|
111
+ next if word.nil? || word.empty?
112
+
113
+ word_lower = word.downcase
114
+ @words.add(word_lower)
115
+
116
+ # Generate only single deletion variants for efficiency
117
+ # Multiple deletions are handled during lookup
118
+ generate_single_deletions(word_lower).each do |variant|
119
+ @deletes[variant] << word_lower
120
+ end
121
+
122
+ # Generate transposition variants if enabled
123
+ if @handle_transpositions
124
+ generate_transpositions(word_lower).each do |variant|
125
+ @deletes[variant] << word_lower
126
+ end
127
+ end
128
+ end
129
+ end
130
+
131
+ # Generate all adjacent transposition variants of a word.
132
+ #
133
+ # For example, "world" → ["owrld", "wrold", "wolrd", "wordl"]
134
+ #
135
+ # @param word [String] The word
136
+ # @return [Array<String>] Array of variants with adjacent characters swapped
137
+ def generate_transpositions(word)
138
+ variants = []
139
+ word.chars.each_with_index do |_, i|
140
+ next if i == word.length - 1 # Can't swap last character
141
+
142
+ variant = word.dup
143
+ variant[i], variant[i + 1] = variant[i + 1], variant[i]
144
+ variants << variant unless variant == word
145
+ end
146
+ variants
147
+ end
148
+
149
+ # Calculate deletion distance between two words.
150
+ #
151
+ # For SymSpell, this is the length of their longest common subsequence
152
+ # based distance (minimum deletions to make them equal).
153
+ #
154
+ # @param str1 [String] First word
155
+ # @param str2 [String] Second word
156
+ # @return [Integer] Deletion distance
157
+ def deletion_distance(str1, str2)
158
+ return str2.length if str1.empty?
159
+ return str1.length if str2.empty?
160
+ return 0 if str1 == str2
161
+
162
+ # Simple approach: find if one can be transformed to the other
163
+ # by only deletions (check if str1 is subsequence of str2 or vice versa)
164
+ if is_subsequence?(str1, str2)
165
+ str2.length - str1.length
166
+ elsif is_subsequence?(str2, str1)
167
+ str1.length - str2.length
168
+ else
169
+ # Fallback to edit distance approximation
170
+ # This shouldn't happen often with proper SymSpell usage
171
+ lcs_len = longest_common_subsequence_length(str1, str2)
172
+ str1.length + str2.length - (2 * lcs_len)
173
+ end
174
+ end
175
+
176
+ private
177
+
178
+ # Generate all single-deletion variants of a word.
179
+ #
180
+ # @param word [String] The word
181
+ # @return [Array<String>] Array of variants with one character deleted
182
+ def generate_single_deletions(word)
183
+ variants = []
184
+ word.chars.each_with_index do |_, i|
185
+ variant = word[0...i] + word[(i + 1)..].to_s
186
+ variants << variant unless variant.empty? || variant == word
187
+ end
188
+ variants
189
+ end
190
+
191
+ # Generate deletion variants from a set of words.
192
+ #
193
+ # @param words_set [Set<String>] Set of words to process
194
+ # @return [Set<String>] New set with all single deletions
195
+ def generate_deletions_from_set(words_set)
196
+ result = Set.new
197
+ words_set.each do |word|
198
+ generate_single_deletions(word).each do |variant|
199
+ result.add(variant)
200
+ end
201
+ end
202
+ result
203
+ end
204
+
205
+ # Check if str1 is a subsequence of str2.
206
+ #
207
+ # @param str1 [String] Potential subsequence
208
+ # @param str2 [String] String to check against
209
+ # @return [Boolean] True if str1 is subsequence of str2
210
+ def is_subsequence?(str1, str2)
211
+ return true if str1.empty?
212
+ return false if str1.length > str2.length
213
+
214
+ i = 0
215
+ str2.each_char do |c|
216
+ i += 1 if c == str1[i]
217
+ return true if i == str1.length
218
+ end
219
+ i == str1.length
220
+ end
221
+
222
+ # Calculate the length of the longest common subsequence.
223
+ #
224
+ # Uses dynamic programming for efficiency.
225
+ #
226
+ # @param str1 [String] First string
227
+ # @param str2 [String] Second string
228
+ # @return [Integer] LCS length
229
+ def longest_common_subsequence_length(str1, str2)
230
+ return 0 if str1.empty? || str2.empty?
231
+
232
+ # Use shorter string for inner loop
233
+ str1, str2 = str2, str1 if str1.length > str2.length
234
+
235
+ # Previous row of DP table
236
+ previous = Array.new(str1.length + 1, 0)
237
+
238
+ str2.each_char do |char2|
239
+ current = [0] # First column is always 0
240
+
241
+ str1.each_char.with_index do |char1, i|
242
+ current << if char1 == char2
243
+ previous[i] + 1
244
+ else
245
+ [current[i], previous[i + 1]].max
246
+ end
247
+ end
248
+
249
+ previous = current
250
+ end
251
+
252
+ previous.last
253
+ end
254
+
255
+ # Get all words from the dictionary.
256
+ #
257
+ # @param dictionary [Object] Dictionary object
258
+ # @return [Array<String>] All words
259
+ def dictionary_words(dictionary)
260
+ case dictionary
261
+ when Kotoshu::Dictionary::Base then dictionary.words # all_words is an alias
262
+ when Array then dictionary.dup
263
+ when Hash then dictionary.keys
264
+ else []
265
+ end
266
+ end
267
+ end
268
+ end
269
+ end
270
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set" if RUBY_VERSION < "3.0"
4
+
5
+ module Kotoshu
6
+ module Suggestions
7
+ module Strategies
8
+ # Exact and case-insensitive word lookup over a dictionary word
9
+ # list, built once per suggestion sweep.
10
+ #
11
+ # {KeyboardProximityStrategy} matches one keyboard variant at a
12
+ # time, and its two-round variant set reaches tens of thousands
13
+ # of lookups per word — each of which used to scan the whole
14
+ # dictionary (with a +downcase+ per candidate on the
15
+ # case-insensitive branch). The index answers the same lookups in
16
+ # constant time.
17
+ #
18
+ # Lookup semantics are KeyboardProximityStrategy's historical
19
+ # +find_word+ exactly:
20
+ #
21
+ # - the exact-match branch answers "is the lowercased input a
22
+ # word of the list as-is?" and returns the INPUT string when it
23
+ # is (the variant as typed, not the dictionary entry);
24
+ # - otherwise the first word in list order whose lowercase form
25
+ # equals the lowercased input wins (first-in-list-order on
26
+ # collisions);
27
+ # - nil/empty input finds nothing.
28
+ #
29
+ # This mirrors WordIndex in kotoshu-rs (kotoshu/src/suggest/mod.rs).
30
+ class WordIndex
31
+ # @return [Set<String>] The words exactly as listed
32
+ attr_reader :exact
33
+
34
+ # @return [Hash<String, String>] Lowercase form => first listed word
35
+ attr_reader :lowered
36
+
37
+ # Build an index over a word list.
38
+ #
39
+ # @param words [Array<String>] Dictionary words in list order
40
+ # @return [WordIndex] The built index
41
+ def self.build(words)
42
+ exact = Set.new
43
+ lowered = {}
44
+ words.each do |word|
45
+ exact.add(word)
46
+ key = word.downcase
47
+ lowered[key] = word unless lowered.key?(key)
48
+ end
49
+ new(exact, lowered)
50
+ end
51
+
52
+ # Create a word index from prebuilt structures.
53
+ #
54
+ # @param exact [Set<String>] Words exactly as listed
55
+ # @param lowered [Hash<String, String>] Lowercase form => first listed word
56
+ def initialize(exact, lowered)
57
+ @exact = exact
58
+ @lowered = lowered
59
+ end
60
+
61
+ # Find a word in the indexed list.
62
+ #
63
+ # @param word [String, nil] The word to find
64
+ # @return [String, nil] The input string on an exact (lowercase)
65
+ # match, else the first listed word with the same lowercase
66
+ # form, else nil
67
+ def find(word)
68
+ return nil if word.nil? || word.empty?
69
+
70
+ word_lower = word.downcase
71
+ return word if @exact.include?(word_lower)
72
+
73
+ @lowered[word_lower]
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end