kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,181 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Abstract base class for word embedding models.
6
+ #
7
+ # Provides a unified interface for loading and querying word embeddings
8
+ # from different sources (FastText, Word2Vec, GloVe, ONNX, etc.).
9
+ #
10
+ # @example Using an embedding model
11
+ # model = FastTextModel.new('cc.en.300.vec')
12
+ # embedding = model.embedding_for('hello')
13
+ # similarity = model.similarity('hello', 'world')
14
+ # neighbors = model.nearest_neighbors('hello', k: 10)
15
+ #
16
+ # @abstract Subclasses must implement {#embedding_for} and {#vocabulary}
17
+ class EmbeddingModel
18
+ attr_reader :language_code, :dimension, :vocabulary_size
19
+
20
+ # Create a new embedding model.
21
+ #
22
+ # @param language_code [String] ISO 639-1 language code
23
+ # @param dimension [Integer] Vector dimensionality (e.g., 300)
24
+ def initialize(language_code:, dimension:)
25
+ raise ArgumentError, "Language code cannot be nil" if language_code.nil?
26
+ raise ArgumentError, "Dimension must be positive" unless dimension&.positive?
27
+
28
+ @language_code = language_code
29
+ @dimension = dimension
30
+ @vocabulary_size = 0
31
+ end
32
+
33
+ # Get embedding vector for a word.
34
+ #
35
+ # @param word [String] The word to lookup
36
+ # @return [WordEmbedding, nil] Embedding vector or nil if not found
37
+ # @abstract Subclass must implement
38
+ def embedding_for(word)
39
+ raise NotImplementedError, "#{self.class} must implement #embedding_for"
40
+ end
41
+
42
+ # Check if a word is in the vocabulary.
43
+ #
44
+ # @param word [String] The word to check
45
+ # @return [Boolean] True if word exists in vocabulary
46
+ def has_word?(word)
47
+ vocabulary.include?(word)
48
+ end
49
+
50
+ # Calculate cosine similarity between two words.
51
+ #
52
+ # @param word1 [String] First word
53
+ # @param word2 [String] Second word
54
+ # @return [Float, nil] Similarity score (0.0 to 1.0) or nil if words not found
55
+ def similarity(word1, word2)
56
+ emb1 = embedding_for(word1)
57
+ emb2 = embedding_for(word2)
58
+
59
+ return nil unless emb1 && emb2
60
+
61
+ emb1.similarity(emb2)
62
+ end
63
+
64
+ # Calculate Euclidean distance between two words.
65
+ #
66
+ # @param word1 [String] First word
67
+ # @param word2 [String] Second word
68
+ # @return [Float, nil] Distance or nil if words not found
69
+ def distance(word1, word2)
70
+ emb1 = embedding_for(word1)
71
+ emb2 = embedding_for(word2)
72
+
73
+ return nil unless emb1 && emb2
74
+
75
+ emb1.distance(emb2)
76
+ end
77
+
78
+ # Find k nearest neighbors for a word.
79
+ #
80
+ # @param word [String] The query word
81
+ # @param k [Integer] Number of neighbors to return
82
+ # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
83
+ def nearest_neighbors(word, k: 10)
84
+ embedding = embedding_for(word)
85
+ return [] unless embedding
86
+
87
+ # Calculate similarity with all words in vocabulary
88
+ neighbors = vocabulary.map do |vocab_word|
89
+ next if vocab_word == word
90
+
91
+ vocab_embedding = embedding_for(vocab_word)
92
+ next unless vocab_embedding
93
+
94
+ sim = embedding.similarity(vocab_embedding)
95
+ NearestNeighbor.new(
96
+ word: vocab_word,
97
+ similarity: sim,
98
+ distance: embedding.distance(vocab_embedding),
99
+ embedding: vocab_embedding
100
+ )
101
+ end.compact
102
+
103
+ # NearestNeighbor#<=> already sorts descending by similarity.
104
+ neighbors.sort.first(k)
105
+ end
106
+
107
+ # Find k nearest neighbors for an embedding vector.
108
+ #
109
+ # @param embedding [WordEmbedding] The query embedding
110
+ # @param k [Integer] Number of neighbors to return
111
+ # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
112
+ def nearest_neighbors_for_embedding(embedding, k: 10)
113
+ return [] unless embedding
114
+
115
+ # Calculate similarity with all words in vocabulary
116
+ neighbors = vocabulary.map do |vocab_word|
117
+ vocab_embedding = embedding_for(vocab_word)
118
+ next unless vocab_embedding
119
+
120
+ sim = embedding.similarity(vocab_embedding)
121
+ NearestNeighbor.new(
122
+ word: vocab_word,
123
+ similarity: sim,
124
+ distance: embedding.distance(vocab_embedding),
125
+ embedding: vocab_embedding
126
+ )
127
+ end.compact
128
+
129
+ # NearestNeighbor#<=> already sorts descending by similarity.
130
+ neighbors.sort.first(k)
131
+ end
132
+
133
+ # Get model metadata.
134
+ #
135
+ # @return [Hash] Model metadata
136
+ def metadata
137
+ {
138
+ language_code: @language_code,
139
+ dimension: @dimension,
140
+ vocabulary_size: @vocabulary_size,
141
+ model_type: self.class.name
142
+ }
143
+ end
144
+
145
+ # Get the vocabulary (all words in the model).
146
+ #
147
+ # @return [Array<String>] Vocabulary words
148
+ # @abstract Subclass must implement
149
+ def vocabulary
150
+ raise NotImplementedError, "#{self.class} must implement #vocabulary"
151
+ end
152
+
153
+ # Check if model is loaded.
154
+ #
155
+ # @return [Boolean] True if model is loaded and ready
156
+ def loaded?
157
+ @vocabulary_size&.positive? || vocabulary&.any?
158
+ end
159
+
160
+ # Get model statistics.
161
+ #
162
+ # @return [Hash] Statistics about the model
163
+ def statistics
164
+ {
165
+ language: @language_code,
166
+ dimension: @dimension,
167
+ vocabulary_size: @vocabulary_size,
168
+ loaded: loaded?
169
+ }
170
+ end
171
+
172
+ # String representation.
173
+ #
174
+ # @return [String] Human-readable representation
175
+ def to_s
176
+ "#{self.class.name}(language: #{@language_code}, dim: #{@dimension}, vocab: #{@vocabulary_size})"
177
+ end
178
+ alias_method :inspect, :to_s
179
+ end
180
+ end
181
+ end
@@ -0,0 +1,212 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # FastText embedding model implementation.
6
+ #
7
+ # Loads FastText pre-trained word vectors from .vec files.
8
+ # Supports Common Crawl and Wikipedia trained vectors.
9
+ #
10
+ # @example Loading from file
11
+ # model = FastTextModel.from_file('cc.en.300.vec')
12
+ # model.embedding_for('hello')
13
+ #
14
+ # @example Loading from GitHub
15
+ # model = FastTextModel.from_github('en')
16
+ # model.nearest_neighbors('hello', k: 10)
17
+ #
18
+ # @see https://fasttext.cc/docs/en/crawl-vectors.html FastText crawl vectors
19
+ # @see https://fasttext.cc/docs/en/english-vectors.html FastText English vectors
20
+ class FastTextModel < EmbeddingModel
21
+ # Standard FastText dimension for crawl vectors
22
+ DEFAULT_DIMENSION = 300
23
+
24
+ # Number of vectors to load when reading from file
25
+ # FastText .vec files contain up to 2M words; we load a subset by default
26
+ DEFAULT_MAX_VECTORS = 1_000_000
27
+
28
+ attr_reader :embeddings, :max_vectors
29
+
30
+ # Create a new FastText model.
31
+ #
32
+ # @param language_code [String] ISO 639-1 language code
33
+ # @param dimension [Integer] Vector dimension (default: 300)
34
+ # @param embeddings [Hash<String, WordEmbedding>] Pre-loaded embeddings
35
+ # @param max_vectors [Integer] Maximum vectors to load from file
36
+ def initialize(language_code:, dimension: DEFAULT_DIMENSION, embeddings: {}, max_vectors: DEFAULT_MAX_VECTORS)
37
+ super(language_code: language_code, dimension: dimension)
38
+ @embeddings = embeddings.freeze
39
+ @max_vectors = max_vectors
40
+ @vocabulary_size = @embeddings.size
41
+ end
42
+
43
+ # Load FastText model from a .vec file.
44
+ #
45
+ # @param file_path [String] Path to FastText .vec file
46
+ # @param max_vectors [Integer] Maximum vectors to load (default: 1M)
47
+ # @param language_code [String] Language code (auto-detected from filename)
48
+ # @return [FastTextModel] Loaded model
49
+ # @raise [ArgumentError] if file doesn't exist
50
+ def self.from_file(file_path, max_vectors: DEFAULT_MAX_VECTORS, language_code: nil)
51
+ raise ArgumentError, "File not found: #{file_path}" unless File.exist?(file_path)
52
+
53
+ # Detect language from filename if not provided
54
+ language_code ||= detect_language_from_path(file_path)
55
+
56
+ # Parse the .vec file
57
+ embeddings = {}
58
+ dimension = nil
59
+ count = 0
60
+
61
+ File.open(file_path, 'r', encoding: 'UTF-8') do |file|
62
+ # First line: vocab_size dimension
63
+ first_line = file.getline
64
+ metadata = first_line.split
65
+ _vocab_size = metadata[0].to_i
66
+ dimension = metadata[1].to_i
67
+
68
+ # Read vectors
69
+ file.each_line do |line|
70
+ break if count >= max_vectors
71
+
72
+ parts = line.split
73
+ word = parts[0]
74
+ vector = parts[1..-1].map(&:to_f)
75
+
76
+ next unless vector.size == dimension
77
+
78
+ embeddings[word] = WordEmbedding.new(word, vector, language_code, dimension: dimension)
79
+ count += 1
80
+ end
81
+ end
82
+
83
+ new(language_code: language_code, dimension: dimension, embeddings: embeddings, max_vectors: max_vectors)
84
+ end
85
+
86
+ # Load FastText model from GitHub (via ModelCache).
87
+ #
88
+ # Downloads the .vec file from kotoshu/dictionaries repository.
89
+ #
90
+ # @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
91
+ # @param max_vectors [Integer] Maximum vectors to load (default: 500K for GitHub)
92
+ # @param cache [ModelCache, nil] Optional cache instance
93
+ # @return [FastTextModel] Loaded model
94
+ # @raise [ArgumentError] if language not supported
95
+ def self.from_github(language_code, max_vectors: 500_000, cache: nil)
96
+ cache ||= Cache::ModelCache.new
97
+
98
+ # Get the .vec file path from cache
99
+ vec_file = cache.get_fasttext_model(language_code)
100
+
101
+ from_file(vec_file, max_vectors: max_vectors, language_code: language_code)
102
+ end
103
+
104
+ # Get embedding vector for a word.
105
+ #
106
+ # @param word [String] The word to lookup
107
+ # @return [WordEmbedding, nil] Embedding vector or nil if not found
108
+ def embedding_for(word)
109
+ return nil if word.nil? || word.empty?
110
+
111
+ # Direct lookup
112
+ @embeddings[word]
113
+ end
114
+
115
+ # Get the vocabulary (all words in the model).
116
+ #
117
+ # @return [Array<String>] Vocabulary words
118
+ def vocabulary
119
+ @embeddings.keys
120
+ end
121
+
122
+ # Check if model is loaded.
123
+ #
124
+ # @return [Boolean] True if embeddings are loaded
125
+ def loaded?
126
+ @embeddings&.any?
127
+ end
128
+
129
+ # Find k nearest neighbors for a word (optimized version).
130
+ #
131
+ # Overrides the base implementation for better performance using
132
+ # pre-loaded embeddings instead of repeated lookups.
133
+ #
134
+ # @param word [String] The query word
135
+ # @param k [Integer] Number of neighbors to return
136
+ # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
137
+ def nearest_neighbors(word, k: 10)
138
+ embedding = embedding_for(word)
139
+ return [] unless embedding
140
+
141
+ # Calculate similarity with all words in vocabulary
142
+ neighbors = @embeddings.map do |vocab_word, vocab_embedding|
143
+ next if vocab_word == word
144
+
145
+ sim = embedding.similarity(vocab_embedding)
146
+ NearestNeighbor.new(
147
+ word: vocab_word,
148
+ similarity: sim,
149
+ embedding: vocab_embedding
150
+ )
151
+ end.compact
152
+
153
+ # Sort by similarity (descending) and take top k
154
+ neighbors.sort.reverse.first(k)
155
+ end
156
+
157
+ # Find k nearest neighbors for an embedding vector (optimized version).
158
+ #
159
+ # @param embedding [WordEmbedding] The query embedding
160
+ # @param k [Integer] Number of neighbors to return
161
+ # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
162
+ def nearest_neighbors_for_embedding(embedding, k: 10)
163
+ return [] unless embedding
164
+
165
+ # Calculate similarity with all words in vocabulary
166
+ neighbors = @embeddings.map do |vocab_word, vocab_embedding|
167
+ sim = embedding.similarity(vocab_embedding)
168
+ NearestNeighbor.new(
169
+ word: vocab_word,
170
+ similarity: sim,
171
+ embedding: vocab_embedding
172
+ )
173
+ end.compact
174
+
175
+ # Sort by similarity (descending) and take top k
176
+ neighbors.sort.reverse.first(k)
177
+ end
178
+
179
+ # Get batch embeddings for multiple words.
180
+ #
181
+ # @param words [Array<String>] Words to lookup
182
+ # @return [Hash<String, WordEmbedding>] Mapping of word to embedding
183
+ def batch_embeddings(words)
184
+ words.each_with_object({}) do |word, hash|
185
+ emb = embedding_for(word)
186
+ hash[word] = emb if emb
187
+ end
188
+ end
189
+
190
+ # Get batch similarities for word pairs.
191
+ #
192
+ # @param pairs [Array<Array<String, String>>] Word pairs
193
+ # @return [Array<Float>] Similarity scores
194
+ def batch_similarities(pairs)
195
+ pairs.map { |word1, word2| similarity(word1, word2) }
196
+ end
197
+
198
+ # Detect language code from file path.
199
+ #
200
+ # @param path [String] File path
201
+ # @return [String] Detected language code
202
+ def self.detect_language_from_path(path)
203
+ # Extract from path like "cc.en.300.vec" or "wiki.de.vec"
204
+ if path =~ /\.([a-z]{2})\./i
205
+ Regexp.last_match(1).downcase
206
+ else
207
+ 'en' # Default to English
208
+ end
209
+ end
210
+ end
211
+ end
212
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Value object for embedding search results (nearest neighbors).
6
+ #
7
+ # Represents a single suggestion from semantic similarity search,
8
+ # with similarity score and optional embedding reference.
9
+ #
10
+ # @example Creating a neighbor
11
+ # neighbor = NearestNeighbor.new(word: "hello", similarity: 0.85, embedding: emb)
12
+ # neighbor.to_s # => "hello [85%]"
13
+ class NearestNeighbor
14
+ attr_reader :word, :similarity, :distance, :embedding
15
+
16
+ # Create a new nearest neighbor result.
17
+ #
18
+ # @param word [String] The suggested word
19
+ # @param similarity [Float] Cosine similarity (0.0 to 1.0)
20
+ # @param distance [Float, nil] Optional precomputed distance metric
21
+ # (e.g., Euclidean). Defaults to the cosine-derived distance (1 - similarity).
22
+ # @param embedding [WordEmbedding, nil] Optional embedding reference
23
+ def initialize(word:, similarity:, distance: nil, embedding: nil)
24
+ similarity = Float(similarity)
25
+ raise ArgumentError, "Similarity must be a finite number" unless similarity.finite?
26
+ unless similarity.between?(-1.0 - 1e-6, 1.0 + 1e-6)
27
+ raise ArgumentError, "Similarity out of range: #{similarity}"
28
+ end
29
+
30
+ @word = word
31
+ # Cosine similarity is mathematically in [-1, 1]: float
32
+ # rounding can drift just past 1.0, and true negatives are
33
+ # noise suggestions (filtered by min_similarity downstream) —
34
+ # both clamp into the documented [0, 1] suggestion range.
35
+ @similarity = similarity.clamp(0.0, 1.0)
36
+ @distance = distance || (1.0 - @similarity)
37
+ @embedding = embedding
38
+ freeze
39
+ end
40
+
41
+ # Comparison for sorting (higher similarity = better).
42
+ #
43
+ # @param other [NearestNeighbor] Another neighbor
44
+ # @return [Integer] Comparison result (-1, 0, 1)
45
+ def <=>(other)
46
+ return 0 unless other.is_a?(NearestNeighbor)
47
+
48
+ # Higher similarity = better rank (sort descending)
49
+ other.similarity <=> @similarity
50
+ end
51
+
52
+ # Check if this equals another neighbor.
53
+ #
54
+ # @param other [Object] Another object
55
+ # @return [Boolean] True if words match
56
+ def ==(other)
57
+ return false unless other.is_a?(NearestNeighbor)
58
+
59
+ @word == other.word
60
+ end
61
+ alias_method :eql?, :==
62
+
63
+ # Hash code for hash table usage.
64
+ #
65
+ # @return [Integer] Hash code
66
+ def hash
67
+ @word.hash
68
+ end
69
+
70
+ # String representation with percentage.
71
+ #
72
+ # @return [String] Human-readable representation
73
+ def to_s
74
+ "#{@word} [#{(@similarity * 100).to_i}%]"
75
+ end
76
+ alias_method :inspect, :to_s
77
+
78
+ # Check if this is a high-confidence suggestion.
79
+ #
80
+ # @return [Boolean] True if similarity > 0.8
81
+ def high_confidence?
82
+ @similarity > 0.8
83
+ end
84
+
85
+ # Get confidence level category.
86
+ #
87
+ # @return [Symbol] :high, :medium, or :low
88
+ def confidence_level
89
+ return :high if @similarity > 0.8
90
+ return :medium if @similarity > 0.5
91
+
92
+ :low
93
+ end
94
+ end
95
+ end
96
+ end