kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,330 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # ONNX embedding model implementation.
6
+ #
7
+ # Loads FastText models converted to ONNX format for faster inference.
8
+ # Uses ONNX Runtime for efficient embedding lookup.
9
+ #
10
+ # @example Loading from file
11
+ # model = OnnxModel.from_file('fasttext.en.onnx')
12
+ # embedding = model.embedding_for('hello')
13
+ #
14
+ # @example Loading from GitHub (via ModelCache)
15
+ # model = OnnxModel.from_github('en')
16
+ # neighbors = model.nearest_neighbors('hello', k: 10)
17
+ class OnnxModel < EmbeddingModel
18
+ # Soft-load onnxruntime. The gem is intentionally NOT a hard runtime
19
+ # dependency — it fails to build on some platforms and would block
20
+ # install for users who only want traditional spell-checking. Semantic
21
+ # features light up automatically when the gem is present.
22
+ #
23
+ # KOTOSHU_NO_ONNX=1 forces semantic analysis off even when the gem is
24
+ # installed (useful for benchmarks / CI determinism).
25
+ ONNX_LOADED = begin
26
+ if ENV["KOTOSHU_NO_ONNX"] == "1"
27
+ false
28
+ else
29
+ require "onnxruntime"
30
+ true
31
+ end
32
+ rescue LoadError
33
+ false
34
+ end
35
+
36
+ # Error raised when semantic features are requested but onnxruntime
37
+ # is unavailable. Caller-friendly message points at the fix.
38
+ class OnnxUnavailable < Kotoshu::Error
39
+ def initialize(detail = nil)
40
+ message = "onnxruntime gem not loaded"
41
+ message += " (#{detail})" if detail
42
+ message += ". Install with: gem install onnxruntime"
43
+ message += ". Or set KOTOSHU_NO_ONNX=1 to silence this in code paths that opt out."
44
+ super(message)
45
+ end
46
+ end
47
+
48
+ # Default dimension for FastText models
49
+ DEFAULT_DIMENSION = 300
50
+
51
+ attr_reader :onnx_path, :vocabulary, :embedding_matrix
52
+
53
+ # Create a new ONNX model.
54
+ #
55
+ # @param language_code [String] ISO 639-1 language code
56
+ # @param dimension [Integer] Vector dimension
57
+ # @param onnx_path [String] Path to .onnx file
58
+ # @param vocabulary [Hash<String, Integer>] Word-to-index mapping
59
+ # @param embedding_matrix [Numo::SFloat] Pre-loaded embeddings (optional)
60
+ def initialize(language_code:, onnx_path:, vocabulary:, dimension: DEFAULT_DIMENSION, embedding_matrix: nil)
61
+ super(language_code: language_code, dimension: dimension)
62
+ @onnx_path = onnx_path
63
+ @vocabulary = vocabulary.freeze
64
+ @vocabulary_size = @vocabulary.size
65
+
66
+ # Pre-load embedding matrix if provided (for faster nearest neighbor search)
67
+ @embedding_matrix = embedding_matrix
68
+
69
+ # Lazy load session
70
+ @session = nil
71
+ @input_name = nil
72
+ @output_name = nil
73
+ @loaded = false
74
+ end
75
+
76
+ # Load ONNX model from a file.
77
+ #
78
+ # @param onnx_path [String] Path to .onnx file
79
+ # @param language_code [String] Language code (auto-detected from filename)
80
+ # @return [OnnxModel] Loaded model
81
+ # @raise [ArgumentError] if file doesn't exist
82
+ def self.from_file(onnx_path, language_code: nil)
83
+ raise ArgumentError, "File not found: #{onnx_path}" unless File.exist?(onnx_path)
84
+
85
+ # Detect language from filename if not provided
86
+ language_code ||= detect_language_from_path(onnx_path)
87
+
88
+ # Load vocabulary from .vocab.json file
89
+ vocab_path = onnx_path.sub('.onnx', '.vocab.json')
90
+ unless File.exist?(vocab_path)
91
+ raise ArgumentError, "Vocabulary file not found: #{vocab_path}"
92
+ end
93
+
94
+ require 'json'
95
+ # The vocab sibling ships in the wrapped {"vocab_size", "word_to_idx"}
96
+ # shape (written by fasttext_to_onnx.py); normalize it to a flat
97
+ # word-to-index map via the shared Vocabulary parser.
98
+ vocabulary = Embeddings::Vocabulary.word_to_index_from_document(
99
+ JSON.parse(File.read(vocab_path))
100
+ )
101
+
102
+ # Load metadata
103
+ metadata_path = onnx_path.sub('.onnx', '.metadata.json')
104
+ dimension = DEFAULT_DIMENSION
105
+
106
+ if File.exist?(metadata_path)
107
+ metadata = JSON.parse(File.read(metadata_path))
108
+ dimension = metadata['dimension']
109
+ end
110
+
111
+ new(
112
+ language_code: language_code,
113
+ dimension: dimension,
114
+ onnx_path: onnx_path,
115
+ vocabulary: vocabulary
116
+ )
117
+ end
118
+
119
+ # Load ONNX model from GitHub (via ModelCache).
120
+ #
121
+ # Downloads the .onnx file from kotoshu/dictionaries repository.
122
+ #
123
+ # @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
124
+ # @param cache [ModelCache, nil] Optional cache instance
125
+ # @return [OnnxModel] Loaded model
126
+ # @raise [ArgumentError] if language not supported
127
+ def self.from_github(language_code, cache: nil)
128
+ cache ||= Cache::ModelCache.new
129
+
130
+ # Get the .onnx file path from cache
131
+ onnx_file = cache.get_onnx_model(language_code)
132
+
133
+ from_file(onnx_file, language_code: language_code)
134
+ end
135
+
136
+ # Get embedding vector for a word.
137
+ #
138
+ # @param word [String] The word to lookup
139
+ # @return [WordEmbedding, nil] Embedding vector or nil if not found
140
+ def embedding_for(word)
141
+ return nil if word.nil? || word.empty?
142
+
143
+ index = @vocabulary[word]
144
+ return nil unless index
145
+
146
+ # Get embedding from ONNX model
147
+ vector = get_embedding_vector(index)
148
+
149
+ WordEmbedding.new(word, vector, @language_code, dimension: @dimension)
150
+ end
151
+
152
+ # Get the vocabulary (all words in the model).
153
+ #
154
+ # @return [Array<String>] Vocabulary words
155
+ def vocabulary
156
+ @vocabulary.keys
157
+ end
158
+
159
+ # Check if model is loaded.
160
+ #
161
+ # @return [Boolean] True if ONNX session is loaded
162
+ def loaded?
163
+ @loaded
164
+ end
165
+
166
+ # Find k nearest neighbors for a word.
167
+ #
168
+ # @param word [String] The query word
169
+ # @param k [Integer] Number of neighbors to return
170
+ # @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
171
+ def nearest_neighbors(word, k: 10)
172
+ ensure_session_loaded
173
+
174
+ # Get query embedding
175
+ query = embedding_for(word)
176
+ return [] unless query
177
+
178
+ # If embedding matrix is pre-loaded, use it for faster search
179
+ if @embedding_matrix
180
+ nearest_neighbors_from_matrix(query, k)
181
+ else
182
+ super
183
+ end
184
+ end
185
+
186
+ # Batch lookup of embeddings for multiple words.
187
+ #
188
+ # More efficient than individual lookups when using ONNX.
189
+ #
190
+ # @param words [Array<String>] Words to lookup
191
+ # @return [Hash<String, WordEmbedding>] Word to embedding mapping
192
+ def batch_embeddings(words)
193
+ ensure_session_loaded
194
+
195
+ indices = words.map { |w| @vocabulary[w] }
196
+ vectors = batch_get_embeddings(indices)
197
+
198
+ words.zip(indices, vectors).each_with_object({}) do |(word, idx, vec)|
199
+ next unless idx && vec
200
+
201
+ [word, WordEmbedding.new(word, vec, @language_code, dimension: @dimension)]
202
+ end
203
+ end
204
+
205
+ # Preload the embedding matrix into memory for faster nearest neighbor search.
206
+ #
207
+ # Useful when doing many nearest neighbor queries.
208
+ #
209
+ # @return [Boolean] True if loaded successfully
210
+ def preload_embedding_matrix
211
+ ensure_session_loaded
212
+
213
+ # Get all embeddings at once
214
+ all_indices = (0...@vocabulary_size).to_a
215
+ vectors = batch_get_embeddings(all_indices)
216
+
217
+ # Convert to matrix (using Numo::SFloat for efficiency)
218
+ require 'numo/narray'
219
+ @embedding_matrix = Numo::Sfloat.cast(vectors).reshape(@vocabulary_size, @dimension)
220
+
221
+ true
222
+ rescue StandardError => e
223
+ warn "Failed to preload embedding matrix: #{e.message}"
224
+ false
225
+ end
226
+
227
+ private
228
+
229
+ # Get embedding vector from ONNX model.
230
+ #
231
+ # @param index [Integer] Word index
232
+ # @return [Array<Float>] Embedding vector
233
+ def get_embedding_vector(index)
234
+ ensure_session_loaded
235
+
236
+ # The converted FastText graph takes a single int64 index (shape
237
+ # [1]) and emits one float32 vector.
238
+ result = @session.run([@output_name], { @input_name => [index] })
239
+
240
+ Array(result.first).map(&:to_f)
241
+ end
242
+
243
+ # Get embeddings for multiple indices.
244
+ #
245
+ # The graph accepts exactly one index per run, so batches are
246
+ # executed as sequential single lookups.
247
+ #
248
+ # @param indices [Array<Integer>] Word indices
249
+ # @return [Array<Array<Float>>] Embedding vectors
250
+ def batch_get_embeddings(indices)
251
+ valid_indices = indices.compact
252
+
253
+ return [] if valid_indices.empty?
254
+
255
+ valid_indices.map { |index| get_embedding_vector(index) }
256
+ end
257
+
258
+ # Find nearest neighbors using pre-loaded embedding matrix.
259
+ #
260
+ # @param query [WordEmbedding] Query embedding
261
+ # @param k [Integer] Number of neighbors
262
+ # @return [Array<NearestNeighbor>] Nearest neighbors
263
+ def nearest_neighbors_from_matrix(query, k)
264
+ return [] unless @embedding_matrix
265
+
266
+ # Compute cosine similarity with all words
267
+ query_vec = Numo::Sfloat.cast(query.vector)
268
+ similarities = []
269
+
270
+ @vocabulary.each_with_index do |(word, idx)|
271
+ vec = @embedding_matrix[idx, true]
272
+ sim = cosine_similarity(query_vec, vec)
273
+ similarities << [word, sim]
274
+ end
275
+
276
+ # Sort by similarity and take top k
277
+ similarities.sort_by { |_, s| -s }.first(k).map do |word, sim|
278
+ NearestNeighbor.new(
279
+ word: word,
280
+ similarity: sim,
281
+ embedding: embedding_for(word)
282
+ )
283
+ end
284
+ end
285
+
286
+ # Calculate cosine similarity between two vectors.
287
+ #
288
+ # @param vec1 [Numo::SFloat] First vector
289
+ # @param vec2 [Numo::SFloat] Second vector
290
+ # @return [Float] Cosine similarity
291
+ def cosine_similarity(vec1, vec2)
292
+ dot = (vec1 * vec2).sum
293
+ norm1 = Math.sqrt((vec1**2).sum)
294
+ norm2 = Math.sqrt((vec2**2).sum)
295
+
296
+ return 0.0 if norm1.zero? || norm2.zero?
297
+
298
+ dot / (norm1 * norm2)
299
+ end
300
+
301
+ # Ensure ONNX session is loaded.
302
+ def ensure_session_loaded
303
+ return if @loaded
304
+
305
+ raise OnnxUnavailable unless ONNX_LOADED
306
+
307
+ # The class is InferenceSession (OnnxRuntime::Session does not
308
+ # exist); input/output names come from the graph — the
309
+ # fasttext_to_onnx converter emits "word_index"/"embedding".
310
+ @session = OnnxRuntime::InferenceSession.new(@onnx_path)
311
+ @input_name = @session.inputs.first[:name] || 'word_index'
312
+ @output_name = @session.outputs.first[:name] || 'embedding'
313
+ @loaded = true
314
+ end
315
+
316
+ # Detect language code from file path.
317
+ #
318
+ # @param path [String] File path
319
+ # @return [String] Detected language code
320
+ def self.detect_language_from_path(path)
321
+ # Extract from path like "fasttext.en.onnx"
322
+ if path =~ /\.([a-z]{2})\./i
323
+ Regexp.last_match(1).downcase
324
+ else
325
+ 'en' # Default to English
326
+ end
327
+ end
328
+ end
329
+ end
330
+ end
@@ -0,0 +1,204 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Unified semantic error (NO artificial spelling/grammar split!).
6
+ #
7
+ # Represents ANY kind of language error detected through semantic analysis.
8
+ # Uses semantic categories instead of traditional "spelling" vs "grammar" labels.
9
+ #
10
+ # Error types (semantic categories):
11
+ # - :word_choice - Wrong word for context (e.g., "desert" vs "dessert")
12
+ # - :verb_agreement - Subject-verb mismatch (e.g., "they is" → "they are")
13
+ # - :tense - Temporal inconsistency (e.g., "Yesterday I will go")
14
+ # - :orthographic - Actual typo/misspelling (e.g., "wrold" → "world")
15
+ # - :preposition - Wrong preposition (e.g., "bored of" → "bored with")
16
+ # - :article - Wrong article (e.g., "a apple" → "an apple")
17
+ # - :morphology - Wrong word form (e.g., "goed" → "went")
18
+ # - :capitalization - Capitalization error (e.g., "i am" → "I am")
19
+ # - :punctuation - Punctuation error (e.g., "its" vs "it's")
20
+ # - :style - Style/usage suggestion
21
+ #
22
+ # @example Creating a semantic error
23
+ # error = SemanticError.new(
24
+ # id: "error_1",
25
+ # location: Location.new(line: 5, column: 12),
26
+ # original: "desert",
27
+ # suggestions: [Suggestion.new("dessert", confidence: 0.92)],
28
+ # error_type: :word_choice,
29
+ # confidence: 0.92,
30
+ # context: context
31
+ # )
32
+ # Raised by {SemanticError.initialize} when +suggestions+ is empty
33
+ # and +allow_empty_suggestions:+ is false. The analyzer rescues
34
+ # this so words with no close matches are silently dropped rather
35
+ # than producing noise.
36
+ class EmptySuggestionsError < ArgumentError
37
+ end
38
+
39
+ class SemanticError
40
+ # Error type definitions with display names
41
+ ERROR_TYPES = {
42
+ word_choice: 'Word Choice',
43
+ verb_agreement: 'Verb Agreement',
44
+ tense: 'Tense',
45
+ orthographic: 'Spelling',
46
+ preposition: 'Preposition',
47
+ article: 'Article',
48
+ morphology: 'Word Form',
49
+ capitalization: 'Capitalization',
50
+ punctuation: 'Punctuation',
51
+ style: 'Style'
52
+ }.freeze
53
+
54
+ attr_reader :id, :source_range, :location, :original, :suggestions,
55
+ :error_type, :confidence, :context
56
+
57
+ # Create a new semantic error.
58
+ #
59
+ # @param id [String, Symbol] Unique identifier for this error
60
+ # @param source_range [Kotoshu::Documents::SourceRange, nil]
61
+ # Where the offending text lives in the original markup-bearing
62
+ # source. Carried verbatim so editors/plugins can highlight the
63
+ # actual range the user wrote.
64
+ # @param location [Object, nil] Legacy location holder (line/column
65
+ # object). Kept for backward compat with old callers; new code
66
+ # should pass +source_range+ instead.
67
+ # @param original [String] The original (incorrect) word/text
68
+ # @param suggestions [Array<Suggestion>] Suggested corrections
69
+ # @param error_type [Symbol] Error type (must be in ERROR_TYPES)
70
+ # @param confidence [Float] Confidence score (0.0 to 1.0)
71
+ # @param context [Context, nil] Context around the error
72
+ # @param allow_empty_suggestions [Boolean] When false (default),
73
+ # raises {EmptySuggestionsError} instead of building an error
74
+ # with zero suggestions.
75
+ # @raise [ArgumentError] if error_type is invalid
76
+ # @raise [ArgumentError] if confidence is outside [0, 1]
77
+ # @raise [EmptySuggestionsError] if suggestions is empty and
78
+ # allow_empty_suggestions is false
79
+ def initialize(id:, original:, suggestions:, error_type:, confidence:, source_range: nil, location: nil, context: nil,
80
+ allow_empty_suggestions: false)
81
+ raise ArgumentError, "Invalid error type: #{error_type}" unless ERROR_TYPES.key?(error_type)
82
+ raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
83
+
84
+ if (suggestions.nil? || suggestions.empty?) && !allow_empty_suggestions
85
+ raise EmptySuggestionsError,
86
+ "Suggestions cannot be empty (pass allow_empty_suggestions: true to override)"
87
+ end
88
+
89
+ @id = id.to_s
90
+ @source_range = source_range
91
+ @location = location || source_range&.start
92
+ @original = original
93
+ @suggestions = (suggestions || []).sort_by(&:confidence).reverse.freeze
94
+ @error_type = error_type
95
+ @confidence = confidence
96
+ @context = context
97
+
98
+ freeze
99
+ end
100
+
101
+ # Get user-friendly display type name.
102
+ #
103
+ # @return [String] Display type name
104
+ def display_type
105
+ ERROR_TYPES[@error_type] || @error_type.to_s.capitalize
106
+ end
107
+
108
+ # Check if this is a high-confidence error.
109
+ #
110
+ # @return [Boolean] True if confidence > 0.8
111
+ def high_confidence?
112
+ @confidence > 0.8
113
+ end
114
+
115
+ # Get confidence level category.
116
+ #
117
+ # @return [Symbol] :high, :medium, or :low
118
+ def confidence_level
119
+ return :high if @confidence > 0.8
120
+ return :medium if @confidence > 0.5
121
+
122
+ :low
123
+ end
124
+
125
+ # Get the recommended (top) suggestion.
126
+ #
127
+ # @return [Suggestion] The highest-confidence suggestion
128
+ def recommended_suggestion
129
+ @suggestions.first
130
+ end
131
+
132
+ # Check if this error equals another.
133
+ #
134
+ # @param other [Object] Another object
135
+ # @return [Boolean] True if IDs match
136
+ def ==(other)
137
+ return false unless other.is_a?(SemanticError)
138
+
139
+ @id == other.id
140
+ end
141
+ alias_method :eql?, :==
142
+
143
+ # Hash code for hash table usage.
144
+ #
145
+ # @return [Integer] Hash code
146
+ def hash
147
+ @id.hash
148
+ end
149
+
150
+ # Comparison for sorting (by source position, then confidence).
151
+ #
152
+ # Errors are sorted by:
153
+ # 1. Source position (source_range.start when present; falls back
154
+ # to legacy +location+ when source_range is nil)
155
+ # 2. Confidence (highest first)
156
+ #
157
+ # @param other [SemanticError] Another error
158
+ # @return [Integer] Comparison result (-1, 0, 1)
159
+ def <=>(other)
160
+ return 0 unless other.is_a?(SemanticError)
161
+
162
+ a_pos = sort_position
163
+ b_pos = other.sort_position
164
+ pos_cmp = a_pos <=> b_pos
165
+ return pos_cmp unless pos_cmp.zero?
166
+
167
+ other.confidence <=> @confidence
168
+ end
169
+
170
+ # String representation.
171
+ #
172
+ # @return [String] Human-readable representation
173
+ def to_s
174
+ "#{@location}: '#{@original}' → #{recommended_suggestion.word} [#{(@confidence * 100).to_i}%]"
175
+ end
176
+ alias_method :inspect, :to_s
177
+
178
+ # Create an abbreviated display for lists.
179
+ #
180
+ # @param max_length [Integer] Maximum line length
181
+ # @return [String] Abbreviated representation
182
+ def abbreviated(max_length: 80)
183
+ orig_display = "'#{@original}'"
184
+ sugg_display = "'#{recommended_suggestion.word}'"
185
+
186
+ "#{@location}: #{orig_display} → #{sugg_display} [#{(@confidence * 100).to_i}%]"
187
+ end
188
+
189
+ protected
190
+
191
+ # The position used for sorting. Prefers source_range.start
192
+ # (the new contract); falls back to legacy +location+ for
193
+ # errors built without a document.
194
+ #
195
+ # Visible to other SemanticError instances (protected, not private)
196
+ # because <=> reads `other.sort_position` when comparing.
197
+ def sort_position
198
+ return @source_range.start if @source_range
199
+
200
+ @location
201
+ end
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Value object for correction suggestions.
6
+ #
7
+ # Represents a suggested correction for a detected error,
8
+ # with confidence score and metadata.
9
+ #
10
+ # @example Creating a suggestion
11
+ # suggestion = Suggestion.new("dessert", confidence: 0.92, source: :semantic)
12
+ # suggestion.to_s # => "dessert [92%]"
13
+ class Suggestion
14
+ attr_reader :word, :confidence, :source, :metadata
15
+
16
+ # Create a new suggestion.
17
+ #
18
+ # @param word [String] The suggested word
19
+ # @param confidence [Float] Confidence score (0.0 to 1.0)
20
+ # @param source [Symbol, nil] Source of the suggestion (e.g., :semantic, :edit_distance)
21
+ # @param metadata [Hash] Additional metadata (optional)
22
+ # @option metadata [WordEmbedding, nil] :embedding The word embedding
23
+ # @option metadata [Float] :edit_distance Edit distance score
24
+ # @option metadata [Float] :frequency_bonus Frequency score bonus
25
+ # @option metadata [String] :explanation Explanation for the suggestion
26
+ def initialize(word, confidence:, source: nil, metadata: {})
27
+ raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
28
+
29
+ @word = word
30
+ @confidence = confidence
31
+ @source = source || :unknown
32
+ @metadata = metadata.freeze
33
+ freeze
34
+ end
35
+
36
+ # Comparison for sorting (higher confidence = better).
37
+ #
38
+ # @param other [Suggestion] Another suggestion
39
+ # @return [Integer] Comparison result (-1, 0, 1)
40
+ def <=>(other)
41
+ return 0 unless other.is_a?(Suggestion)
42
+
43
+ # Higher confidence = better rank (sort descending)
44
+ other.confidence <=> @confidence
45
+ end
46
+
47
+ # Check if this equals another suggestion.
48
+ #
49
+ # @param other [Object] Another object
50
+ # @return [Boolean] True if words match
51
+ def ==(other)
52
+ return false unless other.is_a?(Suggestion)
53
+
54
+ @word == other.word
55
+ end
56
+ alias_method :eql?, :==
57
+
58
+ # Hash code for hash table usage.
59
+ #
60
+ # @return [Integer] Hash code
61
+ def hash
62
+ @word.hash
63
+ end
64
+
65
+ # String representation with percentage.
66
+ #
67
+ # @return [String] Human-readable representation
68
+ def to_s
69
+ if @source && @source != :unknown
70
+ "#{@word} [#{(@confidence * 100).to_i}%] (#{@source})"
71
+ else
72
+ "#{@word} [#{(@confidence * 100).to_i}%]"
73
+ end
74
+ end
75
+ alias_method :inspect, :to_s
76
+
77
+ # Get the embedding if available.
78
+ #
79
+ # @return [WordEmbedding, nil] The embedding or nil
80
+ def embedding
81
+ @metadata[:embedding]
82
+ end
83
+
84
+ # Get the edit distance if available.
85
+ #
86
+ # @return [Float, nil] Edit distance or nil
87
+ def edit_distance
88
+ @metadata[:edit_distance]
89
+ end
90
+
91
+ # Check if this is a high-confidence suggestion.
92
+ #
93
+ # @return [Boolean] True if confidence > 0.8
94
+ def high_confidence?
95
+ @confidence > 0.8
96
+ end
97
+
98
+ # Get explanation text if available.
99
+ #
100
+ # @return [String, nil] Explanation or nil
101
+ def explanation
102
+ @metadata[:explanation]
103
+ end
104
+ end
105
+ end
106
+ end