kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Immutable value object for word embeddings.
6
+ #
7
+ # Represents a word and its vector representation in a semantic space.
8
+ # Used for semantic similarity calculations and nearest neighbor searches.
9
+ #
10
+ # @example Creating an embedding
11
+ # embedding = WordEmbedding.new("hello", [0.1, -0.2, 0.3], "en")
12
+ # embedding.similarity(other_embedding) # => 0.85
13
+ #
14
+ # @see https://fasttext.cc/docs/en/crawl-vectors.html FastText crawl vectors
15
+ class WordEmbedding
16
+ attr_reader :word, :vector, :language_code, :dimension, :magnitude
17
+
18
+ # Create a new word embedding.
19
+ #
20
+ # @param word [String] The word
21
+ # @param vector [Array<Float>] The word's vector representation
22
+ # @param language_code [String] ISO 639-1 language code
23
+ # @param dimension [Integer] Vector dimension (default: 300 for FastText)
24
+ # @raise [ArgumentError] if vector doesn't match dimension
25
+ def initialize(word, vector, language_code, dimension: 300)
26
+ raise ArgumentError, "Vector dimension mismatch" unless vector.size == dimension
27
+
28
+ @word = word
29
+ @vector = vector.freeze
30
+ @language_code = language_code
31
+ @dimension = dimension
32
+ @magnitude = Math.sqrt(@vector.map { |x| x * x }.sum)
33
+
34
+ freeze
35
+ end
36
+
37
+ # Calculate cosine similarity with another embedding.
38
+ #
39
+ # Cosine similarity measures the cosine of the angle between two vectors.
40
+ # Returns 1.0 for identical vectors, 0.0 for orthogonal vectors.
41
+ #
42
+ # @param other [WordEmbedding] Another embedding
43
+ # @return [Float] Similarity score (0.0 to 1.0)
44
+ # @raise [TypeError] if other is not a WordEmbedding
45
+ def similarity(other)
46
+ raise TypeError, "Must be WordEmbedding" unless other.is_a?(WordEmbedding)
47
+
48
+ return 0.0 if @dimension != other.dimension
49
+
50
+ dot_product = @vector.zip(other.vector).map { |a, b| a * b }.sum
51
+
52
+ return 0.0 if magnitude.zero? || other.magnitude.zero?
53
+
54
+ dot_product / (magnitude * other.magnitude)
55
+ end
56
+
57
+ # Calculate Euclidean distance from another embedding.
58
+ #
59
+ # @param other [WordEmbedding] Another embedding
60
+ # @return [Float] Euclidean distance
61
+ # @raise [TypeError] if other is not a WordEmbedding
62
+ def distance(other)
63
+ raise TypeError, "Must be WordEmbedding" unless other.is_a?(WordEmbedding)
64
+
65
+ return Float::INFINITY if @dimension != other.dimension
66
+
67
+ Math.sqrt(@vector.zip(other.vector).map { |a, b| (a - b)**2 }.sum)
68
+ end
69
+
70
+ # Check if this embedding is equal to another.
71
+ #
72
+ # @param other [Object] Another object
73
+ # @return [Boolean] True if words and languages match
74
+ def ==(other)
75
+ return false unless other.is_a?(WordEmbedding)
76
+
77
+ @word == other.word && @language_code == other.language_code
78
+ end
79
+ alias_method :eql?, :==
80
+
81
+ # Hash code for hash table usage.
82
+ #
83
+ # @return [Integer] Hash code
84
+ def hash
85
+ [@word, @language_code].hash
86
+ end
87
+
88
+ # String representation.
89
+ #
90
+ # @return [String] Human-readable representation
91
+ def to_s
92
+ "#{self.class.name}[#{@word}, #{@language_code}, #{@dimension}D]"
93
+ end
94
+ alias_method :inspect, :to_s
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Embedding models for semantic spell checking (FastText via ONNX),
5
+ # and core domain value objects (Word, AffixRule, Result types).
6
+ #
7
+ # Note: Word, AffixRule, and Result types live under Kotoshu::Models
8
+ # for historical reasons even though their files are in core/models/.
9
+ module Models
10
+ autoload :Word, "kotoshu/core/models/word"
11
+ autoload :AffixRule, "kotoshu/core/models/affix_rule"
12
+ autoload :Context, "kotoshu/models/context"
13
+ autoload :EmbeddingModel, "kotoshu/models/embedding_model"
14
+ autoload :FastTextModel, "kotoshu/models/fasttext_model"
15
+ autoload :NearestNeighbor, "kotoshu/models/nearest_neighbor"
16
+ autoload :OnnxModel, "kotoshu/models/onnx_model"
17
+ autoload :SemanticError, "kotoshu/models/semantic_error"
18
+ autoload :Suggestion, "kotoshu/models/suggestion"
19
+ autoload :WordEmbedding, "kotoshu/models/word_embedding"
20
+
21
+ # Result sub-namespace (word/document check results).
22
+ module Result
23
+ autoload :WordResult, "kotoshu/core/models/result/word_result"
24
+ autoload :DocumentResult, "kotoshu/core/models/result/document_result"
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Multi-language document checker.
5
+ #
6
+ # Experimental and not public API: outside the 1.0 stability
7
+ # freeze, may change or be removed without a deprecation cycle.
8
+ #
9
+ # Splits a {Documents::Document} into language-tagged segments via
10
+ # {Language::Segmenter}, resolves a {Spellchecker} for each
11
+ # segment's detected language, and emits a flat list of
12
+ # {Models::SemanticError}s carrying source_range from the original
13
+ # document. The result is one sorted stream of errors that editors
14
+ # / plugins can highlight without caring that the document mixed
15
+ # languages.
16
+ #
17
+ # Real documents mix languages routinely: code comments in English
18
+ # embedded in French prose, a Japanese paper with an English
19
+ # abstract, German quotes in a Spanish essay. Without per-segment
20
+ # detection, every word from the "other" language is reported as a
21
+ # spelling error against the wrong dictionary.
22
+ class MultiLanguageChecker
23
+ # @return [Language::Segmenter]
24
+ attr_reader :segmenter
25
+
26
+ # @return [Hash{String => Spellchecker}] per-language cache so
27
+ # repeated segments of the same language reuse the same checker
28
+ attr_reader :spellcheckers
29
+
30
+ # @param segmenter [Language::Segmenter, nil] defaults to a new
31
+ # Segmenter with the default Detector
32
+ def initialize(segmenter: nil)
33
+ @segmenter = segmenter || Language::Segmenter.new
34
+ @spellcheckers = {}
35
+ end
36
+
37
+ # Check a document, returning one sorted stream of errors across
38
+ # every detected language. Errors carry source_range resolved via
39
+ # the document's text-node mapping.
40
+ #
41
+ # @param document [Kotoshu::Documents::Document]
42
+ # @return [Array<Models::SemanticError>]
43
+ def check(document)
44
+ unless document.is_a?(Kotoshu::Documents::Document)
45
+ raise ArgumentError,
46
+ "document must be a Kotoshu::Documents::Document"
47
+ end
48
+
49
+ @segmenter.segment(document).flat_map do |segment|
50
+ check_segment(segment, document)
51
+ end.sort
52
+ end
53
+
54
+ private
55
+
56
+ # Check one segment: resolve (or reuse) a Spellchecker for the
57
+ # segment's language, walk its text nodes, and build errors with
58
+ # source_range for every unknown word that has at least one
59
+ # suggestion.
60
+ def check_segment(segment, document)
61
+ checker = spellchecker_for(segment.language_code)
62
+ return [] unless checker
63
+
64
+ errors = []
65
+ segment.text_nodes.each do |node|
66
+ tokenize_with_offsets(node.text).each do |word, offset_in_node|
67
+ next if checker.correct?(word)
68
+
69
+ suggestions = checker.suggest(word).to_a
70
+ next if suggestions.empty?
71
+
72
+ flattened_start = node.flattened_offset + offset_in_node
73
+ flattened_end = flattened_start + word.length
74
+ source_range = document.source_range_for(flattened_start, flattened_end)
75
+ errors << build_error(word, suggestions, source_range, segment.language_code)
76
+ end
77
+ end
78
+ errors
79
+ end
80
+
81
+ # Resolve a Spellchecker for a language code, memoized per
82
+ # process. Returns nil when the language isn't set up — callers
83
+ # silently skip unknown-language segments rather than crashing.
84
+ def spellchecker_for(language_code)
85
+ return @spellcheckers[language_code] if @spellcheckers.key?(language_code)
86
+
87
+ @spellcheckers[language_code] = begin
88
+ bundle = ResourceManager.resolve(language: language_code, want: %i[spelling])
89
+ Spellchecker.new(resource_bundle: bundle)
90
+ rescue StandardError
91
+ nil
92
+ end
93
+ end
94
+
95
+ # Build a SemanticError. Uses the orthographic error type (this
96
+ # is a spelling check, not a grammar check). Confidence is a
97
+ # fixed 0.5 (medium) since we don't compute real confidence here.
98
+ def build_error(word, suggestions, source_range, language_code)
99
+ Models::SemanticError.new(
100
+ id: Digest::SHA256.hexdigest("#{language_code}-#{word}-#{source_range}")[0...16],
101
+ source_range: source_range,
102
+ original: word,
103
+ suggestions: wrap_suggestions(suggestions),
104
+ error_type: :orthographic,
105
+ confidence: 0.5,
106
+ context: nil
107
+ )
108
+ end
109
+
110
+ # Wrap raw suggestion strings into Models::Suggestion objects so
111
+ # the SemanticError invariant (array of Suggestion) is satisfied.
112
+ def wrap_suggestions(words)
113
+ words.first(5).map do |word|
114
+ Models::Suggestion.new(word, confidence: 0.5, source: :multi_language)
115
+ end
116
+ end
117
+
118
+ # Tokenize +text+ into [word, char_offset] pairs.
119
+ def tokenize_with_offsets(text)
120
+ return [] unless text
121
+
122
+ text.scan(/[a-zà-ÿ]+(?:['’-][a-zà-ÿ]+)*/i).map do |match|
123
+ [match.downcase, Regexp.last_match.offset(0).first]
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Ruby-side home of the native engine (plan 66, P4b).
5
+ #
6
+ # The Rust extension +kotoshu/kotoshu_native+ (built from
7
+ # ext/kotoshu_native) defines this module's engine surface when it loads:
8
+ #
9
+ # Kotoshu::Native::VERSION # kotoshu-rs crate version
10
+ # Kotoshu::Native.available? # => true
11
+ # Kotoshu::Native::Error # engine failures (RuntimeError)
12
+ # dictionary = Kotoshu::Native::Dictionary.load(aff_path, dic_path)
13
+ # dictionary.correct?("hello") # => true / false
14
+ # dictionary.suggest("hlelo", 5) # => [{ "word" => "hello", ... }]
15
+ #
16
+ # This file adds two things the extension cannot:
17
+ #
18
+ # * the safe loader — +require "kotoshu"+ never fails when the extension
19
+ # is absent. A missing Rust toolchain, an uncompiled checkout, or a
20
+ # slim install simply leaves {Native.available?} false, and the
21
+ # pure-Ruby backend remains the complete engine. Mirrors the parsanol
22
+ # loader pattern (lib/parsanol/native.rb): rescue LoadError, stay quiet.
23
+ # * the adapter — native suggestion rows are Hashes carrying exactly the
24
+ # four public keys of {Suggestions::Suggestion}; {Native.suggestion}
25
+ # materializes the model object from a row. No serialization is
26
+ # hand-rolled here: the row is constructor input, the model's
27
+ # (de)serialization stays lutaml-model's.
28
+ module Native
29
+ # Raised by backend selection when the native engine is explicitly
30
+ # requested ({Configuration#backend} = "native") but cannot serve the
31
+ # request: extension not built, or a dictionary type the native engine
32
+ # does not load. Failing loudly is deliberate — "native" exists for
33
+ # debugging and benchmarking, where a silent fallback to Ruby would
34
+ # produce misleading measurements.
35
+ class Unavailable < Kotoshu::Error; end
36
+
37
+ # The extension's require path (rb_sys installs it as
38
+ # kotoshu/kotoshu_native.<dlext> under lib/).
39
+ EXTENSION = "kotoshu/kotoshu_native"
40
+
41
+ class << self
42
+ # Whether the native extension is loaded.
43
+ #
44
+ # Overwritten by the extension itself when it loads (its own
45
+ # +available?+ answers true), so this definition only stands when the
46
+ # require below failed.
47
+ #
48
+ # @return [Boolean]
49
+ def available?
50
+ false
51
+ end
52
+
53
+ # Materialize a {Suggestions::Suggestion} from a native suggestion
54
+ # row.
55
+ #
56
+ # The Rust engine returns one Hash per suggestion with exactly the
57
+ # four public keys of the gem's Suggestion shape ("word" String,
58
+ # "distance" Integer, "confidence" Float in [0, 1], "source" String)
59
+ # — the same four keys the conformance vectors freeze
60
+ # ({ConformanceExporter::SUGGESTION_KEYS}). This constructor hop is
61
+ # the whole adapter; rows and model attributes are one-to-one.
62
+ #
63
+ # @param row [Hash] native suggestion row
64
+ # @return [Suggestions::Suggestion]
65
+ def suggestion(row)
66
+ Suggestions::Suggestion.new(
67
+ word: row.fetch("word"),
68
+ distance: row.fetch("distance"),
69
+ confidence: row.fetch("confidence"),
70
+ source: row.fetch("source")
71
+ )
72
+ end
73
+ end
74
+ end
75
+
76
+ begin
77
+ require Native::EXTENSION
78
+ rescue LoadError
79
+ # Extension not built (no Rust toolchain, or `rake compile` not run).
80
+ # Nothing else in the gem requires it: the pure-Ruby backend is the
81
+ # complete engine and every public API works without the extension.
82
+ end
83
+ end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Native engine adapter (plan 66, P4b): +correct?+ and +suggest+ for one
5
+ # loaded Hunspell dictionary, computed by the kotoshu-rs core through
6
+ # {Native}.
7
+ #
8
+ # == Boundary (deliberate, documented)
9
+ #
10
+ # The native engine accelerates exactly the two hot-path calls of the
11
+ # traditional checking path — correct?(word) and suggest(word) — over a
12
+ # file-backed Hunspell dictionary, where kotoshu-rs implements the full
13
+ # dictionary + affix + suggest composite and is held to byte-identical
14
+ # results by the conformance vectors (rake kotoshu:conformance:compare).
15
+ # Everything else stays Ruby: non-Hunspell dictionary backends, the
16
+ # suggestion strategy pipeline beyond the conformance surface, the
17
+ # semantic path ({Analyzers::SemanticAnalyzer}, ONNX models), grammar
18
+ # rules, caches, and document tokenizing. The extension is a pure
19
+ # accelerator, never a dependency.
20
+ #
21
+ # == Selection
22
+ #
23
+ # See {Configuration#backend} / +KOTOSHU_BACKEND+:
24
+ #
25
+ # ruby :: never uses the native engine (default).
26
+ # auto :: uses it when the extension is loaded AND the dictionary is a
27
+ # file-backed Hunspell; silently pure-Ruby otherwise.
28
+ # native :: uses it or raises {Native::Unavailable} — loud failure for
29
+ # debugging and benchmarking.
30
+ class NativeBackend
31
+ # @return [Kotoshu::Native::Dictionary] the loaded native dictionary
32
+ attr_reader :native_dictionary
33
+
34
+ # Resolve which backend serves a Spellchecker built over +dictionary+.
35
+ #
36
+ # @param dictionary [Dictionary::Base, Object] the dictionary the
37
+ # Spellchecker was built with
38
+ # @param backend [String, Symbol] "ruby", "auto", or "native"
39
+ # @param max_suggestions [Integer] default suggestion limit
40
+ # @return [NativeBackend, nil] nil when the pure-Ruby engine serves the
41
+ # request
42
+ # @raise [Native::Unavailable] backend "native" and the engine cannot
43
+ # serve the request
44
+ # @raise [ConfigurationError] unknown backend value
45
+ def self.resolve(dictionary:, backend:, max_suggestions: 10)
46
+ case backend.to_s
47
+ when "ruby"
48
+ nil
49
+ when "auto"
50
+ paths = hunspell_paths(dictionary)
51
+ return nil unless paths && Kotoshu::Native.available?
52
+
53
+ new(aff_path: paths.fetch(0), dic_path: paths.fetch(1), max_suggestions: max_suggestions)
54
+ when "native"
55
+ unless Kotoshu::Native.available?
56
+ raise Native::Unavailable,
57
+ "native backend requested (KOTOSHU_BACKEND=native) but the extension is not " \
58
+ "available — build it with `rake compile` or install the gem where it compiled"
59
+ end
60
+
61
+ paths = hunspell_paths(dictionary)
62
+ unless paths
63
+ raise Native::Unavailable,
64
+ "native backend requested (KOTOSHU_BACKEND=native) but it only loads " \
65
+ "Hunspell dictionaries — got #{dictionary.class}"
66
+ end
67
+
68
+ new(aff_path: paths.fetch(0), dic_path: paths.fetch(1), max_suggestions: max_suggestions)
69
+ else
70
+ raise ConfigurationError,
71
+ "unknown backend #{backend.inspect} (expected \"ruby\", \"auto\", or \"native\")"
72
+ end
73
+ end
74
+
75
+ # The (aff, dic) paths of a file-backed Hunspell dictionary — the only
76
+ # dictionary form the native engine loads. Everything else answers nil.
77
+ #
78
+ # @param dictionary [Object]
79
+ # @return [Array<String>, nil]
80
+ def self.hunspell_paths(dictionary)
81
+ return nil unless dictionary.is_a?(Dictionary::Hunspell)
82
+
83
+ aff = dictionary.aff_path
84
+ dic = dictionary.dic_path
85
+ return nil if aff.nil? || dic.nil?
86
+
87
+ [aff, dic]
88
+ end
89
+
90
+ # @param aff_path [String] path to the .aff file
91
+ # @param dic_path [String] path to the .dic file
92
+ # @param max_suggestions [Integer] default suggestion limit
93
+ # @raise [Native::Error] the dictionary failed to load (message from
94
+ # the Rust engine)
95
+ def initialize(aff_path:, dic_path:, max_suggestions: 10)
96
+ @max_suggestions = max_suggestions
97
+ @native_dictionary = Kotoshu::Native::Dictionary.load(aff_path, dic_path)
98
+ end
99
+
100
+ # Check a word against the native dictionary.
101
+ #
102
+ # @param word [String]
103
+ # @return [Boolean]
104
+ def correct?(word)
105
+ @native_dictionary.correct?(word)
106
+ end
107
+
108
+ # Suggest corrections for a word through the native engine, returning
109
+ # the same {Suggestions::SuggestionSet} the Ruby path returns.
110
+ #
111
+ # The Rust engine returns rows already ranked, deduplicated, and
112
+ # limited — its ranking is conformance-frozen to the Ruby engine's
113
+ # behavior (rake kotoshu:conformance:compare), and its tie order can
114
+ # depend on ranking signals that do not live on the Suggestion objects.
115
+ # The set therefore adopts the native order verbatim (ranked: true)
116
+ # instead of re-sorting.
117
+ #
118
+ # @param word [String]
119
+ # @param max_suggestions [Integer, nil] limit override
120
+ # @return [Suggestions::SuggestionSet]
121
+ def suggest(word, max_suggestions: nil)
122
+ limit = max_suggestions || @max_suggestions
123
+ rows = @native_dictionary.suggest(word, limit)
124
+ Suggestions::SuggestionSet.new(
125
+ rows.map { |row| Kotoshu::Native.suggestion(row) },
126
+ max_size: limit,
127
+ ranked: true
128
+ )
129
+ end
130
+ end
131
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ module Kotoshu
6
+ # Centralized XDG Base Directory paths.
7
+ #
8
+ # Resolves cache, config, and data paths per the XDG Base Directory
9
+ # Specification. Kotoshu-specific env vars (KOTOSHU_CACHE_PATH etc.)
10
+ # take precedence over the generic XDG_*_HOME vars, which take
11
+ # precedence over the hardcoded defaults.
12
+ #
13
+ # Default layout:
14
+ # ~/.cache/kotoshu/ downloaded dicts, models, frequency lists
15
+ # ~/.config/kotoshu/ user-edited config, personal dictionary
16
+ # ~/.local/share/kotoshu/ append-only data (audit log)
17
+ module Paths
18
+ class << self
19
+ # Root directory for downloaded resources (dictionaries,
20
+ # frequency lists, ONNX models).
21
+ #
22
+ # @return [String] KOTOSHU_CACHE_PATH, or
23
+ # `$XDG_CACHE_HOME/kotoshu` (default `~/.cache/kotoshu`)
24
+ def cache_path
25
+ ENV.fetch("KOTOSHU_CACHE_PATH", nil) || xdg("CACHE", "cache")
26
+ end
27
+
28
+ # Root directory for user configuration (kotoshu.cfg, personal
29
+ # dictionary).
30
+ #
31
+ # @return [String] KOTOSHU_CONFIG_PATH, or
32
+ # `$XDG_CONFIG_HOME/kotoshu` (default `~/.config/kotoshu`)
33
+ def config_path
34
+ ENV.fetch("KOTOSHU_CONFIG_PATH", nil) || xdg("CONFIG", "config")
35
+ end
36
+
37
+ # Root directory for shared state (audit log).
38
+ #
39
+ # @return [String] KOTOSHU_DATA_PATH, or
40
+ # `$XDG_DATA_HOME/kotoshu` (default `~/.local/share/kotoshu`)
41
+ def data_path
42
+ ENV.fetch("KOTOSHU_DATA_PATH", nil) || xdg("DATA", "local/share")
43
+ end
44
+
45
+ # Where audited operations are appended (with rotation; see
46
+ # Configuration#audit_max_bytes / #audit_rotations).
47
+ #
48
+ # @return [String] KOTOSHU_AUDIT_LOG, or `<data_path>/audit.log`
49
+ def audit_log_path
50
+ ENV.fetch("KOTOSHU_AUDIT_LOG", nil) || File.join(data_path, "audit.log")
51
+ end
52
+
53
+ # Where {PersonalDictionary} stores its words (Hunspell word-list
54
+ # format).
55
+ #
56
+ # @return [String] KOTOSHU_PERSONAL_DIC, or
57
+ # `<config_path>/personal.dic`
58
+ def personal_dictionary_path
59
+ ENV.fetch("KOTOSHU_PERSONAL_DIC", nil) || File.join(config_path, "personal.dic")
60
+ end
61
+
62
+ # Create the cache, config, and data directories. Idempotent.
63
+ #
64
+ # @return [void]
65
+ def ensure_exist!
66
+ FileUtils.mkdir_p(cache_path)
67
+ FileUtils.mkdir_p(config_path)
68
+ FileUtils.mkdir_p(data_path)
69
+ end
70
+
71
+ private
72
+
73
+ def xdg(suffix, default_subdir)
74
+ base = ENV.fetch("XDG_#{suffix}_HOME", nil) || File.join(Dir.home, ".#{default_subdir}")
75
+ File.join(base, "kotoshu")
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ module Kotoshu
6
+ # Personal dictionary for user-specific words.
7
+ #
8
+ # Stored in ~/.config/kotoshu/personal.dic (Hunspell format) under the
9
+ # XDG config directory. Override via KOTOSHU_PERSONAL_DIC.
10
+ #
11
+ # The on-disk path is resolved lazily on every call so env-var
12
+ # overrides take effect at runtime — handy for tests and for users
13
+ # who set KOTOSHU_PERSONAL_DIC after the gem loads.
14
+ class PersonalDictionary
15
+ class << self
16
+ # Resolve the personal-dictionary file path from Paths (which
17
+ # honors KOTOSHU_PERSONAL_DIC). Read fresh each call so an
18
+ # env override applied after gem load still takes effect.
19
+ #
20
+ # @return [String]
21
+ def file_path
22
+ Kotoshu::Paths.personal_dictionary_path
23
+ end
24
+
25
+ # Add a word to personal dictionary.
26
+ #
27
+ # @param word [String] Word to add
28
+ # @return [Boolean] True if the word was newly added (i.e. it
29
+ # wasn't already in the dictionary); false otherwise.
30
+ def add_word(word)
31
+ return false if word.nil? || word.empty?
32
+
33
+ ensure_directory
34
+ words = load_words
35
+
36
+ return false if words.include?(word.downcase)
37
+
38
+ words << word.downcase
39
+ save_words(words)
40
+ true
41
+ end
42
+
43
+ # Get all personal words.
44
+ #
45
+ # @return [Array<String>] All personal words
46
+ def words
47
+ load_words
48
+ end
49
+
50
+ # Remove a word from personal dictionary.
51
+ #
52
+ # @param word [String] Word to remove
53
+ # @return [Boolean] True if removed
54
+ def remove_word(word)
55
+ return false if word.nil? || word.empty?
56
+
57
+ words = load_words
58
+ if words.delete(word.downcase)
59
+ save_words(words)
60
+ true
61
+ else
62
+ false
63
+ end
64
+ end
65
+
66
+ # Check if word is in personal dictionary.
67
+ #
68
+ # @param word [String] Word to check
69
+ # @return [Boolean] True if present
70
+ def include?(word)
71
+ return false if word.nil? || word.empty?
72
+
73
+ load_words.include?(word.downcase)
74
+ end
75
+
76
+ private
77
+
78
+ # Ensure personal dictionary's parent directory exists.
79
+ def ensure_directory
80
+ FileUtils.mkdir_p(File.dirname(file_path))
81
+ end
82
+
83
+ # Load words from personal dictionary file.
84
+ #
85
+ # @return [Array<String>] List of words
86
+ def load_words
87
+ return [] unless File.exist?(file_path)
88
+
89
+ File.readlines(file_path, chomp: true)
90
+ .reject { |line| line.empty? || line.start_with?("#") }
91
+ .map(&:strip)
92
+ end
93
+
94
+ # Save words to personal dictionary file.
95
+ #
96
+ # @param words [Array<String>] Words to save
97
+ def save_words(words)
98
+ ensure_directory
99
+ File.open(file_path, "w") do |f|
100
+ words.sort.uniq.each { |word| f.puts word }
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end