kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,600 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Two-stage resource resolution.
5
+ #
6
+ # Stage 1 — setup (slow, network-required, explicit):
7
+ # Kotoshu.setup(:en) # download from kotoshu/dictionaries
8
+ # Kotoshu.setup(:en, want: %i[spelling frequency])
9
+ # Kotoshu.setup(:en, aff: "/path/to.en.aff", dic: "/path/to/en.dic") # local files
10
+ # Kotoshu.setup(:en, from: "/path/to/dict/dir/") # local directory
11
+ #
12
+ # Stage 2 — resolve (instant, cache-only, raises on miss):
13
+ # bundle = Kotoshu::ResourceManager.resolve(language: "en")
14
+ # bundle.dictionary # => #<Dictionary::Hunspell ...>
15
+ #
16
+ # The hot path (Kotoshu.correct?, .check, .suggest, .spellchecker_for) calls
17
+ # resolve and lets ResourceNotSetupError propagate. Setup is never implicit.
18
+ class ResourceManager
19
+ # Default `want:` value: spelling only. Everything else (frequency,
20
+ # model) is an explicit opt-in.
21
+ DEFAULT_WANT = %i[spelling].freeze
22
+
23
+ # The Stage-1 result: per-resource status for one language's
24
+ # setup run. A resource that was not requested is nil; one that
25
+ # was requested but failed is :unavailable.
26
+ #
27
+ # Statuses: :downloaded | :local | :cached | :unavailable | nil.
28
+ # The most common ones, as a quick reference:
29
+ #
30
+ # result = Kotoshu.setup(:en, want: %i[spelling frequency model])
31
+ # result.spelling # => :downloaded (or :cached on re-run)
32
+ # result.frequency # => :downloaded | :unavailable (soft dep)
33
+ # result.model # => :downloaded | :cached | :unavailable
34
+ # result.model_tier # => :full | :fluency | :mini | nil
35
+ # result.source # => :kotoshu | :local
36
+ #
37
+ # See also {ResourceBundle} — the Stage-2 result.
38
+ SetupResult = Struct.new(
39
+ :language, # [String] Language code that was set up
40
+ :spelling, # :downloaded | :local | :cached | nil
41
+ :frequency, # :downloaded | :local | :cached | :unavailable | nil
42
+ :model, # :downloaded | :cached | :unavailable | nil
43
+ :model_tier, # :full | :fluency | :mini | nil (tier actually set up)
44
+ :source, # :kotoshu | :local
45
+ keyword_init: true
46
+ ) do
47
+ # Whether the setup run produced anything usable: at least one
48
+ # of spelling/frequency succeeded. A nil everywhere means the
49
+ # run failed outright (network, integrity) — with `strict:`,
50
+ # such failures raise instead.
51
+ #
52
+ # @return [Boolean]
53
+ def success?
54
+ !spelling.nil? || !frequency.nil?
55
+ end
56
+ end
57
+
58
+ class << self
59
+ # Stage 1, class-level convenience. Delegates to a new instance;
60
+ # see the instance method for parameters and semantics.
61
+ #
62
+ # @param language [String, Symbol]
63
+ # @param want [Array<Symbol>] Resource types (default: [:spelling])
64
+ # @param opts [Hash] Forwarded to {#setup} (force:, strict:,
65
+ # tier:, aff:, dic:, from:, frequency:)
66
+ # @return [SetupResult]
67
+ # @see #setup
68
+ # @see Kotoshu.setup
69
+ def setup(language, want: DEFAULT_WANT, force: false, strict: false, **opts)
70
+ new.setup(language: language, want: want, force: force, strict: strict, **opts)
71
+ end
72
+
73
+ # Stage 1 from local files, class-level convenience. Delegates to
74
+ # a new instance; see the instance method.
75
+ #
76
+ # @param language [String, Symbol]
77
+ # @param aff [String] Path to the local .aff file
78
+ # @param dic [String] Path to the local .dic file
79
+ # @param frequency [String, nil] Optional path to frequency.json
80
+ # @param force [Boolean] Re-register even if already cached
81
+ # @return [SetupResult]
82
+ # @raise [ArgumentError] when a file is missing
83
+ # @see #setup_from_local
84
+ def setup_from_local(language:, aff:, dic:, frequency: nil, force: false)
85
+ new.setup_from_local(language: language, aff: aff, dic: dic, frequency: frequency, force: force)
86
+ end
87
+
88
+ # Stage 2, class-level convenience. Delegates to a new instance;
89
+ # see the instance method for the tier contract.
90
+ #
91
+ # @param language [String, Symbol]
92
+ # @param want [Array<Symbol>] Resource types (default: [:spelling])
93
+ # @param tier [Symbol, nil] Model tier to resolve
94
+ # @return [ResourceBundle]
95
+ # @raise [ResourceNotSetupError] when a wanted resource is not cached
96
+ # @see #resolve
97
+ def resolve(language:, want: DEFAULT_WANT, tier: nil)
98
+ new.resolve(language: language, want: want, tier: tier)
99
+ end
100
+
101
+ # Predicate, class-level convenience. See {#setup?}.
102
+ #
103
+ # @param language [String, Symbol]
104
+ # @param resource [Symbol, nil] :spelling, :frequency, :model, nil for any
105
+ # @param tier [Symbol, nil] Model tier to check (:any = any tier)
106
+ # @return [Boolean]
107
+ def setup?(language, resource: nil, tier: nil)
108
+ new.setup?(language, resource: resource, tier: tier)
109
+ end
110
+
111
+ # Languages with a cached spelling dictionary, sorted.
112
+ #
113
+ # @return [Array<String>]
114
+ # @see Kotoshu.languages_setup
115
+ def languages_setup
116
+ new.languages_setup
117
+ end
118
+ end
119
+
120
+ # ---- Stage 1: setup ----
121
+
122
+ # Download or register resources for one language (slow,
123
+ # network-required when remote, explicit). Idempotent: re-running
124
+ # with the same args is a no-op unless `force:` is true.
125
+ #
126
+ # Pass `aff:`/`dic:` or `from:` to register local Hunspell files
127
+ # instead of downloading (single-language only). See
128
+ # {Kotoshu.setup} — the facade takes the same options and fans
129
+ # them out over multiple languages.
130
+ #
131
+ # @param language [String, Symbol] Language code
132
+ # @param want [Array<Symbol>] Resource types (default: [:spelling])
133
+ # @param force [Boolean] Re-fetch even if already cached
134
+ # @param strict [Boolean] Re-raise on optional-resource failure
135
+ # (frequency/model); without it, failures degrade to
136
+ # :unavailable with a warning
137
+ # @param tier [Symbol, String, nil] Model tier when want includes
138
+ # :model (:full default, :fluency, :mini)
139
+ # @param aff [String, nil] Path to local .aff file
140
+ # @param dic [String, nil] Path to local .dic file
141
+ # @param from [String, nil] Directory containing lang.aff/lang.dic
142
+ # @param frequency [String, nil] Path to local frequency.json
143
+ # @return [SetupResult]
144
+ # @raise [ArgumentError] when local files are named but missing
145
+ def setup(language:, want: DEFAULT_WANT, force: false, strict: false, tier: nil,
146
+ aff: nil, dic: nil, from: nil, frequency: nil)
147
+ lang = normalize_language(language)
148
+
149
+ if aff || dic || from
150
+ setup_from_local(language: lang, aff: aff, dic: dic, from: from,
151
+ frequency: frequency, force: force)
152
+ else
153
+ setup_from_remote(lang, want: want, force: force, strict: strict, tier: tier)
154
+ end
155
+ end
156
+
157
+ # Register local Hunspell files as the language's cached spelling
158
+ # dictionary (no network). `from:` looks for `lang.aff` and
159
+ # `lang.dic` inside the directory; explicit `aff:`/`dic:` paths win.
160
+ #
161
+ # @param language [String, Symbol] Language code
162
+ # @param aff [String, nil] Path to the .aff file
163
+ # @param dic [String, nil] Path to the .dic file
164
+ # @param from [String, nil] Directory to find lang.aff/lang.dic in
165
+ # @param frequency [String, nil] Optional path to frequency.json
166
+ # @param force [Boolean] Re-register even if already cached
167
+ # @return [SetupResult]
168
+ # @raise [ArgumentError] when a file is missing
169
+ #
170
+ # @example
171
+ # ResourceManager.setup_from_local(
172
+ # language: "en",
173
+ # aff: "/usr/share/hunspell/en_US.aff",
174
+ # dic: "/usr/share/hunspell/en_US.dic"
175
+ # ).source # => :local
176
+ def setup_from_local(language:, aff:, dic:, from: nil, frequency: nil, force: false)
177
+ lang = normalize_language(language)
178
+
179
+ aff_path, dic_path = resolve_local_paths(lang, aff: aff, dic: dic, from: from)
180
+ raise ArgumentError, "aff file not found: #{aff_path}" unless File.exist?(aff_path)
181
+ raise ArgumentError, "dic file not found: #{dic_path}" unless File.exist?(dic_path)
182
+
183
+ spelling_cache = spelling_cache_for(lang)
184
+ spelling_cache.install_local(lang, aff: aff_path, dic: dic_path, force: force)
185
+ spelling_status = :local
186
+
187
+ frequency_status = nil
188
+ if frequency
189
+ raise ArgumentError, "frequency file not found: #{frequency}" unless File.exist?(frequency)
190
+
191
+ freq_cache = frequency_cache_for
192
+ freq_cache.install_local(lang, path: frequency, force: force)
193
+ frequency_status = :local
194
+ end
195
+
196
+ SetupResult.new(
197
+ language: lang,
198
+ spelling: spelling_status,
199
+ frequency: frequency_status,
200
+ model: nil,
201
+ source: :local
202
+ )
203
+ end
204
+
205
+ # ---- Stage 2: resolve (cache-only) ----
206
+ #
207
+ # `tier:` selects which cached model tier to resolve. Default: the
208
+ # configured `model_tier` ("fluency" — owner decision 2026-09-04).
209
+ # A missing tier raises ResourceNotSetupError exactly like any
210
+ # other unset resource — resolve never downloads.
211
+ #
212
+ # Tier-less fallback (legacy-cache bridge): when `tier:` is NOT
213
+ # given and the configured tier is not cached, exactly one cached
214
+ # tier satisfies the request. Caches written before the fluency
215
+ # default hold the legacy `full` layout; without this bridge a
216
+ # tier-less resolve would raise for them. An explicit `tier:`
217
+ # never falls back, and zero or multiple cached tiers raise the
218
+ # missing-tier error for the configured tier.
219
+ #
220
+ # `tier: :any` (explicit opt-in only) maps to the single cached
221
+ # tier; with zero or multiple cached tiers it raises.
222
+ #
223
+ # @param language [String, Symbol] Language code
224
+ # @param want [Array<Symbol>] Resource types to resolve
225
+ # (default: [:spelling])
226
+ # @param tier [Symbol, nil] Model tier to resolve (:full, :fluency,
227
+ # :mini, or :any for "the single cached tier")
228
+ # @return [ResourceBundle] Bundle with a member per resolved
229
+ # resource; unrequested resources are nil
230
+ # @raise [ResourceNotSetupError] when a wanted resource is not
231
+ # cached — resolve never downloads
232
+ # @raise [ResourceResolutionError] when multiple model tiers are
233
+ # cached and `tier: :any` cannot disambiguate
234
+ def resolve(language:, want: DEFAULT_WANT, tier: nil)
235
+ lang = normalize_language(language)
236
+ effective_tier = effective_tier(tier)
237
+
238
+ spelling_dict = want.include?(:spelling) ? resolve_spelling_cached(lang) : nil
239
+ frequency_data = want.include?(:frequency) ? resolve_frequency_cached(lang) : nil
240
+ model = if want.include?(:model)
241
+ resolve_model_cached(lang, tier: effective_tier, from_default: tier.nil?)
242
+ else
243
+ nil
244
+ end
245
+
246
+ ResourceBundle.new(
247
+ language: lang,
248
+ dictionary: spelling_dict,
249
+ frequency: frequency_data,
250
+ model: model,
251
+ rules: nil,
252
+ cached: true,
253
+ source_urls: []
254
+ )
255
+ end
256
+
257
+ # ---- Predicates ----
258
+
259
+ # Whether a language (or a specific resource for it) is already
260
+ # cached and resolvable. Mirrors the tier-less resolve contract
261
+ # for models: with no `tier:`, the configured tier counts plus a
262
+ # single cached tier of any kind.
263
+ #
264
+ # @param language [String, Symbol] Language code
265
+ # @param resource [Symbol, nil] :spelling (default), :frequency,
266
+ # :model, or nil for "spelling"
267
+ # @param tier [Symbol, nil] Model tier to check (:full, :fluency,
268
+ # :mini, or :any); only meaningful with resource: :model
269
+ # @return [Boolean]
270
+ #
271
+ # @example
272
+ # ResourceManager.setup?(:en) # spelling
273
+ # ResourceManager.setup?(:en, resource: :frequency) # Kelly list
274
+ # ResourceManager.setup?(:en, resource: :model, tier: :mini)
275
+ def setup?(language, resource: nil, tier: nil)
276
+ lang = normalize_language(language)
277
+ case resource&.to_sym
278
+ when nil, :spelling
279
+ spelling_cache_for(lang).available?("#{lang}:spelling")
280
+ when :frequency
281
+ fc = frequency_cache_for
282
+ fc.supports_resource?(lang) && fc.available?(lang)
283
+ when :model
284
+ cache = model_cache_for
285
+ if tier.nil?
286
+ # Mirrors the tier-less resolve contract: the configured
287
+ # tier counts, and so does exactly one cached tier of any
288
+ # kind (legacy single-tier caches).
289
+ cache.available?(cache.tier_resource_id(lang, effective_tier(nil))) ||
290
+ cache.cached_tiers(lang).size == 1
291
+ elsif tier.to_sym == :any
292
+ cache.cached_tiers(lang).any?
293
+ else
294
+ cache.available?(cache.tier_resource_id(lang, tier))
295
+ end
296
+ else
297
+ false
298
+ end
299
+ end
300
+
301
+ # Languages with a cached spelling dictionary, sorted.
302
+ #
303
+ # @return [Array<String>] e.g. ["de", "en", "fr"]
304
+ def languages_setup
305
+ spelling_cache_for(nil).cached_resources
306
+ .map { |r| r.to_s.split(":").first }
307
+ .uniq
308
+ .sort
309
+ end
310
+
311
+ private
312
+
313
+ def setup_from_remote(lang, want:, force:, strict:, tier: nil)
314
+ config = Configuration.instance
315
+ spelling_status = nil
316
+ frequency_status = nil
317
+ model_status = nil
318
+ model_tier = nil
319
+
320
+ if want.include?(:spelling)
321
+ cache = spelling_cache_for(lang, config: config)
322
+ was_cached = cache.available?("#{lang}:spelling")
323
+ if was_cached && !force
324
+ spelling_status = :cached
325
+ else
326
+ warn "[#{lang}] downloading spelling dictionary..." unless quiet?
327
+ cache.get_spelling(lang, force_download: force)
328
+ spelling_status = :downloaded
329
+ end
330
+ end
331
+
332
+ if want.include?(:frequency)
333
+ frequency_status = setup_frequency_remote(lang, force: force, strict: strict, config: config)
334
+ end
335
+
336
+ if want.include?(:model)
337
+ model_tier = effective_tier(tier, config: config)
338
+ model_status = setup_model_remote(lang, want: want, force: force, strict: strict, config: config,
339
+ tier: model_tier, from_default: tier.nil?)
340
+ if model_status == :cached
341
+ model_tier = setup_model_remote_satisfied_tier(lang, config: config, requested: model_tier,
342
+ from_default: tier.nil?)
343
+ end
344
+ end
345
+
346
+ SetupResult.new(
347
+ language: lang,
348
+ spelling: spelling_status,
349
+ frequency: frequency_status,
350
+ model: model_status,
351
+ model_tier: model_tier,
352
+ source: :kotoshu
353
+ )
354
+ end
355
+
356
+ def setup_frequency_remote(lang, force:, strict:, config:)
357
+ cache = frequency_cache_for(config: config)
358
+ return :unavailable unless cache.supports_resource?(lang)
359
+
360
+ was_cached = cache.available?(lang)
361
+ return :cached if was_cached && !force
362
+
363
+ warn "[#{lang}] downloading frequency data..." unless quiet?
364
+ cache.get(lang, force_download: force)
365
+ :downloaded
366
+ rescue StandardError => e
367
+ raise if strict
368
+
369
+ warn "[#{lang}] frequency data unavailable: #{e.class} (#{e.message})" unless quiet?
370
+ :unavailable
371
+ end
372
+
373
+ # The tier a :cached status actually satisfied: the requested tier
374
+ # when it is cached, else the single cached tier the tier-less
375
+ # bridge matched. SetupResult reports the truth, not the request.
376
+ def setup_model_remote_satisfied_tier(lang, config:, requested:, from_default:)
377
+ cache = model_cache_for(config: config)
378
+ requested = Cache::ModelCache.normalize_tier(requested)
379
+ return requested if cache.available?(cache.tier_resource_id(lang, requested))
380
+
381
+ cached = cache.cached_tiers(lang) if from_default
382
+ cached&.size == 1 ? cached.first : requested
383
+ end
384
+
385
+ # Set up the ONNX model for `lang` at `tier`.
386
+ #
387
+ # full: today's behavior, byte for byte — AVAILABLE_MODELS gate,
388
+ # git-tree URL, download-or-convert fallback. fluency/mini: the
389
+ # registry-driven path with sha256 verification against the
390
+ # registry entry. In both cases a cached model short-circuits to
391
+ # :cached BEFORE any registry consultation, so offline hosts with
392
+ # a warm cache (but no cached registry) still work.
393
+ def setup_model_remote(lang, want:, force:, strict:, config:, tier: :full, from_default: false)
394
+ tier = Cache::ModelCache.normalize_tier(tier)
395
+ cache = model_cache_for(config: config)
396
+ resource_id = cache.tier_resource_id(lang, tier)
397
+
398
+ if tier == :full && !Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym)
399
+ return :unavailable
400
+ end
401
+
402
+ was_cached = cache.available?(resource_id)
403
+ return :cached if was_cached && !force
404
+
405
+ # Tier-less bridge (mirrors resolve_model_cached): a cache written
406
+ # before the fluency default holds only the legacy full layout;
407
+ # its owner never named a tier. Exactly one cached tier satisfies
408
+ # the request; explicit tiers never fall back.
409
+ if from_default && !force
410
+ cached_tiers = cache.cached_tiers(lang)
411
+ if cached_tiers.size == 1
412
+ legacy_id = cache.tier_resource_id(lang, cached_tiers.first)
413
+ return :cached if cache.available?(legacy_id)
414
+ end
415
+ end
416
+
417
+ if tier != :full && cache.registry_entry_for(lang, tier).nil?
418
+ warn "[#{lang}] no registry entry for tier #{tier}; available: #{tiers_for_language(cache, lang).join(', ')}" unless quiet?
419
+ return :unavailable
420
+ end
421
+
422
+ warn "[#{lang}] downloading ONNX model (#{tier} tier)..." unless quiet?
423
+ if tier == :full
424
+ cache.get(resource_id, force_download: force)
425
+ else
426
+ cache.download_tiered_model(lang, tier: tier, force_download: force)
427
+ end
428
+ :downloaded
429
+ rescue StandardError => e
430
+ raise if strict
431
+
432
+ warn "[#{lang}] ONNX model unavailable: #{e.class} (#{e.message})" unless quiet?
433
+ :unavailable
434
+ end
435
+
436
+ def resolve_spelling_cached(lang)
437
+ cache = spelling_cache_for(lang)
438
+ resource_id = "#{lang}:spelling"
439
+ raise ResourceNotSetupError.new(lang, "spelling") unless cache.available?(resource_id)
440
+
441
+ # Cache-only by construction: resolve must never download, so read
442
+ # via load_cached instead of the download-on-miss get.
443
+ result = cache.load_cached(resource_id)
444
+ raise ResourceNotSetupError.new(lang, "spelling") unless result
445
+
446
+ Dictionary::Hunspell.new(
447
+ dic_path: result[:dic_path] || result["dic_path"],
448
+ aff_path: result[:aff_path] || result["aff_path"],
449
+ language_code: lang
450
+ )
451
+ end
452
+
453
+ def resolve_frequency_cached(lang)
454
+ cache = frequency_cache_for
455
+ return nil unless cache.supports_resource?(lang)
456
+ raise ResourceNotSetupError.new(lang, "frequency") unless cache.available?(lang)
457
+
458
+ begin
459
+ cache.load_cached(lang)
460
+ rescue StandardError
461
+ nil
462
+ end
463
+ end
464
+
465
+ # Resolve the cached model for `lang` at `tier` (cache-only —
466
+ # never downloads, never consults the network-side registry).
467
+ #
468
+ # :any maps to the single cached tier; zero or multiple cached
469
+ # tiers raise (deterministic: ambiguity errors list tiers in
470
+ # Cache::ModelCache::TIER_PREFERENCE order — mini, fluency, full —
471
+ # which is a reporting order, not a fallback chain).
472
+ #
473
+ # `from_default` marks a tier-less call (the tier came from the
474
+ # configuration, not an argument). For those calls only, a missing
475
+ # configured tier falls back to the single cached tier: caches
476
+ # written before the fluency default (owner decision 2026-09-04)
477
+ # hold the legacy full layout, and their owners never asked for a
478
+ # tier by name. Zero or multiple cached tiers keep the normal
479
+ # missing-tier error. Explicit tiers and :any never fall back.
480
+ def resolve_model_cached(lang, tier: :full, from_default: false)
481
+ cache = model_cache_for
482
+ return nil unless model_language_supported?(lang, cache)
483
+
484
+ if tier.to_sym == :any
485
+ cached = cache.cached_tiers(lang)
486
+ case cached.size
487
+ when 1
488
+ return resolve_tier_cached(lang, cached.first, cache)
489
+ when 0
490
+ raise ResourceNotSetupError.new(lang, "model (no tier cached)")
491
+ else
492
+ raise ResourceResolutionError.new(
493
+ lang,
494
+ "multiple model tiers cached (#{cached.join(', ')}); " \
495
+ "pass tier: one of #{cached.join('|')} explicitly"
496
+ )
497
+ end
498
+ end
499
+
500
+ requested = Cache::ModelCache.normalize_tier(tier)
501
+ return resolve_tier_cached(lang, requested, cache) if cache.available?(cache.tier_resource_id(lang, requested))
502
+
503
+ if from_default
504
+ cached = cache.cached_tiers(lang)
505
+ return resolve_tier_cached(lang, cached.first, cache) if cached.size == 1
506
+ end
507
+
508
+ resolve_tier_cached(lang, requested, cache)
509
+ end
510
+
511
+ def resolve_tier_cached(lang, tier, cache)
512
+ resource_id = cache.tier_resource_id(lang, tier)
513
+ missing = tier == :full ? "model" : "model tier '#{tier}'"
514
+ raise ResourceNotSetupError.new(lang, missing) unless cache.available?(resource_id)
515
+
516
+ begin
517
+ cache.load_cached(resource_id)
518
+ rescue StandardError
519
+ nil
520
+ end
521
+ end
522
+
523
+ # Whether `lang` can have a model at all: listed in AVAILABLE_MODELS
524
+ # (full-tier gate, unchanged) or any tier already cached on disk.
525
+ def model_language_supported?(lang, cache)
526
+ Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym) ||
527
+ cache.cached_tiers(lang).any?
528
+ end
529
+
530
+ # The tier a tier-less call means: an explicit argument wins, then
531
+ # the configured `model_tier` (default "fluency" — owner decision
532
+ # 2026-09-04; "full" and "mini" are explicit opt-ins).
533
+ def effective_tier(tier, config: nil)
534
+ return :any if tier && tier.to_sym == :any
535
+
536
+ Cache::ModelCache.normalize_tier(tier || (config || Configuration.instance).model_tier)
537
+ end
538
+
539
+ # Registry tiers available for a language (for diagnostics; may
540
+ # consult the cached registry, never used on the resolve path).
541
+ def tiers_for_language(cache, lang)
542
+ Cache::ModelCache::TIERS.select do |t|
543
+ t == :full ? Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym) : cache.registry_entry_for(lang, t)
544
+ end
545
+ end
546
+
547
+ def resolve_local_paths(lang, aff:, dic:, from:)
548
+ if from
549
+ dir = File.expand_path(from)
550
+ aff_path = aff || File.join(dir, "#{lang}.aff")
551
+ dic_path = dic || File.join(dir, "#{lang}.dic")
552
+ [aff_path, dic_path]
553
+ else
554
+ [File.expand_path(aff), File.expand_path(dic)]
555
+ end
556
+ end
557
+
558
+ # Collapse a user-given code to its cache key: lowercase base
559
+ # language, plus the script subtag when one is present. Region
560
+ # variants share their base dictionary (en-GB -> en, sv-FI -> sv),
561
+ # but a BCP-47 script subtag (four letters: Latn in sr-Latn and
562
+ # tlh-Latn) names a dictionary the manifest stages separately, so
563
+ # it survives normalization (plan 110).
564
+ def normalize_language(code)
565
+ parts = code.to_s.split(/[-_]/)
566
+ base = parts.first.downcase
567
+ subtag = parts[1]
568
+ return base unless subtag&.match?(/\A[A-Za-z]{4}\z/)
569
+
570
+ "#{base}-#{subtag.downcase.capitalize}"
571
+ end
572
+
573
+ def spelling_cache_for(_lang = nil, config: nil)
574
+ cfg = config || Configuration.instance
575
+ Cache::LanguageCache.new(
576
+ cache_path: cfg.cache_path,
577
+ resource_pin: cfg.resource_pin
578
+ )
579
+ end
580
+
581
+ def frequency_cache_for(config: nil)
582
+ cfg = config || Configuration.instance
583
+ Cache::FrequencyCache.new(
584
+ cache_path: cfg.cache_path,
585
+ resource_pin: cfg.resource_pin
586
+ )
587
+ end
588
+
589
+ def model_cache_for(config: nil)
590
+ cfg = config || Configuration.instance
591
+ Cache::ModelCache.new(
592
+ cache_path: cfg.cache_path
593
+ )
594
+ end
595
+
596
+ def quiet?
597
+ !$stderr.tty? || ENV["KOTOSHU_QUIET"] == "1"
598
+ end
599
+ end
600
+ end