kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "lutaml/model"
4
+
5
+ module Kotoshu
6
+ module Cache
7
+ # Parsed view of `registry.json` from kotoshu/models-fasttext-onnx
8
+ # (Resource Spec v1, TODO.impl/07 of that repo; gem side: plan 67 M1).
9
+ #
10
+ # Shape:
11
+ #
12
+ # {
13
+ # "spec": "kotoshu.resources/v1",
14
+ # "registry_version": 1,
15
+ # "generated_at": "2026-09-02T22:16:05Z",
16
+ # "release_tag": "v1.0.0",
17
+ # "resources": {
18
+ # "kotoshu://models/en/mini": {
19
+ # "type": "model",
20
+ # "language": "en",
21
+ # "tier": { "name": "mini", "dims": 300, ... },
22
+ # "version": "1.0.0",
23
+ # "urls": { "primary": "...", "mirror": "..." },
24
+ # "vocab_url": "...",
25
+ # "sha256": "...", "size_bytes": 3040752,
26
+ # "license": "CC-BY-SA-3.0",
27
+ # "min_engine_version": "0.7",
28
+ # "eval_ref": "eval/reports/en.mini.json"
29
+ # }
30
+ # }
31
+ # }
32
+ #
33
+ # Ids are stable forever (`kotoshu://models/{lang}/{tier}`); content
34
+ # changes bump `version` and the release tag, never the id.
35
+ #
36
+ # Serialized via lutaml-model: use {ModelRegistry.from_json} /
37
+ # {Resource.from_hash} for the wire forms — no hand-rolled
38
+ # `to_h` / `from_h` on the model (see CLAUDE.md serialization rule).
39
+ class ModelRegistry < Lutaml::Model::Serializable
40
+ # One downloadable model artifact, keyed by its stable
41
+ # `kotoshu://models/{lang}/{tier}` id in the parent registry.
42
+ class Resource < Lutaml::Model::Serializable
43
+ # Download URLs. `primary` is the GitHub Release asset;
44
+ # `mirror` points at the git tree and is a fallback only.
45
+ class Urls < Lutaml::Model::Serializable
46
+ attribute :primary, :string
47
+ attribute :mirror, :string
48
+ end
49
+
50
+ # Tier metadata block. `quantization` is null for the full tier.
51
+ class Tier < Lutaml::Model::Serializable
52
+ attribute :name, :string
53
+ attribute :dims, :integer
54
+ attribute :vocab_size, :integer
55
+ attribute :quantization, :string
56
+ end
57
+
58
+ attribute :type, :string
59
+ attribute :language, :string
60
+ attribute :tier, Tier
61
+ attribute :version, :string
62
+ attribute :urls, Urls
63
+ attribute :vocab_url, :string
64
+ attribute :sha256, :string
65
+ attribute :size_bytes, :integer
66
+ attribute :license, :string
67
+ attribute :min_engine_version, :string
68
+ attribute :eval_ref, :string
69
+
70
+ # Stable registry id for a (language, tier) pair. This is the key
71
+ # under which the resource appears in {ModelRegistry#resources}.
72
+ #
73
+ # @param language [String] ISO 639-1 code
74
+ # @param tier [String, Symbol] tier name ("mini", "fluency", "full")
75
+ # @return [String] e.g. "kotoshu://models/en/mini"
76
+ def self.id_for(language, tier)
77
+ "kotoshu://models/#{language}/#{tier}"
78
+ end
79
+ end
80
+
81
+ attribute :spec, :string
82
+ attribute :registry_version, :integer
83
+ attribute :generated_at, :string
84
+ attribute :release_tag, :string
85
+ # Id => per-resource hash. Values are deserialized on lookup via
86
+ # {Resource.from_hash} (the framework API); lutaml-model 0.8 does
87
+ # not support typed values inside a `:hash` attribute.
88
+ attribute :resources, :hash
89
+
90
+ # Typed lookup of a model resource by language and tier.
91
+ #
92
+ # @param language [String] ISO 639-1 code
93
+ # @param tier [String, Symbol] tier name
94
+ # @return [Resource, nil] the entry, or nil when the registry has
95
+ # no such (language, tier) combination
96
+ def find(language, tier)
97
+ raw = resources[Resource.id_for(language, tier)]
98
+ raw && Resource.from_hash(raw)
99
+ end
100
+
101
+ # All languages the registry knows about (derived from ids).
102
+ #
103
+ # @return [Array<String>] sorted, unique language codes
104
+ def languages
105
+ resources.keys.map { |id| id.split("/")[3] }.compact.uniq.sort
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Cache
5
+ # Offline-safe snapshot of the languages staged in the
6
+ # kotoshu/dictionaries manifest (plan 107).
7
+ #
8
+ # `manifest.json` in the dictionaries repo is the single source of
9
+ # truth for what `kotoshu setup` can download; this module vendors
10
+ # its language codes so the hot path never needs the network to
11
+ # answer "is this language downloadable?" (resolve is cache-only
12
+ # by contract, and the spelling gate runs inside #available?).
13
+ #
14
+ # The list is regenerated by
15
+ #
16
+ # rake kotoshu:staged_languages:sync
17
+ #
18
+ # which fetches the manifest at the SourceRegistry pin, extracts
19
+ # the distinct language codes from its resources, and rewrites
20
+ # CODES in place. Edit the dictionaries repo, not this file.
21
+ #
22
+ # Two tiers of support (plan 107 vocabulary):
23
+ # - full feature: a language module under Kotoshu::Languages
24
+ # (keyboard layout + verified specimens + tokenizer care) —
25
+ # see LanguageCache.full_feature_languages;
26
+ # - basic: manifest-present but module-less — the script fallback
27
+ # tokenizer and generic keyboard still serve check/suggest.
28
+ module StagedLanguages
29
+ # Language codes carried by the dictionaries manifest at its
30
+ # SourceRegistry pin; refreshed by the sync rake task above.
31
+ CODES = %w[
32
+ ar bg br ca ca-valencia cs cy da de de-AT
33
+ de-CH el el-polyton en en-AU en-CA en-GB en-ZA eo es
34
+ es-AR es-BO es-CL es-CO es-CR es-CU es-DO es-EC es-GT es-HN
35
+ es-MX es-NI es-PA es-PE es-PH es-PR es-PY es-SV es-US es-UY
36
+ es-VE et eu fa fo fr fur fy ga gd
37
+ gl he hr hu hy hyw ia id ie is
38
+ it ka ko la lb lt ltg lv mk mn
39
+ nb nds ne nl nn oc pl pt pt-BR pt-PT
40
+ ro ru rw sk sl sr sr-Latn sv sv-FI tk
41
+ tlh tlh-Latn tr uk vi
42
+ ].freeze
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,113 @@
1
+ # frozen_string_literal: true
2
+
3
+ # LookupCache autoloaded via Kotoshu::Cache
4
+
5
+ module Kotoshu
6
+ module Cache
7
+ # LRU cache specifically for suggestion results.
8
+ #
9
+ # Extends LookupCache with suggestion-specific features like
10
+ # caching by word + max_results combination.
11
+ #
12
+ # @example Caching suggestions
13
+ # cache = SuggestionCache.new(max_size: 5000)
14
+ # cache.write("helo", ["hello", "help"], max_results: 10)
15
+ # cache.read("helo", max_results: 10) # => ["hello", "help"]
16
+ class SuggestionCache < LookupCache
17
+ # Default maximum cache size for suggestions
18
+ DEFAULT_MAX_SIZE = 5000
19
+
20
+ # Create a new suggestion cache.
21
+ #
22
+ # @param max_size [Integer] Maximum number of entries (default: 5000)
23
+ def initialize(max_size: DEFAULT_MAX_SIZE)
24
+ super
25
+ end
26
+
27
+ # Write suggestions to cache.
28
+ #
29
+ # @param word [String] The misspelled word
30
+ # @param suggestions [Array<String>] Suggested words
31
+ # @param max_results [Integer] Max results used for this query
32
+ # @return [Array<String>] The stored suggestions
33
+ def write(word, suggestions, max_results: 10)
34
+ cache_key = cache_key_for(word, max_results)
35
+ super(cache_key, suggestions)
36
+ end
37
+
38
+ # Read suggestions from cache.
39
+ #
40
+ # @param word [String] The misspelled word
41
+ # @param max_results [Integer] Max results used for this query
42
+ # @return [Array<String>, nil] Cached suggestions or nil
43
+ def read(word, max_results: 10)
44
+ cache_key = cache_key_for(word, max_results)
45
+ super(cache_key)
46
+ end
47
+
48
+ # Fetch suggestions from cache or compute them.
49
+ #
50
+ # @param word [String] The misspelled word
51
+ # @param max_results [Integer] Max results for this query
52
+ # @yield Block to compute suggestions on cache miss
53
+ # @return [Array<String>] Cached or computed suggestions
54
+ def fetch(word, max_results: 10)
55
+ cache_key = cache_key_for(word, max_results)
56
+
57
+ if @data.key?(cache_key)
58
+ record_hit
59
+ @access_order += 1
60
+ @data[cache_key][1] = @access_order # Update access order
61
+ @data[cache_key][0] # Return value
62
+ else
63
+ record_miss
64
+ suggestions = yield
65
+ write(word, suggestions, max_results: max_results)
66
+ suggestions
67
+ end
68
+ end
69
+
70
+ # Delete suggestions from cache.
71
+ #
72
+ # @param word [String] The misspelled word
73
+ # @param max_results [Integer] Max results for this query
74
+ # @return [Array<String>, nil] Deleted suggestions or nil
75
+ def delete(word, max_results: 10)
76
+ cache_key = cache_key_for(word, max_results)
77
+ super(cache_key)
78
+ end
79
+
80
+ # Check if suggestions are cached for this word.
81
+ #
82
+ # @param word [String] The misspelled word
83
+ # @param max_results [Integer] Max results for this query
84
+ # @return [Boolean] True if cached
85
+ def key?(word, max_results: 10)
86
+ cache_key = cache_key_for(word, max_results)
87
+ super(cache_key)
88
+ end
89
+
90
+ # Invalidate all cached suggestions for a word.
91
+ #
92
+ # @param word [String] The word to invalidate
93
+ # @return [self] Self for chaining
94
+ def invalidate_word(word)
95
+ # Find and delete all cache entries for this word
96
+ keys_to_delete = @data.keys.select { |key| key.start_with?("#{word}|") }
97
+ keys_to_delete.each { |key| @data.delete(key) }
98
+ self
99
+ end
100
+
101
+ private
102
+
103
+ # Generate cache key for word + max_results.
104
+ #
105
+ # @param word [String] The word
106
+ # @param max_results [Integer] Max results
107
+ # @return [String] Cache key
108
+ def cache_key_for(word, max_results)
109
+ "#{word.downcase}|#{max_results}"
110
+ end
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Cache module for Kotoshu
5
+ #
6
+ # This module provides access to various cache implementations for
7
+ # dictionaries, models, and other resources.
8
+ #
9
+ # @example Using the language cache
10
+ # cache = Kotoshu::Cache::LanguageCache.new
11
+ # dict = cache.get_spelling('en')
12
+ # # => { dic_path: "~/.cache/kotoshu/languages/en/spelling/index.dic",
13
+ # # aff_path: "~/.cache/kotoshu/languages/en/spelling/index.aff",
14
+ # # metadata: { ... } }
15
+ #
16
+ module Cache
17
+ autoload :BaseCache, "kotoshu/cache/base_cache"
18
+ autoload :Cache, "kotoshu/cache/cache"
19
+ autoload :EvictionPolicy, "kotoshu/cache/eviction_policy"
20
+ autoload :FrequencyCache, "kotoshu/cache/frequency_cache"
21
+ autoload :LanguageCache, "kotoshu/cache/language_cache"
22
+ autoload :LookupCache, "kotoshu/cache/lookup_cache"
23
+ autoload :ModelCache, "kotoshu/cache/model_cache"
24
+ autoload :ModelRegistry, "kotoshu/cache/model_registry"
25
+ autoload :SuggestionCache, "kotoshu/cache/suggestion_cache"
26
+ # Vendored snapshot of the dictionaries manifest languages
27
+ # (plan 107); refreshed by kotoshu:staged_languages:sync.
28
+ autoload :StagedLanguages, "kotoshu/cache/staged_languages"
29
+
30
+ class << self
31
+ # Create a new language cache instance
32
+ #
33
+ # @param cache_path [String] optional custom cache directory
34
+ # @param url_base [String] optional custom GitHub URL
35
+ # @return [LanguageCache] new language cache instance
36
+ def language_cache(cache_path: nil, url_base: nil)
37
+ LanguageCache.new(cache_path: cache_path, url_base: url_base)
38
+ end
39
+
40
+ # Create a new model cache instance
41
+ #
42
+ # @param cache_path [String] optional custom cache directory
43
+ # @return [ModelCache] new model cache instance
44
+ def model_cache(cache_path: nil)
45
+ ModelCache.new(cache_path: cache_path)
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "io/console"
4
+
5
+ module Kotoshu
6
+ module Cli
7
+ # Interactive prompt that wraps the strict two-stage setup/resolve flow
8
+ # for the human-facing CLI.
9
+ #
10
+ # The library API (`Kotoshu.correct?`, `Kotoshu.suggest`) still raises
11
+ # `ResourceNotSetupError` strictly — no surprise downloads on metered
12
+ # networks. This class catches that error in the CLI dispatcher, asks
13
+ # the user once, and retries the original command. Programmatic users
14
+ # never see it.
15
+ #
16
+ # Non-TTY contexts (pipes, CI) and offline mode never prompt. The caller
17
+ # decides how to surface a nil result — the CLI dispatcher raises
18
+ # Errors::ResourceUnavailable so scripts see stable exit codes.
19
+ class AutoSetup
20
+ # @param input [IO] Stdin (or override for tests)
21
+ # @param output [IO] Stderr (or override for tests)
22
+ def initialize(input: $stdin, output: $stderr)
23
+ @input = input
24
+ @output = output
25
+ end
26
+
27
+ # Prompt the user to set up the missing language.
28
+ #
29
+ # @param error [Kotoshu::ResourceNotSetupError] The error raised by resolve
30
+ # @param want [Array<Symbol>] Resource types to fetch (default [:spelling])
31
+ # @return [String, nil] Language code on success; nil when non-TTY,
32
+ # offline, or user declined.
33
+ def call(error, want: %i[spelling])
34
+ language = error.language
35
+ return nil if skip_prompt?
36
+
37
+ @output.puts prompt_message(language, error.resource_type, want)
38
+ answer = @input.gets&.strip&.downcase
39
+ return nil unless affirmative?(answer)
40
+
41
+ Kotoshu.setup(language, want: want)
42
+ language
43
+ end
44
+
45
+ private
46
+
47
+ def skip_prompt?
48
+ Kotoshu.configuration.offline || !@input.tty?
49
+ end
50
+
51
+ def prompt_message(language, resource, want)
52
+ size_hint = size_hint_for(want)
53
+ "Language '#{language}' is not set up (missing #{resource}).\n" \
54
+ "Download now (~#{size_hint} from github.com/kotoshu/dictionaries)? [Y/n]"
55
+ end
56
+
57
+ def size_hint_for(want)
58
+ case want
59
+ when %i[spelling] then "5 MB"
60
+ when %i[spelling frequency] then "6 MB"
61
+ when %i[spelling frequency model] then "120 MB"
62
+ else "unknown size"
63
+ end
64
+ end
65
+
66
+ def affirmative?(answer)
67
+ answer.nil? || answer.empty? || answer.start_with?("y")
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ module Kotoshu
6
+ module Cli
7
+ # `kotoshu baseline` — CI baseline management (plan 82, Track B),
8
+ # wired as a subcommand in cli.rb.
9
+ class BaselineCommand < Thor
10
+ namespace :baseline
11
+
12
+ class_option :language,
13
+ type: :string,
14
+ default: "auto",
15
+ desc: "Language code (auto, de, en, es, fr, pt, ru)",
16
+ aliases: ["-l"]
17
+
18
+ desc "init FILE [FILE ...]", "Write a baseline of the current findings"
19
+ long_desc <<~DESC
20
+ Checks each FILE (directories and globs expand exactly like
21
+ `kotoshu check` directory mode; inline ignore directives apply)
22
+ and writes the canonical baseline JSON: one entry per (file,
23
+ word) with the misspelling count, stable across reformatting.
24
+ The personal dictionary is NOT consulted - baselines freeze what
25
+ every machine would see, and CI has no personal dictionary.
26
+ Suggestions are skipped: a baseline only needs positions.
27
+
28
+ The baseline only covers real errors present right now; new
29
+ errors still fail `kotoshu check --baseline`.
30
+
31
+ Examples:
32
+
33
+ kotoshu baseline init . # directories and globs expand like check
34
+ kotoshu baseline init src/*.txt --output .kotoshu-baseline.json
35
+ DESC
36
+ method_option :output,
37
+ type: :string,
38
+ default: Kotoshu::Baseline::Store::DEFAULT_FILENAME,
39
+ desc: "Baseline file to write (default: #{Kotoshu::Baseline::Store::DEFAULT_FILENAME})"
40
+ def init(*targets)
41
+ if targets.empty?
42
+ raise Errors::UsageError,
43
+ "Give at least one file: kotoshu baseline init FILE [FILE ...]"
44
+ end
45
+
46
+ missing = targets.reject do |target|
47
+ File.directory?(target) || File.file?(target) || !Dir[target].empty?
48
+ end
49
+ unless missing.empty?
50
+ raise Errors::UsageError, "File not found: #{missing.join(' ')}"
51
+ end
52
+
53
+ files = targets.flat_map { |t| expand_target(t) }
54
+ if files.empty?
55
+ raise Errors::UsageError,
56
+ "No text files found (known extensions: " \
57
+ "#{Cli::DirectoryWalker::TEXT_EXTENSIONS.join(' ')})"
58
+ end
59
+
60
+ checks = {}
61
+ files.sort.each do |file|
62
+ text = File.read(file, encoding: Kotoshu.configuration.encoding).scrub
63
+ checks[file] = [run_check(text), text]
64
+ end
65
+
66
+ store = Kotoshu::Baseline::Store.from_checks(checks)
67
+ store.save(options[:output])
68
+ total = store.entries.sum(&:count)
69
+ noun = store.entries.size == 1 ? "entry" : "entries"
70
+ error_noun = total == 1 ? "error" : "errors"
71
+ puts "Wrote #{options[:output]}: #{store.entries.size} #{noun} " \
72
+ "(#{total} #{error_noun} baselined)"
73
+ end
74
+
75
+ no_commands do
76
+ # Directories expand through the same walker directory mode uses
77
+ # (gitignore-aware, known text extensions); globs expand via
78
+ # Dir; plain files pass through unchanged.
79
+ def expand_target(target)
80
+ if File.directory?(target)
81
+ DirectoryWalker.new.files(target)
82
+ elsif File.file?(target)
83
+ [target]
84
+ else
85
+ Dir[target].select { |f| File.file?(f) }
86
+ end
87
+ end
88
+
89
+ # Baselines must be machine-independent: the personal dictionary
90
+ # of whichever machine runs init must never leak into the frozen
91
+ # set, or CI (which has none) flags words the baseline never
92
+ # recorded. Disabled for the whole init run.
93
+ def run_check(text)
94
+ previous = Kotoshu.configuration.personal_dictionary
95
+ Kotoshu.configuration.personal_dictionary = false
96
+ Kotoshu.reset_spellchecker
97
+ language = resolve_language(text)
98
+ Kotoshu.spellchecker_for(language).check(text, suggestions: false)
99
+ rescue Kotoshu::ResourceNotSetupError => e
100
+ raise Errors::ResourceUnavailable, e.message
101
+ rescue Kotoshu::DictionaryNotFoundError => e
102
+ raise Errors::ResourceUnavailable, e.message
103
+ ensure
104
+ Kotoshu.configuration.personal_dictionary = previous
105
+ Kotoshu.reset_spellchecker
106
+ end
107
+
108
+ def resolve_language(text)
109
+ result = LanguageResolver.new(
110
+ flag_value: options[:language],
111
+ default_language: Kotoshu.configuration.default_language
112
+ ).resolve(text: text)
113
+ warn "# #{result.note}" if result.note
114
+ result.language
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end