kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,321 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'csv'
5
+
6
+ module Kotoshu
7
+ module Cli
8
+ # Batch reporter for non-interactive error reporting.
9
+ #
10
+ # Outputs error reports in various formats (JSON, YAML, CSV, text).
11
+ # Used for automated checking and CI/CD integration.
12
+ #
13
+ # @example Generate JSON report
14
+ # reporter = BatchReporter.new(document, navigation)
15
+ # reporter.to_json('errors.json')
16
+ #
17
+ # @example Generate CSV report
18
+ # reporter.to_csv('errors.csv')
19
+ #
20
+ # @example Generate text summary
21
+ # puts reporter.to_text
22
+ class BatchReporter
23
+ attr_reader :document, :navigation, :formatter
24
+
25
+ # Display name lookup for document formats.
26
+ FORMAT_NAMES = {
27
+ text: 'Plain Text',
28
+ markdown: 'Markdown',
29
+ asciidoc: 'AsciiDoc',
30
+ code: 'Code'
31
+ }.freeze
32
+
33
+ # Create a new batch reporter.
34
+ #
35
+ # @param document [#name,#format,#language_code] Document being reported
36
+ # @param navigation [NavigationManager] Navigation state
37
+ # @param formatter [DisplayFormatter, nil] Display formatter
38
+ def initialize(document, navigation, formatter: nil)
39
+ @document = document
40
+ @navigation = navigation
41
+ @formatter = formatter || DisplayFormatter.new
42
+ end
43
+
44
+ # Generate JSON report.
45
+ #
46
+ # @param filepath [String] Output file path (optional, returns string if nil)
47
+ # @param pretty [Boolean] Pretty-print JSON (default: true)
48
+ # @return [String, nil] JSON string or nil if written to file
49
+ def to_json(filepath: nil, pretty: true)
50
+ data = generate_report_data
51
+ json = pretty ? JSON.pretty_generate(data) : JSON.generate(data)
52
+
53
+ if filepath
54
+ File.write(filepath, json)
55
+ nil
56
+ else
57
+ json
58
+ end
59
+ end
60
+
61
+ # Generate YAML report.
62
+ #
63
+ # @param filepath [String] Output file path (optional, returns string if nil)
64
+ # @return [String, nil] YAML string or nil if written to file
65
+ def to_yaml(filepath: nil)
66
+ require 'yaml'
67
+
68
+ data = generate_report_data
69
+ yaml = data.to_yaml
70
+
71
+ if filepath
72
+ File.write(filepath, yaml)
73
+ nil
74
+ else
75
+ yaml
76
+ end
77
+ end
78
+
79
+ # Generate CSV report.
80
+ #
81
+ # @param filepath [String] Output file path (optional, returns string if nil)
82
+ # @return [String, nil] CSV string or nil if written to file
83
+ def to_csv(filepath: nil)
84
+ csv_string = CSV.generate do |csv|
85
+ # Header
86
+ csv << ['ID', 'Line', 'Original', 'Suggestion', 'Confidence', 'Error Type']
87
+
88
+ # Data rows
89
+ @navigation.errors.each do |error|
90
+ suggestion = error.recommended_suggestion
91
+ csv << [
92
+ error.id,
93
+ error.location.line,
94
+ error.original,
95
+ suggestion&.word || '',
96
+ "#{(error.confidence * 100).round(1)}%",
97
+ error.error_type.to_s.capitalize
98
+ ]
99
+ end
100
+ end
101
+
102
+ if filepath
103
+ File.write(filepath, csv_string)
104
+ nil
105
+ else
106
+ csv_string
107
+ end
108
+ end
109
+
110
+ # Generate text summary.
111
+ #
112
+ # @return [String] Formatted text summary
113
+ def to_text
114
+ lines = []
115
+ lines << ""
116
+ lines << @formatter.colorize("╔═══════════════════════════════════════════════════════════════╗", :bold)
117
+ lines << @formatter.colorize("║ Batch Error Report ║", :bold)
118
+ lines << @formatter.colorize("╚═══════════════════════════════════════════════════════════════╝", :bold)
119
+ lines << ""
120
+ lines << "Document: #{@document.name}"
121
+ lines << "Format: #{FORMAT_NAMES[@document.format] || @document.format}"
122
+ lines << "Language: #{@document.language_code}"
123
+ lines << ""
124
+ lines << @formatter.colorize("Summary", :bold)
125
+ lines << ("─" * 70)
126
+
127
+ stats = @navigation.statistics
128
+ lines << "Total errors: #{stats[:total]}"
129
+ lines << " • High confidence (>0.8): #{stats[:by_confidence][:high]}"
130
+ lines << " • Medium confidence (0.5-0.8): #{stats[:by_confidence][:medium]}"
131
+ lines << " • Low confidence (≤0.5): #{stats[:by_confidence][:low]}"
132
+ lines << ""
133
+
134
+ # Breakdown by type
135
+ if stats[:by_type]&.any?
136
+ lines << @formatter.colorize("By Type", :bold)
137
+ stats[:by_type].each do |type, count|
138
+ label = Models::SemanticError::ERROR_TYPES[type] || type.to_s.capitalize
139
+ lines << " • #{label}: #{count}"
140
+ end
141
+ lines << ""
142
+ end
143
+
144
+ # Top errors
145
+ if @navigation.errors.any?
146
+ lines << @formatter.colorize("Top Errors", :bold)
147
+ lines << ("─" * 70)
148
+
149
+ @navigation.errors.first(10).each_with_index do |error, idx|
150
+ lines << "#{idx + 1}. [#{error.location}] #{error.original}"
151
+ lines << " Type: #{error.error_type}"
152
+ lines << " Confidence: #{(error.confidence * 100).round(1)}%"
153
+
154
+ if error.suggestions&.any?
155
+ top_suggestion = error.suggestions.first
156
+ lines << " Suggestion: #{top_suggestion.word} (#{(top_suggestion.confidence * 100).round(0)}%)"
157
+ end
158
+
159
+ lines << ""
160
+ end
161
+
162
+ if @navigation.errors.size > 10
163
+ lines << "... and #{@navigation.errors.size - 10} more"
164
+ lines << ""
165
+ end
166
+ end
167
+
168
+ lines.join("\n")
169
+ end
170
+
171
+ # Generate SARIF report (Static Analysis Results Interchange Format).
172
+ #
173
+ # SARIF is a standard format for static analysis tools.
174
+ # Useful for CI/CD integration and IDE integration.
175
+ #
176
+ # @param filepath [String] Output file path (optional, returns string if nil)
177
+ # @return [String, nil] SARIF JSON string or nil if written to file
178
+ def to_sarif(filepath: nil)
179
+ sarif = {
180
+ version: "2.1.0",
181
+ "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
182
+ runs: [
183
+ {
184
+ tool: {
185
+ driver: {
186
+ name: "Kotoshu",
187
+ version: Kotoshu::VERSION,
188
+ informationUri: "https://github.com/kotoshu/kotoshu",
189
+ rules: []
190
+ }
191
+ },
192
+ results: @navigation.errors.map do |error|
193
+ {
194
+ ruleId: error.error_type.to_s,
195
+ level: error.high_confidence? ? "error" : "warning",
196
+ message: {
197
+ text: "Potential #{error.error_type} error: '#{error.original}'"
198
+ },
199
+ locations: [
200
+ {
201
+ physicalLocation: {
202
+ artifactLocation: {
203
+ uri: @document.name
204
+ },
205
+ region: {
206
+ startLine: error.location.line || 1,
207
+ startColumn: error.location.column || 0
208
+ }
209
+ }
210
+ }
211
+ ],
212
+ suggestions: error.suggestions&.map do |sugg|
213
+ {
214
+ text: sugg.word
215
+ }
216
+ end
217
+ }
218
+ end
219
+ }
220
+ ]
221
+ }
222
+
223
+ json = JSON.pretty_generate(sarif)
224
+
225
+ if filepath
226
+ File.write(filepath, json)
227
+ nil
228
+ else
229
+ json
230
+ end
231
+ end
232
+
233
+ # Get exit code based on error severity.
234
+ #
235
+ # Useful for CI/CD pipelines.
236
+ #
237
+ # @param max_errors [Integer] Maximum errors allowed (default: 0)
238
+ # @return [Integer] Exit code (0 = success, 1 = errors found)
239
+ def exit_code(max_errors: 0)
240
+ return 0 if @navigation.errors.size <= max_errors
241
+
242
+ 1
243
+ end
244
+
245
+ # Get report summary as hash.
246
+ #
247
+ # @return [Hash] Report summary
248
+ def summary
249
+ @navigation.statistics.merge(
250
+ document: {
251
+ name: @document.name,
252
+ format: @document.format,
253
+ language: @document.language_code
254
+ },
255
+ has_errors: @navigation.errors.any?
256
+ )
257
+ end
258
+
259
+ # Print report to stdout.
260
+ #
261
+ # @param format [Symbol] Output format (:text, :json, :yaml)
262
+ def print(format: :text)
263
+ case format
264
+ when :text
265
+ puts to_text
266
+ when :json
267
+ puts to_json
268
+ when :yaml
269
+ puts to_yaml
270
+ else
271
+ raise ArgumentError, "Unknown format: #{format}"
272
+ end
273
+ end
274
+
275
+ private
276
+
277
+ # Generate report data hash.
278
+ #
279
+ # @return [Hash] Report data
280
+ def generate_report_data
281
+ {
282
+ metadata: {
283
+ tool: "Kotoshu",
284
+ version: Kotoshu::VERSION,
285
+ generated_at: Time.now.utc.iso8601
286
+ },
287
+ document: {
288
+ name: @document.name,
289
+ format: @document.format.to_s,
290
+ language: @document.language_code,
291
+ word_count: @document.word_count,
292
+ line_count: @document.line_count
293
+ },
294
+ statistics: @navigation.statistics,
295
+ errors: @navigation.errors.map do |error|
296
+ {
297
+ id: error.id,
298
+ location: {
299
+ line: error.location.line,
300
+ column: error.location.column,
301
+ node_path: error.location.node_path
302
+ },
303
+ original: error.original,
304
+ suggestions: error.suggestions&.map do |sugg|
305
+ {
306
+ word: sugg.word,
307
+ confidence: sugg.confidence,
308
+ source: sugg.source
309
+ }
310
+ end,
311
+ error_type: error.error_type.to_s,
312
+ confidence: error.confidence,
313
+ recommended_suggestion: error.recommended_suggestion&.word
314
+ }
315
+ end,
316
+ corrections: @navigation.export_corrections
317
+ }
318
+ end
319
+ end
320
+ end
321
+ end
@@ -0,0 +1,282 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+ require "json"
5
+ require "time"
6
+
7
+ module Kotoshu
8
+ module Cli
9
+ # Cache management CLI — wired as `kotoshu cache <subcommand>` in cli.rb.
10
+ #
11
+ # Operates on the disk-backed `Cache::LanguageCache`. Reads and writes
12
+ # go through the real cache API (`available?`, `cached_resources`,
13
+ # `clear_all`, `clear`, `clean`, `stats`, `get_spelling`, `get_grammar`,
14
+ # `frequency_available?`, `language_info`). Kelly frequency data lives
15
+ # in a separate `Cache::FrequencyCache`.
16
+ class CacheCommand < Thor
17
+ class_option :cache_path,
18
+ type: :string,
19
+ default: nil,
20
+ desc: "Override the cache directory for this invocation"
21
+ class_option :verbose,
22
+ type: :boolean,
23
+ default: false,
24
+ desc: "Enable verbose output",
25
+ aliases: ["-v"]
26
+
27
+ desc "list", "List cached languages"
28
+ method_option :json,
29
+ type: :boolean,
30
+ default: false,
31
+ desc: "Emit JSON instead of text"
32
+ def list
33
+ cache = create_cache
34
+ languages = cached_languages(cache)
35
+
36
+ if options[:json]
37
+ puts JSON.pretty_generate(languages: languages)
38
+ return
39
+ end
40
+
41
+ if languages.empty?
42
+ puts "No cached languages found"
43
+ return
44
+ end
45
+
46
+ puts "Cached languages:"
47
+ languages.each { |lang| puts " #{lang}" }
48
+ end
49
+
50
+ desc "info", "Show cache statistics"
51
+ method_option :json,
52
+ type: :boolean,
53
+ default: false,
54
+ desc: "Emit JSON instead of text"
55
+ def info
56
+ cache = create_cache
57
+ stats = cache.stats
58
+
59
+ if options[:json]
60
+ puts JSON.pretty_generate(
61
+ hits: stats[:hits],
62
+ misses: stats[:misses],
63
+ size: stats[:size_bytes],
64
+ hit_rate: stats[:hit_rate],
65
+ cached_resources: stats[:cached_resources],
66
+ location: cache.cache_path
67
+ )
68
+ return
69
+ end
70
+
71
+ puts "Cache Statistics"
72
+ puts " Location: #{cache.cache_path}"
73
+ puts " Languages cached: #{cached_languages(cache).size}"
74
+ puts " Size: #{format_bytes(stats[:size_bytes])}"
75
+ puts " Hits: #{stats[:hits]}, Misses: #{stats[:misses]}"
76
+ puts " Hit rate: #{(stats[:hit_rate] * 100).round(1)}%"
77
+ end
78
+
79
+ desc "clean", "Remove expired cache entries"
80
+ method_option :dry_run,
81
+ type: :boolean,
82
+ default: false,
83
+ desc: "Show what would be removed without removing"
84
+ def clean
85
+ cache = create_cache
86
+
87
+ if options[:dry_run]
88
+ stats = cache.stats
89
+ puts "Dry run — no changes made"
90
+ puts " Cached resources: #{stats[:cached_resources].size}"
91
+ puts " Size: #{format_bytes(stats[:size_bytes])}"
92
+ return
93
+ end
94
+
95
+ result = cache.clean
96
+ puts "Cache cleaned:"
97
+ puts " Expired entries removed: #{result[:expired_entries_removed]}"
98
+ puts " Bytes reclaimed: #{format_bytes(result[:bytes_reclaimed])}"
99
+ end
100
+
101
+ desc "evict", "Evict oldest entries to enforce the configured size cap"
102
+ method_option :dry_run,
103
+ type: :boolean,
104
+ default: false,
105
+ desc: "Show what would be evicted without removing"
106
+ def evict
107
+ cache = create_cache
108
+ plan = cache.evict(dry_run: options[:dry_run])
109
+
110
+ if options[:dry_run]
111
+ if plan[:evict].empty?
112
+ puts "Nothing to evict — cache is under the size cap"
113
+ return
114
+ end
115
+
116
+ puts "Dry run — no changes made"
117
+ puts "Would evict #{plan[:evict].size} entries, " \
118
+ "reclaiming #{format_bytes(plan[:bytes_reclaimed])}:"
119
+ print_eviction_list(plan[:evict])
120
+ return
121
+ end
122
+
123
+ if plan[:evict].empty?
124
+ puts "Nothing to evict — cache is under the size cap"
125
+ return
126
+ end
127
+
128
+ puts "Evicted #{plan[:evict].size} entries, " \
129
+ "reclaimed #{format_bytes(plan[:bytes_reclaimed])}:"
130
+ print_eviction_list(plan[:evict])
131
+ end
132
+
133
+ desc "download LANGUAGE", "Download spelling or grammar for a language"
134
+ method_option :type,
135
+ type: :string,
136
+ default: "spelling",
137
+ desc: "Resource type: spelling, grammar, or frequency"
138
+ method_option :force,
139
+ type: :boolean,
140
+ default: false,
141
+ desc: "Force re-download even if cached"
142
+ def download(language)
143
+ cache = create_cache
144
+ type = options[:type]
145
+
146
+ case type
147
+ when "spelling"
148
+ result = cache.get_spelling(language, force_download: options[:force])
149
+ puts "Spelling dictionary for '#{language}' at #{File.dirname(result[:dic_path])}"
150
+ when "grammar"
151
+ result = cache.get_grammar(language, force_download: options[:force])
152
+ puts "Grammar rules for '#{language}' at #{result[:rules_path]}"
153
+ when "frequency"
154
+ freq = Cache::FrequencyCache.new(cache_path: frequency_cache_path(cache))
155
+ result = freq.get_frequency(language, force_download: options[:force])
156
+ puts "Frequency data for '#{language}' at #{result[:frequency_path]}"
157
+ else
158
+ raise Errors::UsageError, "Unknown --type: #{type}"
159
+ end
160
+ rescue Kotoshu::ResourceNotCachedError, Kotoshu::IntegrityError => e
161
+ # Surface resource errors (offline mode, integrity mismatches
162
+ # that reach this frame) as exit 3, not a raw stack trace.
163
+ raise Errors::ResourceUnavailable, e.message
164
+ end
165
+
166
+ desc "purge", "Remove all cached data"
167
+ method_option :confirm,
168
+ type: :boolean,
169
+ default: false,
170
+ desc: "Skip the interactive confirmation prompt"
171
+ def purge
172
+ cache = create_cache
173
+
174
+ unless options[:confirm]
175
+ print "This will remove all cached dictionaries and frequency data. [y/N] "
176
+ response = $stdin.gets
177
+ return unless response && response.chomp =~ /\A[yY]\z/
178
+ end
179
+
180
+ cache.clear_all
181
+ puts "Cache purged"
182
+ end
183
+
184
+ desc "validate LANGUAGE", "Validate cached resources for a language"
185
+ def validate(language)
186
+ cache = create_cache
187
+ puts "Validating #{language}"
188
+ puts " Spelling: #{resource_status(cache, language, 'spelling')}"
189
+ puts " Grammar: #{resource_status(cache, language, 'grammar')}"
190
+ end
191
+
192
+ # Helper methods, not CLI subcommands. Declared inside no_commands
193
+ # so Thor does not treat them as commands — otherwise it prints a
194
+ # `[WARNING]` at load time which pollutes machine-readable output
195
+ # of other subcommands (`check --format json`, SARIF, etc.).
196
+ no_commands do
197
+ # Construct a LanguageCache honoring the --cache_path option.
198
+ #
199
+ # Public so specs can substitute a cache with a temp directory.
200
+ def create_cache
201
+ opts = {}
202
+ opts[:cache_path] = options[:cache_path] if options[:cache_path]
203
+ Cache::LanguageCache.new(**opts)
204
+ end
205
+
206
+ # Format bytes as a human-readable string with one decimal place.
207
+ #
208
+ # @param bytes [Integer, nil] Bytes
209
+ # @return [String] Formatted string
210
+ def format_bytes(bytes)
211
+ return "0 B" if bytes.nil? || bytes.zero?
212
+
213
+ units = %w[B KB MB GB TB]
214
+ exp = [Math.log(bytes, 1024).floor, units.size - 1].min
215
+ "#{format('%.1f', bytes.to_f / (1024**exp))} #{units[exp]}"
216
+ end
217
+
218
+ # Human-readable "time ago" string for an ISO8601 timestamp.
219
+ #
220
+ # @param iso_time [String, nil] ISO8601 timestamp
221
+ # @return [String] Time ago string
222
+ def time_ago(iso_time)
223
+ return "unknown" unless iso_time
224
+
225
+ time = Time.iso8601(iso_time)
226
+ seconds = Time.now - time
227
+
228
+ return "just now" if seconds < 60
229
+
230
+ minutes = (seconds / 60).to_i
231
+ return "#{minutes}m ago" if minutes < 60
232
+
233
+ hours = (minutes / 60).to_i
234
+ return "#{hours}h ago" if hours < 24
235
+
236
+ days = (hours / 24).to_i
237
+ return "#{days}d ago" if days < 30
238
+
239
+ months = (days / 30).to_i
240
+ return "#{months}mo ago" if months < 12
241
+
242
+ years = (months / 12).to_i
243
+ "#{years}y ago"
244
+ end
245
+ end
246
+
247
+ private
248
+
249
+ # Distinct language codes that have at least one cached resource.
250
+ def cached_languages(cache)
251
+ cache.cached_resources.map { |r| r.split(":").first }.uniq
252
+ end
253
+
254
+ # Print each entry in an eviction plan: path, size, cached-at.
255
+ #
256
+ # @param entries [Array<Hash>] each has :path, :size, :cached_at
257
+ # @return [void]
258
+ def print_eviction_list(entries)
259
+ entries.each do |e|
260
+ puts " #{e[:path]} (#{format_bytes(e[:size])}, cached #{e[:cached_at] || 'unknown'})"
261
+ end
262
+ end
263
+
264
+ # Status line for a (language, type) pair.
265
+ def resource_status(cache, language, type)
266
+ if cache.available?("#{language}:#{type}")
267
+ "✓"
268
+ else
269
+ "✗ Not cached"
270
+ end
271
+ end
272
+
273
+ # FrequencyCache lives under a different cache directory; mirror the
274
+ # override when --cache_path is given so tests stay isolated.
275
+ def frequency_cache_path(language_cache)
276
+ return nil unless options[:cache_path]
277
+
278
+ File.join(options[:cache_path], "frequency-lists")
279
+ end
280
+ end
281
+ end
282
+ end