kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,880 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ module Kotoshu
6
+ # Command-line interface namespace.
7
+ #
8
+ # The entry class is {Cli} (driven by exe/kotoshu); helper
9
+ # components — subcommands, the interactive reviewer, reporting and
10
+ # directory walking — are autoloaded from cli/.
11
+ module Cli
12
+ # LAZY: CLI helper components (autoloaded on first reference)
13
+ autoload :CacheCommand, "kotoshu/cli/cache_command"
14
+ autoload :DictCommand, "kotoshu/cli/dict_command"
15
+ autoload :CompletionsCommand, "kotoshu/cli/completions_command"
16
+ autoload :Errors, "kotoshu/cli/errors"
17
+ autoload :NavigationManager, "kotoshu/cli/navigation_manager"
18
+ autoload :DisplayFormatter, "kotoshu/cli/display_formatter"
19
+ autoload :PersonalCommand, "kotoshu/cli/personal_command"
20
+ autoload :InteractiveReviewer, "kotoshu/cli/interactive_reviewer"
21
+ autoload :BatchReporter, "kotoshu/cli/batch_reporter"
22
+ autoload :BaselineCommand, "kotoshu/cli/baseline_command"
23
+ autoload :AutoSetup, "kotoshu/cli/auto_setup"
24
+ autoload :StatusReport, "kotoshu/cli/status_report"
25
+ autoload :LanguageResolver, "kotoshu/cli/language_resolver"
26
+ autoload :ProgressReporter, "kotoshu/cli/progress_reporter"
27
+ autoload :IgnoreMatcher, "kotoshu/cli/ignore_matcher"
28
+ autoload :DirectoryWalker, "kotoshu/cli/directory_walker"
29
+ autoload :DirectoryCheck, "kotoshu/cli/directory_check"
30
+
31
+ # Command-line interface for Kotoshu spell checker.
32
+ #
33
+ # Two-stage model:
34
+ # Stage 1 (slow, network): `kotoshu setup LANG` downloads/registers resources
35
+ # Stage 2 (instant, cache-only): `kotoshu check FILE` uses cached resources
36
+ #
37
+ # Exit codes:
38
+ # 0 — no errors found / setup succeeded
39
+ # 1 — spelling errors found
40
+ # 2 — usage error (file not found, bad flags)
41
+ # 3 — resource not set up / setup failure (network, integrity)
42
+ #
43
+ # Commands raise Errors::CliError subclasses; the dispatcher in .start
44
+ # catches them and exits with the error's exit_status.
45
+ class Cli < Thor
46
+ class_option :language,
47
+ type: :string,
48
+ default: "auto",
49
+ desc: "Language code (auto, en, de, es, fr, pt, ru)",
50
+ aliases: ["-l"]
51
+
52
+ class_option :format,
53
+ type: :string,
54
+ enum: %w[text json sarif],
55
+ default: "text",
56
+ desc: "Output format (text, json, sarif)",
57
+ aliases: ["-f"]
58
+
59
+ class_option :interactive,
60
+ type: :boolean,
61
+ default: false,
62
+ desc: "Interactively review each error after check",
63
+ aliases: ["-i"]
64
+
65
+ class_option :verbose,
66
+ type: :boolean,
67
+ default: false,
68
+ desc: "Enable verbose output",
69
+ aliases: ["-v"]
70
+
71
+ desc "check [FILE] [DIR ...]", "Check spelling in files, directories, or stdin"
72
+ long_desc <<~DESC
73
+ Checks spelling in the given file (or stdin if no target is
74
+ given). Cache-only — never downloads. Run `kotoshu setup LANG`
75
+ first.
76
+
77
+ Directory targets (plan 88): walks the tree and checks every
78
+ file with a known text extension (md markdown asciidoc adoc
79
+ txt rst mdx), unless --include/--exclude globs say otherwise.
80
+ Hidden files and the .git, node_modules, vendor and target
81
+ directories are always skipped, and .gitignore/.ignore files
82
+ are honored (standard glob subset: *, **, ?, ! negation,
83
+ trailing / for directories; the last matching pattern wins; a
84
+ file inside an ignored directory cannot be re-included).
85
+ Output is one section per file in text mode, and one combined
86
+ JSON/SARIF document with per-file runs otherwise. Interactive
87
+ mode is file-only.
88
+
89
+ Exit codes:
90
+ 0 — no errors
91
+ 1 — spelling errors found
92
+ 2 — usage error (bad flags, file not found)
93
+ 3 — language not set up (run `kotoshu setup LANG`)
94
+ DESC
95
+ method_option :baseline,
96
+ type: :string,
97
+ desc: "Baseline JSON file — errors it covers pass, only new errors fail (see `kotoshu baseline init`)"
98
+ method_option :show_suppressed,
99
+ type: :boolean,
100
+ default: false,
101
+ desc: "List entries suppressed by inline directives or the baseline"
102
+ method_option :include,
103
+ type: :array,
104
+ desc: "Check only files matching these globs (replaces the known-extension default; " \
105
+ "a glob without a slash matches the basename, with a slash the path from the root)"
106
+ method_option :exclude,
107
+ type: :array,
108
+ desc: "Skip files matching these globs (always wins over --include and the extension default)"
109
+ method_option :personal,
110
+ type: :boolean,
111
+ default: true,
112
+ desc: "Accept words from the personal dictionary (personal.dic); --no-personal flags them again"
113
+ # Check spelling in files, directories, or stdin. Cache-only —
114
+ # never downloads; run `kotoshu setup LANG` first. See the
115
+ # command long description (kotoshu help check) for flags,
116
+ # directory-mode semantics, and exit codes.
117
+ #
118
+ # @param targets [Array<String>] File(s) and/or directory(ies);
119
+ # empty reads stdin
120
+ # @return [void]
121
+ # @see Kotoshu.check
122
+ def check(*targets)
123
+ apply_configuration!
124
+ if targets.empty? || (targets.one? && File.file?(targets.first))
125
+ check_single_file(targets.first)
126
+ else
127
+ check_targets(targets)
128
+ end
129
+ end
130
+
131
+ desc "setup [LANGUAGE] [LANGUAGE ...]", "Set up languages (download or register local files)"
132
+ long_desc <<~DESC
133
+ Stage 1 of the two-stage model. Downloads spelling/frequency/model
134
+ resources for the named language(s), or registers local .aff/.dic
135
+ files you already have on disk. After setup, `kotoshu check` runs
136
+ instantly with no network access.
137
+
138
+ With no args, lists currently set up languages.
139
+
140
+ Sources (one per invocation, applies to all listed languages):
141
+ --aff FILE --dic FILE use specific local Hunspell files
142
+ --from DIR look for {lang}.aff and {lang}.dic in DIR
143
+ (neither) download from kotoshu/dictionaries
144
+
145
+ Examples:
146
+ kotoshu setup en de fr # download from GitHub
147
+ kotoshu setup en --want spelling,frequency # also fetch Kelly list
148
+ kotoshu setup en --model --tier mini # model at the mini tier
149
+ kotoshu setup en --aff /p/en.aff --dic /p/en.dic
150
+ kotoshu setup en --from /usr/share/hunspell/
151
+ kotoshu setup --force en # re-download
152
+ kotoshu setup --list # show what's set up
153
+
154
+ Exit codes:
155
+ 0 — every language set up successfully
156
+ 3 — at least one language failed (network down, integrity, etc.)
157
+ DESC
158
+ method_option :aff, type: :string, desc: "Path to local .aff file"
159
+ method_option :dic, type: :string, desc: "Path to local .dic file"
160
+ method_option :from, type: :string, desc: "Directory containing local .aff/.dic"
161
+ method_option :frequency, type: :string, desc: "Path to local frequency.json"
162
+ method_option :want,
163
+ type: :string,
164
+ default: "spelling",
165
+ desc: "Comma-separated: spelling,frequency,model"
166
+ method_option :model,
167
+ type: :boolean,
168
+ default: false,
169
+ desc: "Fetch the ONNX model (shorthand for adding model to --want)"
170
+ method_option :tier,
171
+ type: :string,
172
+ desc: "Model tier: full, fluency, or mini (default: full)"
173
+ method_option :force,
174
+ type: :boolean,
175
+ default: false,
176
+ desc: "Re-fetch even if already cached"
177
+ method_option :strict,
178
+ type: :boolean,
179
+ default: false,
180
+ desc: "Re-raise on optional-resource failure during setup"
181
+ method_option :list,
182
+ type: :boolean,
183
+ default: false,
184
+ desc: "List currently set up languages and exit"
185
+ def setup(*languages)
186
+ apply_configuration!
187
+
188
+ if options[:list] || languages.empty?
189
+ list_setup
190
+ return
191
+ end
192
+
193
+ want = (options[:want] || "spelling").split(",").map(&:strip).map(&:to_sym)
194
+ want << :model if options[:model] && !want.include?(:model)
195
+ opts = setup_source_options(languages)
196
+ opts[:want] = want
197
+ opts[:tier] = options[:tier].to_sym if options[:tier]
198
+ opts[:force] = options[:force]
199
+ opts[:strict] = options[:strict]
200
+
201
+ results = languages.map do |lang|
202
+ print "Setup #{lang}... "
203
+ begin
204
+ result = with_progress_reporter(label: lang) do
205
+ Kotoshu.setup(lang, **opts)
206
+ end
207
+ describe_setup_result(result)
208
+ { lang: lang, ok: true }
209
+ rescue Kotoshu::Error, ArgumentError => e
210
+ puts "FAIL: #{e.message}"
211
+ { lang: lang, ok: false }
212
+ end
213
+ end
214
+
215
+ failed = results.reject { |r| r[:ok] }
216
+ puts "Set up #{results.size} language(s)."
217
+ return if failed.empty?
218
+
219
+ raise Errors::ResourceUnavailable,
220
+ "failed to set up: #{failed.map { |r| r[:lang] }.join(', ')}"
221
+ end
222
+
223
+ # Back-compat alias. New code should use `setup`.
224
+ desc "fetch LANGUAGE [LANGUAGE ...]", "Alias for `setup` (deprecated)", hide: true
225
+ method_option :aff, type: :string, desc: "Path to local .aff file"
226
+ method_option :dic, type: :string, desc: "Path to local .dic file"
227
+ method_option :from, type: :string, desc: "Directory containing local .aff/.dic"
228
+ method_option :frequency, type: :string, desc: "Path to local frequency.json"
229
+ method_option :want,
230
+ type: :string,
231
+ default: "spelling",
232
+ desc: "Comma-separated: spelling,frequency,model"
233
+ method_option :model,
234
+ type: :boolean,
235
+ default: false,
236
+ desc: "Fetch the ONNX model (shorthand for adding model to --want)"
237
+ method_option :tier,
238
+ type: :string,
239
+ desc: "Model tier: full, fluency, or mini (default: full)"
240
+ method_option :force,
241
+ type: :boolean,
242
+ default: false,
243
+ desc: "Re-fetch even if already cached"
244
+ method_option :strict,
245
+ type: :boolean,
246
+ default: false,
247
+ desc: "Re-raise on optional-resource failure during setup"
248
+ method_option :list,
249
+ type: :boolean,
250
+ default: false,
251
+ desc: "List currently set up languages and exit"
252
+ # Deprecated alias for {#setup} (kept hidden from help output).
253
+ #
254
+ # @param languages [Array<String>] One or more language codes
255
+ # @return [void]
256
+ # @see #setup
257
+ def fetch(*languages)
258
+ setup(*languages)
259
+ end
260
+
261
+ desc "dict SUBCOMMAND", "Dictionary operations"
262
+ subcommand "dict", DictCommand
263
+
264
+ desc "cache SUBCOMMAND", "Cache management"
265
+ subcommand "cache", CacheCommand
266
+
267
+ desc "personal SUBCOMMAND", "Personal dictionary management"
268
+ subcommand "personal", PersonalCommand
269
+
270
+ desc "completions SUBCOMMAND", "Emit shell completion scripts"
271
+ subcommand "completions", CompletionsCommand
272
+ desc "baseline SUBCOMMAND", "CI baseline management (see `check --baseline`)"
273
+ subcommand "baseline", BaselineCommand
274
+
275
+ desc "status", "Show setup, cache, and runtime status"
276
+ long_desc <<~DESC
277
+ Prints a snapshot of the kotoshu installation: which languages are
278
+ set up (with per-resource status), cache disk usage, audit log path,
279
+ default language, offline flag, and whether onnxruntime is loaded.
280
+
281
+ With --json, emits the same report as a JSON object for tooling.
282
+ DESC
283
+ method_option :json,
284
+ type: :boolean,
285
+ default: false,
286
+ desc: "Emit the report as JSON"
287
+ # Print a snapshot of the installation: set-up languages with
288
+ # per-resource status, cache disk usage, audit log, default
289
+ # language, and whether onnxruntime is loaded. `--json` emits
290
+ # the same report as one JSON object.
291
+ #
292
+ # @return [void]
293
+ def status
294
+ report = StatusReport.build(version: Kotoshu::VERSION)
295
+ if options[:json]
296
+ puts status_json(report)
297
+ else
298
+ puts status_text(report)
299
+ end
300
+ end
301
+
302
+ desc "version", "Show version information"
303
+ # Print the kotoshu and Ruby versions (also via `--version`).
304
+ #
305
+ # @return [void]
306
+ def version
307
+ puts "Kotoshu version #{Kotoshu::VERSION}"
308
+ puts "Ruby #{RUBY_VERSION}"
309
+ end
310
+
311
+ map %w[--version -V] => :version
312
+
313
+ # Dispatch entry point — bypasses Thor's start rescue so we can honor
314
+ # exit_status from Errors::CliError subclasses. Thor::Error still falls
315
+ # back to exit 1 for framework-level errors (bad flags, etc.).
316
+ #
317
+ # ResourceNotSetupError from the strict two-stage model is intercepted
318
+ # here: AutoSetup asks the user once, then we retry the dispatch. In
319
+ # non-TTY or offline mode AutoSetup re-raises so scripts see stable
320
+ # behavior.
321
+ def self.start(given_args = ARGV, config = {})
322
+ config[:shell] ||= Thor::Base.shell.new
323
+ dispatch(nil, given_args.dup, nil, config)
324
+ rescue Kotoshu::ResourceNotSetupError => e
325
+ raise Errors::ResourceUnavailable, e.message unless AutoSetup.new.call(e)
326
+
327
+ retry
328
+ rescue Errors::CliError => e
329
+ warn "Error: #{e.message}"
330
+ exit e.exit_status
331
+ rescue Thor::Error => e
332
+ warn e.message
333
+ exit 1
334
+ end
335
+
336
+ def self.exit_on_failure?
337
+ false
338
+ end
339
+
340
+ private
341
+
342
+ def apply_configuration!
343
+ Kotoshu::Configuration.reset
344
+ cfg = Kotoshu::Configuration.instance
345
+ cfg.default_language = options[:language] if options[:language] && options[:language] != "auto"
346
+ # `kotoshu check --no-personal` opts out of the personal
347
+ # dictionary for this run (plan 105). Other commands do not
348
+ # declare the flag, so options[:personal] is nil there.
349
+ cfg.personal_dictionary = options[:personal] unless options[:personal].nil?
350
+ end
351
+
352
+ # Install a ProgressReporter on Configuration.download_reporter
353
+ # for the duration of the block, then restore the prior value.
354
+ # The reporter writes to $stderr; in non-TTY contexts it still
355
+ # emits periodic line messages so CI logs show progress.
356
+ def with_progress_reporter(label:)
357
+ prior = Kotoshu.configuration.download_reporter
358
+ Kotoshu.configuration.download_reporter = ProgressReporter.new(
359
+ output: $stderr,
360
+ label: label
361
+ )
362
+ yield
363
+ ensure
364
+ Kotoshu.configuration.download_reporter = prior
365
+ end
366
+
367
+ def status_text(report)
368
+ lines = []
369
+ lines << "Kotoshu #{report.version}"
370
+ lines << ""
371
+
372
+ lines << "Setup:"
373
+ if report.resources.empty?
374
+ lines << " (no languages set up — run `kotoshu setup LANG`)"
375
+ else
376
+ report.resources.each do |r|
377
+ mark = r.available ? "✓" : "✗"
378
+ size = r.available ? StatusReport.format_bytes(r.size_bytes) : "—"
379
+ when_str = r.cached_at ? "cached #{r.cached_at.strftime('%Y-%m-%d')}" : ""
380
+ lines << format(" %-4s %-10s %s %s%s",
381
+ r.language, r.resource, mark, size,
382
+ when_str.empty? ? "" : ", #{when_str}")
383
+ end
384
+ end
385
+ lines << ""
386
+
387
+ lines << "Cache:"
388
+ lines << " Path #{report.cache_path}"
389
+ lines << " Size #{StatusReport.format_bytes(report.cache_size_bytes)}"
390
+ lines << " Languages #{report.languages_setup.size}"
391
+ lines << ""
392
+
393
+ lines << "Semantic:"
394
+ onnx_state = report.onnx_loaded ? "loaded" : "not loaded (gem install onnxruntime to enable)"
395
+ lines << " onnxruntime #{onnx_state}"
396
+ active_models = report.languages_with_model
397
+ models_str = active_models.empty? ? "0" : "#{active_models.size} (#{active_models.join(', ')})"
398
+ lines << " Active models #{models_str}"
399
+ lines << ""
400
+
401
+ lines << "Other:"
402
+ lines << if report.audit_log_path
403
+ " Audit log #{report.audit_log_path} (#{StatusReport.format_bytes(report.audit_log_size_bytes)})"
404
+ else
405
+ " Audit log (none yet — created on first audited operation)"
406
+ end
407
+ lines << " Default lang #{report.default_language || '(none)'}"
408
+ lines << " Offline mode #{report.offline ? 'yes' : 'no'}"
409
+ lines.join("\n")
410
+ end
411
+
412
+ def status_json(report)
413
+ require "json"
414
+
415
+ payload = {
416
+ version: report.version,
417
+ setup: report.resources.map do |r|
418
+ {
419
+ language: r.language,
420
+ resource: r.resource.to_s,
421
+ available: r.available,
422
+ size_bytes: r.size_bytes,
423
+ cached_at: r.cached_at&.iso8601
424
+ }
425
+ end,
426
+ cache: {
427
+ path: report.cache_path,
428
+ size_bytes: report.cache_size_bytes,
429
+ languages: report.languages_setup.size
430
+ },
431
+ semantic: {
432
+ onnxruntime_loaded: report.onnx_loaded,
433
+ active_models: report.languages_with_model
434
+ },
435
+ audit_log: report.audit_log_path && {
436
+ path: report.audit_log_path,
437
+ size_bytes: report.audit_log_size_bytes
438
+ },
439
+ default_language: report.default_language,
440
+ offline: report.offline
441
+ }
442
+ JSON.pretty_generate(payload)
443
+ end
444
+
445
+ # Load the --baseline file and apply it to +result+ (count-based
446
+ # semantics; see Kotoshu::Baseline). Returns nil when no baseline
447
+ # was given.
448
+ def apply_baseline(result, source)
449
+ store = check_baseline_store
450
+ return nil unless store
451
+
452
+ store.apply(result, file: source)
453
+ end
454
+
455
+ # The --baseline file's store, loaded once per invocation.
456
+ def check_baseline_store
457
+ path = options[:baseline]
458
+ return nil unless path
459
+
460
+ @check_baseline_store ||= begin
461
+ raise Errors::UsageError, "Baseline file not found: #{path}" unless File.exist?(path)
462
+
463
+ Kotoshu::Baseline::Store.load(path)
464
+ end
465
+ rescue Errors::UsageError
466
+ raise
467
+ rescue StandardError => e
468
+ raise Errors::UsageError, "Invalid baseline file #{path}: #{e.message}"
469
+ end
470
+
471
+ # Baseline-covered occurrences in the single-file path skip the
472
+ # suggestion sweep too (plan 116) — same budget the directory
473
+ # runner builds per file.
474
+ def single_file_filter(source)
475
+ store = check_baseline_store
476
+ store&.suggestions_filter_for(source)
477
+ end
478
+
479
+ def read_target(target)
480
+ if target.nil?
481
+ [$stdin.read, "<stdin>"]
482
+ elsif File.exist?(target)
483
+ [File.read(target, encoding: Kotoshu.configuration.encoding), target]
484
+ else
485
+ raise Errors::UsageError, "File not found: #{target}"
486
+ end
487
+ end
488
+
489
+ def run_check(text, suggestions_filter: nil)
490
+ language = resolve_language(text)
491
+ spellchecker = Kotoshu.spellchecker_for(language)
492
+ spellchecker.check(text, suggestions_filter: suggestions_filter)
493
+ rescue Kotoshu::DictionaryNotFoundError => e
494
+ raise Errors::ResourceUnavailable, e.message
495
+ end
496
+
497
+ def resolve_language(text)
498
+ result = LanguageResolver.new(
499
+ flag_value: options[:language],
500
+ default_language: Kotoshu.configuration.default_language
501
+ ).resolve(text: text)
502
+
503
+ warn "# #{result.note}" if result.note
504
+ result.language
505
+ end
506
+
507
+ def setup_source_options(languages)
508
+ opts = {}
509
+ if options[:aff] || options[:dic]
510
+ raise Errors::UsageError, "--aff and --dic require exactly one language" unless languages.size == 1
511
+
512
+ raise Errors::UsageError, "--aff and --dic must both be given" unless options[:aff] && options[:dic]
513
+
514
+ opts[:aff] = options[:aff]
515
+ opts[:dic] = options[:dic]
516
+ elsif options[:from]
517
+ opts[:from] = options[:from]
518
+ end
519
+ opts[:frequency] = options[:frequency] if options[:frequency]
520
+ opts
521
+ end
522
+
523
+ def describe_setup_result(result)
524
+ spelling = result.spelling || "skipped"
525
+ frequency = result.frequency || "skipped"
526
+ model = result.model || "skipped"
527
+ model += " (#{result.model_tier} tier)" if result.model && result.model_tier
528
+ source = result.source
529
+ puts "OK (spelling: #{spelling}, frequency: #{frequency}, model: #{model}, source: #{source})"
530
+ end
531
+
532
+ def list_setup
533
+ langs = Kotoshu.languages_setup
534
+ if langs.empty?
535
+ puts "No languages set up. Run `kotoshu setup LANG` to add one."
536
+ return
537
+ end
538
+
539
+ puts "Set up languages:"
540
+ langs.each { |lang| puts " #{lang}" }
541
+ end
542
+
543
+ def display_result(result, source, application: nil)
544
+ case options[:format]
545
+ when "json"
546
+ puts format_as_json(result, source, application: application)
547
+ when "sarif"
548
+ puts format_as_sarif(result, source, application: application,
549
+ include_suppressed: options[:show_suppressed])
550
+ else
551
+ puts format_as_text(result, source, application: application)
552
+ end
553
+ end
554
+
555
+ def format_as_text(result, source, application: nil)
556
+ lines = []
557
+ if result.success?
558
+ lines << "OK #{source} (#{result.word_count} words, no errors)"
559
+ else
560
+ lines << "FAIL #{source} (#{result.error_count} errors)"
561
+ result.each_error do |error|
562
+ suggestions_str = if error.has_suggestions?
563
+ " -> #{error.top_suggestions(3).join(', ')}"
564
+ else
565
+ ""
566
+ end
567
+ lines << " #{error.word}#{suggestions_str}"
568
+ end
569
+ end
570
+ if application && application.suppressed_count.positive?
571
+ lines << "#{application.suppressed_count} error(s) suppressed by baseline"
572
+ end
573
+ if application && application.stale_count.positive?
574
+ noun = application.stale_count == 1 ? "entry" : "entries"
575
+ lines << "Baseline: #{application.stale_count} stale #{noun} (no longer present) - " \
576
+ "refresh with `kotoshu baseline init`"
577
+ end
578
+ if options[:show_suppressed] && result.suppressed_count.positive?
579
+ result.suppressed_errors.each do |error|
580
+ lines << " suppressed [#{error.suppressed_by}]: #{error.word}"
581
+ end
582
+ end
583
+ lines.join("\n")
584
+ end
585
+
586
+ # Documented machine-readable shape (see CHANGELOG): top-level
587
+ # `success`/`wordCount`/`errorCount`/`uniqueErrorCount`/`errors`/
588
+ # `source`. The derived top-level keys are composed here in the
589
+ # presenter; each error element serializes through lutaml-model
590
+ # (WordResult) so no hand-rolled model serialization exists.
591
+ def format_as_json(result, source, application: nil)
592
+ require "json"
593
+ payload = result_json_fields(result, source)
594
+ payload["baseline"] = baseline_json_fields(application) if application
595
+ JSON.pretty_generate(payload)
596
+ end
597
+
598
+ # Per-result JSON fields shared by single-file and directory
599
+ # output.
600
+ def result_json_fields(result, source)
601
+ {
602
+ "success" => result.success?,
603
+ "wordCount" => result.word_count,
604
+ "errorCount" => result.error_count,
605
+ "uniqueErrorCount" => result.unique_error_count,
606
+ "errors" => result.errors.map { |err| Models::Result::WordResult.as_json(err) },
607
+ "suppressedCount" => result.suppressed_count,
608
+ "suppressedErrors" => result.suppressed_errors.map do |err|
609
+ Models::Result::WordResult.as_json(err)
610
+ end,
611
+ "source" => source
612
+ }
613
+ end
614
+
615
+ # Baseline block for the JSON output.
616
+ def baseline_json_fields(application)
617
+ {
618
+ "suppressedCount" => application.suppressed_count,
619
+ "staleCount" => application.stale_count
620
+ }
621
+ end
622
+
623
+ def format_as_sarif(result, source, application: nil, include_suppressed: false)
624
+ JSON.pretty_generate(
625
+ sarif_document([
626
+ sarif_run(result, source,
627
+ application: application, include_suppressed: include_suppressed)
628
+ ])
629
+ )
630
+ end
631
+
632
+ # The SARIF tool/driver descriptor, shared by every run.
633
+ def sarif_tool
634
+ {
635
+ "driver" => {
636
+ "name" => "kotoshu",
637
+ "version" => Kotoshu::VERSION,
638
+ "informationUri" => "https://github.com/kotoshu/kotoshu",
639
+ "rules" => [
640
+ {
641
+ "id" => "kotoshu/spelling",
642
+ "name" => "SpellingError",
643
+ "shortDescription" => {
644
+ "text" => "Word not found in the active dictionary."
645
+ }
646
+ }
647
+ ]
648
+ }
649
+ }
650
+ end
651
+
652
+ # Wrap runs into one SARIF 2.1.0 document. Directory mode emits
653
+ # one run per file (plan 88).
654
+ def sarif_document(runs)
655
+ require "json"
656
+ {
657
+ "version" => "2.1.0",
658
+ "$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
659
+ "runs" => runs
660
+ }
661
+ end
662
+
663
+ # Build one SARIF run for a single file result.
664
+ def sarif_run(result, source, application: nil, include_suppressed: false)
665
+ results = result.errors.map { |err| sarif_result_for(err, source, level: "warning") }
666
+ if application
667
+ result.suppressed_errors
668
+ .select { |err| err.suppressed_by == Models::Result::WordResult::SUPPRESSED_BY_BASELINE }
669
+ .each { |err| results << sarif_result_for(err, source, level: "note", suppressed_by: "baseline") }
670
+ end
671
+ if include_suppressed
672
+ result.suppressed_errors
673
+ .select { |err| err.suppressed_by == Models::Result::WordResult::SUPPRESSED_BY_INLINE }
674
+ .each { |err| results << sarif_result_for(err, source, level: "note", suppressed_by: "inline") }
675
+ end
676
+ { "tool" => sarif_tool, "results" => results }
677
+ end
678
+
679
+ # Build one SARIF result object for +err+. Suppressed entries carry a
680
+ # suppressions entry (kind external / status accepted) so SARIF
681
+ # consumers show them as suppressed rather than dropped (plan 82).
682
+ def sarif_result_for(err, source, level:, suppressed_by: nil)
683
+ suggestions = err.top_suggestions(3)
684
+ suggestion_text = suggestions.empty? ? "" : " Suggestions: #{suggestions.join(', ')}"
685
+ result = {
686
+ "ruleId" => "kotoshu/spelling",
687
+ "level" => level,
688
+ "message" => {
689
+ "text" => "'#{err.word}' is not in the dictionary.#{suggestion_text}"
690
+ },
691
+ "locations" => [
692
+ {
693
+ "physicalLocation" => {
694
+ "artifactLocation" => { "uri" => source_for_sarif(source) },
695
+ "region" => {
696
+ "charOffset" => err.position || 0,
697
+ "charLength" => err.word.length
698
+ }
699
+ }
700
+ }
701
+ ]
702
+ }
703
+ if suppressed_by
704
+ result["suppressions"] = [
705
+ { "kind" => "external", "status" => "accepted", "justification" => suppressed_by }
706
+ ]
707
+ end
708
+ result
709
+ end
710
+
711
+ def source_for_sarif(source)
712
+ source == "<stdin>" ? "stdin" : source
713
+ end
714
+
715
+ def interactive_review(result, source)
716
+ errors = result.errors
717
+ return if errors.empty?
718
+
719
+ index = 0
720
+ accepted = {}
721
+ skipped = Set.new
722
+
723
+ puts
724
+ puts "Interactive review: #{errors.size} error(s) in #{source}"
725
+ puts "Commands: [1-9] accept, [s] skip, [n]/Enter next, [p] prev, [l] list, [q] quit"
726
+
727
+ while index < errors.size
728
+ err = errors[index]
729
+ puts
730
+ puts "[#{index + 1}/#{errors.size}] '#{err.word}' (offset #{err.position || '?'})"
731
+ suggestions = err.top_suggestions(9)
732
+ if suggestions.empty?
733
+ puts " (no suggestions)"
734
+ else
735
+ suggestions.each_with_index { |s, i| puts " [#{i + 1}] #{s}" }
736
+ end
737
+ print "> "
738
+ input = $stdin.gets
739
+ break if input.nil?
740
+
741
+ input = input.chomp.downcase
742
+
743
+ case input
744
+ when "q"
745
+ puts "Quitting review."
746
+ break
747
+ when "n", ""
748
+ index += 1
749
+ when "p"
750
+ index = [index - 1, 0].max
751
+ when "l"
752
+ errors.each_with_index do |e, i|
753
+ marker = if accepted.key?(i)
754
+ "✓"
755
+ elsif skipped.include?(i)
756
+ "s"
757
+ else " "
758
+ end
759
+ puts " #{marker} #{i + 1}. #{e.word}"
760
+ end
761
+ when "s"
762
+ skipped << index
763
+ index += 1
764
+ when /\A[1-9]\z/
765
+ choice = input.to_i - 1
766
+ suggestion = suggestions[choice]
767
+ if suggestion
768
+ accepted[index] = suggestion
769
+ puts " → '#{err.word}' → '#{suggestion}' (recorded)"
770
+ index += 1
771
+ else
772
+ puts " No suggestion at that number."
773
+ end
774
+ else
775
+ puts " Unknown command."
776
+ end
777
+ end
778
+
779
+ puts
780
+ puts "Review complete: #{accepted.size} accepted, #{skipped.size} skipped, " \
781
+ "#{errors.size - accepted.size - skipped.size} unhandled."
782
+ puts "Note: 0.3 records decisions but does not rewrite source files." unless accepted.empty?
783
+ end
784
+
785
+ # Single-file / stdin check (the pre-directory-mode behavior,
786
+ # unchanged).
787
+ def check_single_file(target)
788
+ text, source = read_target(target)
789
+ result = run_check(text, suggestions_filter: single_file_filter(source))
790
+ application = apply_baseline(result, source)
791
+ result = application.result if application
792
+ display_result(result, source, application: application)
793
+ interactive_review(result, source) if options[:interactive] && result.failed?
794
+ exit 1 if result.failed?
795
+ end
796
+
797
+ # Multi-target / directory check (plan 88).
798
+ def check_targets(targets)
799
+ targets.each do |target|
800
+ raise Errors::UsageError, "File not found: #{target}" unless File.exist?(target)
801
+ end
802
+
803
+ runner = DirectoryCheck.new(
804
+ targets,
805
+ check: method(:run_check),
806
+ baseline_path: options[:baseline],
807
+ include_globs: Array(options[:include]),
808
+ exclude_globs: Array(options[:exclude])
809
+ )
810
+ runner.run
811
+ display_directory_result(runner)
812
+ if options[:interactive]
813
+ warn "# Interactive mode is file-only; skipping interactive review for directory targets"
814
+ end
815
+ exit 1 if runner.failed?
816
+ end
817
+
818
+ # Render the directory-mode result in the selected format.
819
+ def display_directory_result(runner)
820
+ case options[:format]
821
+ when "json"
822
+ puts directory_json(runner)
823
+ when "sarif"
824
+ puts directory_sarif(runner)
825
+ else
826
+ puts directory_text(runner)
827
+ end
828
+ end
829
+
830
+ # Text output: one section per file (the same section the
831
+ # single-file mode prints), plus a summary line.
832
+ def directory_text(runner)
833
+ if runner.file_count.zero?
834
+ extensions = DirectoryWalker::TEXT_EXTENSIONS.join(" ")
835
+ return "No text files found to check (known extensions: #{extensions})"
836
+ end
837
+
838
+ sections = runner.outcomes.map do |outcome|
839
+ format_as_text(outcome.result, outcome.path, application: outcome.application)
840
+ end
841
+ file_noun = runner.file_count == 1 ? "file" : "files"
842
+ error_noun = runner.error_count == 1 ? "error" : "errors"
843
+ sections << "#{runner.file_count} #{file_noun} checked, " \
844
+ "#{runner.word_count} words, #{runner.error_count} #{error_noun}"
845
+ sections.join("\n\n")
846
+ end
847
+
848
+ # Combined JSON document: totals plus one entry per file.
849
+ def directory_json(runner)
850
+ require "json"
851
+ payload = {
852
+ "success" => !runner.failed?,
853
+ "fileCount" => runner.file_count,
854
+ "wordCount" => runner.word_count,
855
+ "errorCount" => runner.error_count,
856
+ "suppressedCount" => runner.suppressed_count,
857
+ "files" => runner.outcomes.map { |outcome| file_json(outcome) }
858
+ }
859
+ JSON.pretty_generate(payload)
860
+ end
861
+
862
+ # Per-file JSON entry: the single-file payload keys.
863
+ def file_json(outcome)
864
+ payload = result_json_fields(outcome.result, outcome.path)
865
+ payload["baseline"] = baseline_json_fields(outcome.application) if outcome.application
866
+ payload
867
+ end
868
+
869
+ # Combined SARIF document: one run per file (plan 88).
870
+ def directory_sarif(runner)
871
+ runs = runner.outcomes.map do |outcome|
872
+ sarif_run(outcome.result, outcome.path,
873
+ application: outcome.application,
874
+ include_suppressed: options[:show_suppressed])
875
+ end
876
+ JSON.pretty_generate(sarif_document(runs))
877
+ end
878
+ end
879
+ end
880
+ end