kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,829 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Configuration for Kotoshu spell checker.
5
+ #
6
+ # This class manages configuration options for spell checking,
7
+ # including dictionary settings, suggestion limits, and language options.
8
+ #
9
+ # Configuration priority: CLI > ENV > Programmatic > Defaults
10
+ #
11
+ # @example Creating a configuration
12
+ # config = Configuration.new
13
+ # config.dictionary_path = "/usr/share/dict/words"
14
+ # config.dictionary_type = :unix_words
15
+ # config.max_suggestions = 15
16
+ #
17
+ # @example Using a block
18
+ # Configuration.new do |c|
19
+ # c.dictionary_path = "words.txt"
20
+ # c.language = "en-US"
21
+ # end
22
+ #
23
+ # @example Using environment variables
24
+ # ENV['KOTOSHU_LANGUAGE'] = 'de'
25
+ # config = Configuration.new
26
+ # config.language # => 'de'
27
+ class Configuration
28
+ autoload :Builder, "kotoshu/configuration/builder"
29
+ autoload :Resolver, "kotoshu/configuration/resolver"
30
+
31
+ # Configuration schema with ENV variable mappings.
32
+ #
33
+ # Each key maps to a hash with:
34
+ # - :env - Environment variable name
35
+ # - :default - Default value (can be a proc for dynamic defaults)
36
+ # - :description - Human-readable description
37
+ # - :type - Expected type (for validation/conversion)
38
+ SCHEMA = {
39
+ dictionary_path: {
40
+ env: "KOTOSHU_DICTIONARIES_PATH",
41
+ default: nil,
42
+ description: "Path to dictionary file",
43
+ type: String
44
+ },
45
+ cache_path: {
46
+ env: "KOTOSHU_CACHE_PATH",
47
+ default: -> { default_cache_path },
48
+ description: "Path to cache directory (~/.cache/kotoshu)",
49
+ type: String
50
+ },
51
+ config_path: {
52
+ env: "KOTOSHU_CONFIG_PATH",
53
+ default: -> { default_config_path },
54
+ description: "Path to user config directory (~/.config/kotoshu)",
55
+ type: String
56
+ },
57
+ data_path: {
58
+ env: "KOTOSHU_DATA_PATH",
59
+ default: -> { default_data_path },
60
+ description: "Path to data directory (~/.local/share/kotoshu)",
61
+ type: String
62
+ },
63
+ dictionaries_url: {
64
+ env: "KOTOSHU_DICTIONARIES_URL",
65
+ default: "https://raw.githubusercontent.com/kotoshu/dictionaries/main",
66
+ description: "Deprecated: use repos_base_url + dictionaries_pin via SourceRegistry",
67
+ type: String
68
+ },
69
+ models_url: {
70
+ env: "KOTOSHU_MODELS_URL",
71
+ default: "https://github.com/kotoshu/models-fasttext-onnx/raw/main",
72
+ description: "Deprecated: use repos_base_url + models_pin via SourceRegistry",
73
+ type: String
74
+ },
75
+ repos_base_url: {
76
+ env: "KOTOSHU_REPOS_BASE_URL",
77
+ default: -> { Kotoshu::SourceRegistry::DEFAULT_BASE_URL },
78
+ description: "GitHub raw root for all kotoshu content repos",
79
+ type: String
80
+ },
81
+ dictionaries_pin: {
82
+ env: "KOTOSHU_DICTIONARIES_PIN",
83
+ default: "v1",
84
+ description: "Branch/tag/commit pinned for kotoshu/dictionaries",
85
+ type: String
86
+ },
87
+ frequency_pin: {
88
+ env: "KOTOSHU_FREQUENCY_PIN",
89
+ default: "main",
90
+ description: "Branch/tag/commit pinned for kotoshu/frequency-list-kelly",
91
+ type: String
92
+ },
93
+ models_pin: {
94
+ env: "KOTOSHU_MODELS_PIN",
95
+ default: "main",
96
+ description: "Branch/tag/commit pinned for kotoshu/models-fasttext-onnx",
97
+ type: String
98
+ },
99
+ model_tier: {
100
+ env: "KOTOSHU_MODEL_TIER",
101
+ default: "fluency",
102
+ description: "Model tier used when none is given (fluency, full, or mini)",
103
+ type: String
104
+ },
105
+ auto_download: {
106
+ env: "KOTOSHU_AUTO_DOWNLOAD",
107
+ default: true,
108
+ description: "Automatically download missing dictionaries",
109
+ type: :boolean
110
+ },
111
+ cache_ttl: {
112
+ env: "KOTOSHU_CACHE_TTL",
113
+ default: 86_400, # 24 hours in seconds
114
+ description: "Cache TTL in seconds",
115
+ type: Integer
116
+ },
117
+ max_cache_size: {
118
+ env: "KOTOSHU_MAX_CACHE_SIZE",
119
+ default: 1_073_741_824, # 1GB
120
+ description: "Maximum cache size in bytes",
121
+ type: Integer
122
+ },
123
+ audit_max_bytes: {
124
+ env: "KOTOSHU_AUDIT_MAX_BYTES",
125
+ default: 10_485_760, # 10 MB
126
+ description: "Audit log size threshold (bytes) that triggers rotation",
127
+ type: Integer
128
+ },
129
+ audit_rotations: {
130
+ env: "KOTOSHU_AUDIT_ROTATIONS",
131
+ default: 5,
132
+ description: "Number of historical audit log rotations to keep",
133
+ type: Integer
134
+ },
135
+ dictionary_type: {
136
+ env: "KOTOSHU_DICTIONARY_TYPE",
137
+ default: :unix_words,
138
+ description: "Dictionary type (:unix_words, :plain_text, :hunspell, :cspell, :custom)",
139
+ type: Symbol
140
+ },
141
+ language: {
142
+ env: "KOTOSHU_LANGUAGE",
143
+ default: "en-US",
144
+ description: "Language code (e.g., en-US, de-DE, ja-JP)",
145
+ type: String
146
+ },
147
+ locale: {
148
+ env: "KOTOSHU_LOCALE",
149
+ default: nil,
150
+ description: "Locale (e.g., en, en_US, de_DE)",
151
+ type: String
152
+ },
153
+ max_suggestions: {
154
+ env: "KOTOSHU_MAX_SUGGESTIONS",
155
+ default: 10,
156
+ description: "Maximum number of suggestions",
157
+ type: Integer
158
+ },
159
+ case_sensitive: {
160
+ env: "KOTOSHU_CASE_SENSITIVE",
161
+ default: false,
162
+ description: "Enable case-sensitive lookups",
163
+ type: :boolean
164
+ },
165
+ verbose: {
166
+ env: "KOTOSHU_VERBOSE",
167
+ default: false,
168
+ description: "Enable verbose output",
169
+ type: :boolean
170
+ },
171
+ encoding: {
172
+ env: "KOTOSHU_ENCODING",
173
+ default: "UTF-8",
174
+ description: "Character encoding",
175
+ type: String
176
+ },
177
+ dictionaries_path: {
178
+ env: "KOTOSHU_DICTIONARIES_PATH",
179
+ default: nil,
180
+ description: "Path to dictionaries directory (for grammar rules)",
181
+ type: String
182
+ },
183
+ offline: {
184
+ env: "KOTOSHU_OFFLINE",
185
+ default: false,
186
+ description: "Use only cached resources; never download",
187
+ type: :boolean
188
+ },
189
+ resource_pin: {
190
+ env: "KOTOSHU_RESOURCE_PIN",
191
+ default: "main",
192
+ description: "Branch/tag/commit pinned for resource downloads",
193
+ type: String
194
+ },
195
+ default_language: {
196
+ env: "KOTOSHU_DEFAULT_LANGUAGE",
197
+ default: "en",
198
+ description: "Fallback language when detection is inconclusive",
199
+ type: String
200
+ },
201
+ semantic_cascade_threshold: {
202
+ env: "KOTOSHU_SEMANTIC_CASCADE_THRESHOLD",
203
+ default: 1.0,
204
+ description: "Skip semantic reranking when composite confidence is already at or above this (1.0 = always rerank)",
205
+ type: Float
206
+ },
207
+ backend: {
208
+ env: "KOTOSHU_BACKEND",
209
+ default: "ruby",
210
+ description: "Engine backend for correct?/suggest (auto by default: native when the extension is available, ruby otherwise; or force either)",
211
+ type: String
212
+ },
213
+ personal_dictionary: {
214
+ env: "KOTOSHU_PERSONAL_DICTIONARY",
215
+ default: true,
216
+ description: "Words in the personal dictionary (personal.dic) pass the check path",
217
+ type: :boolean
218
+ }
219
+ }.freeze
220
+
221
+ # Default configuration values (legacy, for backward compatibility).
222
+ DEFAULTS = {
223
+ dictionary_path: nil,
224
+ dictionary_type: :unix_words,
225
+ language: "en-US",
226
+ locale: nil,
227
+ max_suggestions: 10,
228
+ case_sensitive: false,
229
+ verbose: false,
230
+ suggestion_algorithms: nil, # Use defaults
231
+ custom_words: [],
232
+ encoding: "UTF-8",
233
+ dictionaries_path: nil, # Path to dictionaries directory (for grammar rules)
234
+ offline: false,
235
+ default_language: "en",
236
+ resource_pin: "main",
237
+ semantic_cascade_threshold: 1.0,
238
+ # "auto" (plan 133): the Rust engine whenever the native
239
+ # extension loaded and the dictionary is a file-backed Hunspell,
240
+ # pure Ruby otherwise — identical outputs either way (the
241
+ # conformance contract), so the default is the faster engine
242
+ # with the pure fallback, never a failure.
243
+ backend: "auto",
244
+ personal_dictionary: true
245
+ }.freeze
246
+
247
+ # @return [String, nil] Path to the dictionary file
248
+ attr_accessor :dictionary_path
249
+
250
+ # @return [Symbol] Dictionary type (:unix_words, :plain_text, :hunspell, :cspell, :custom)
251
+ attr_accessor :dictionary_type
252
+
253
+ # @return [String] Language code (e.g., "en-US", "en-GB")
254
+ attr_accessor :language
255
+
256
+ # @return [String, nil] Locale (e.g., "en", "en_US")
257
+ attr_accessor :locale
258
+
259
+ # @return [Integer] Maximum number of suggestions to return
260
+ attr_accessor :max_suggestions
261
+
262
+ # @return [Boolean] Whether lookups are case-sensitive
263
+ attr_accessor :case_sensitive
264
+
265
+ # @return [Boolean] Whether to enable verbose output
266
+ attr_accessor :verbose
267
+
268
+ # @return [Array<Class>, nil] Suggestion algorithms to use
269
+ attr_accessor :suggestion_algorithms
270
+
271
+ # @return [Array<String>] Custom words to add to dictionary
272
+ attr_accessor :custom_words
273
+
274
+ # @return [String] Character encoding
275
+ attr_accessor :encoding
276
+
277
+ # @return [Dictionary::Base, nil] The loaded dictionary (lazy loaded)
278
+ attr_accessor :dictionary
279
+
280
+ # @return [String, nil] Path to dictionaries directory (for grammar rules)
281
+ attr_accessor :dictionaries_path
282
+
283
+ # @return [Boolean] Whether to use only cached resources (no downloads)
284
+ attr_accessor :offline
285
+
286
+ # @return [String] Fallback language when detection is inconclusive
287
+ attr_accessor :default_language
288
+
289
+ # @return [String] Branch/tag/commit pinned for resource downloads
290
+ attr_accessor :resource_pin
291
+
292
+ # @return [Float] Confidence at or above which the semantic rerank
293
+ # is skipped (1.0 = always rerank; see Suggestions::SemanticCascade)
294
+ attr_accessor :semantic_cascade_threshold
295
+
296
+ # @return [String] Engine backend for correct?/suggest — "ruby"
297
+ # (pure Ruby, default), "native" (Rust extension; raises
298
+ # Native::Unavailable when it cannot serve), or "auto" (native when
299
+ # the extension is loaded and the dictionary is a file-backed
300
+ # Hunspell, else pure Ruby). Env: KOTOSHU_BACKEND. See
301
+ # NativeBackend for the boundary — the semantic path stays Ruby.
302
+ attr_accessor :backend
303
+
304
+ # @return [Boolean] Whether Spellchecker#check accepts words from
305
+ # the user personal dictionary (~/.config/kotoshu/personal.dic,
306
+ # KOTOSHU_PERSONAL_DIC override) instead of flagging them as
307
+ # errors — the README story and the LSP diagnostics behavior
308
+ # (plan 105). Opt out with KOTOSHU_PERSONAL_DICTIONARY=false or
309
+ # `kotoshu check --no-personal`; the suggest path is unaffected,
310
+ # so personal words stay spellcheckable.
311
+ attr_accessor :personal_dictionary
312
+
313
+ # @return [String, nil] Path to cache directory
314
+ attr_accessor :cache_path
315
+
316
+ # @return [String, nil] Path to user config directory
317
+ attr_accessor :config_path
318
+
319
+ # @return [String, nil] Path to data directory (audit log, etc.)
320
+ attr_accessor :data_path
321
+
322
+ # @return [String] Base URL for downloading dictionaries (deprecated)
323
+ attr_accessor :dictionaries_url
324
+
325
+ # @return [String] Base URL for FastText ONNX models (deprecated)
326
+ attr_accessor :models_url
327
+
328
+ # @return [String] GitHub raw root for all kotoshu content repos
329
+ attr_accessor :repos_base_url
330
+
331
+ # @return [String] Pin for kotoshu/dictionaries
332
+ attr_accessor :dictionaries_pin
333
+
334
+ # @return [String] Pin for kotoshu/frequency-list-kelly
335
+ attr_accessor :frequency_pin
336
+
337
+ # @return [String] Branch/tag/commit pinned for model downloads
338
+ attr_accessor :models_pin
339
+
340
+ # @return [String] Model tier used when a setup/resolve call omits
341
+ # `tier:` — "fluency" (the medium tier, ~15 MB; owner decision
342
+ # 2026-09-04), "full" (maximum accuracy, ~120 MB), or "mini"
343
+ # (light/edge, ~3 MB). Validated against
344
+ # Cache::ModelCache::TIERS at use time; ResourceManager#resolve
345
+ # bridges legacy single-tier caches to this default (see its
346
+ # tier-less fallback contract).
347
+ attr_accessor :model_tier
348
+
349
+ # @return [#start,#update,#maybe_report_periodic,#finish,nil]
350
+ # Optional progress reporter for downloads. Typically set by the
351
+ # CLI (Cli::ProgressReporter) for human-facing setup runs; nil
352
+ # (silent) for programmatic API usage.
353
+ attr_accessor :download_reporter
354
+
355
+ # @return [Boolean] Whether to automatically download missing dictionaries
356
+ attr_accessor :auto_download
357
+
358
+ # @return [Integer] Cache TTL in seconds
359
+ attr_accessor :cache_ttl
360
+
361
+ # @return [Integer] Maximum cache size in bytes
362
+ attr_accessor :max_cache_size
363
+
364
+ # @return [Integer] Audit log size threshold (bytes) for rotation
365
+ attr_accessor :audit_max_bytes
366
+
367
+ # @return [Integer] Number of historical audit log rotations to keep
368
+ attr_accessor :audit_rotations
369
+
370
+ # @return [Resolver] The configuration resolver
371
+ attr_reader :resolver
372
+
373
+ # Create a new configuration.
374
+ #
375
+ # @param args [Array] Variable arguments (positional hash or nothing)
376
+ # @param kwargs [Hash] Keyword arguments for configuration
377
+ # @param block [Proc] Optional block for configuration
378
+ #
379
+ # @example With hash
380
+ # config = Configuration.new(
381
+ # dictionary_path: "/usr/share/dict/words",
382
+ # language: "en-US"
383
+ # )
384
+ #
385
+ # @example With block
386
+ # Configuration.new do |c|
387
+ # c.dictionary_path = "words.txt"
388
+ # c.max_suggestions = 15
389
+ # end
390
+ #
391
+ # @example With CLI options (higher priority)
392
+ # config = Configuration.new(
393
+ # language: "en-US",
394
+ # cli_options: { language: "ja" } # ja will be used
395
+ # )
396
+ def initialize(*args, **kwargs, &block)
397
+ # Handle both positional hash and keyword arguments
398
+ settings = args.first.is_a?(Hash) ? args.first : {}
399
+ settings = settings.merge(kwargs)
400
+
401
+ # Extract cli_options if provided
402
+ cli_options = settings.delete(:cli_options) || {}
403
+
404
+ # Build the resolver with settings as programmatic defaults
405
+ @resolver = Resolver.new(
406
+ env: settings[:env] || {},
407
+ programmatic: settings,
408
+ cli: cli_options,
409
+ defaults: DEFAULTS
410
+ )
411
+
412
+ apply_defaults
413
+ apply_resolver_values
414
+ apply_explicit_settings(settings)
415
+
416
+ yield self if block
417
+ end
418
+
419
+ # Get a configuration value using the resolver.
420
+ #
421
+ # This respects the priority: CLI > ENV > Programmatic > Defaults
422
+ #
423
+ # @param key [Symbol] The configuration key
424
+ # @return [Object] The resolved value
425
+ #
426
+ # @example
427
+ # config.get(:language) # => resolved language value
428
+ def get(key)
429
+ @resolver.get(key)
430
+ end
431
+
432
+ # Check if a configuration key has a value set.
433
+ #
434
+ # @param key [Symbol] The configuration key
435
+ # @return [Boolean] True if the key is set somewhere
436
+ def key?(key)
437
+ @resolver.key?(key)
438
+ end
439
+
440
+ # Load or get the dictionary.
441
+ #
442
+ # @return [Dictionary::Base] The loaded dictionary
443
+ def dictionary
444
+ @dictionary ||= load_dictionary
445
+ end
446
+
447
+ # Load the dictionary based on configuration.
448
+ #
449
+ # @return [Dictionary::Base] The loaded dictionary
450
+ # @raise [DictionaryNotFoundError] If dictionary file not found
451
+ # @raise [ConfigurationError] If dictionary type is invalid
452
+ def load_dictionary
453
+ dict = case @dictionary_type
454
+ when :unix_words
455
+ load_unix_words_dictionary
456
+ when :plain_text
457
+ load_plain_text_dictionary
458
+ when :custom
459
+ load_custom_dictionary
460
+ when :hunspell
461
+ load_hunspell_dictionary
462
+ when :cspell
463
+ load_cspell_dictionary
464
+ else
465
+ raise ConfigurationError, "Unknown dictionary type: #{@dictionary_type}"
466
+ end
467
+
468
+ # Add custom words
469
+ @custom_words.each do |word|
470
+ dict.add_word(word)
471
+ end
472
+
473
+ dict
474
+ end
475
+
476
+ # Reset the dictionary (force reload on next access).
477
+ #
478
+ # @return [self] Self for chaining
479
+ def reset_dictionary
480
+ @dictionary = nil
481
+ self
482
+ end
483
+
484
+ # Convert to hash.
485
+ #
486
+ # @return [Hash] Hash representation
487
+ def to_h
488
+ {
489
+ dictionary_path: @dictionary_path,
490
+ dictionary_type: @dictionary_type,
491
+ language: @language,
492
+ locale: @locale,
493
+ max_suggestions: @max_suggestions,
494
+ case_sensitive: @case_sensitive,
495
+ verbose: @verbose,
496
+ suggestion_algorithms: @suggestion_algorithms&.map(&:name),
497
+ custom_words: @custom_words,
498
+ encoding: @encoding,
499
+ dictionaries_path: @dictionaries_path,
500
+ cache_path: @cache_path,
501
+ config_path: @config_path,
502
+ data_path: @data_path,
503
+ dictionaries_url: @dictionaries_url,
504
+ repos_base_url: @repos_base_url,
505
+ dictionaries_pin: @dictionaries_pin,
506
+ frequency_pin: @frequency_pin,
507
+ models_pin: @models_pin,
508
+ model_tier: @model_tier,
509
+ auto_download: @auto_download,
510
+ cache_ttl: @cache_ttl,
511
+ max_cache_size: @max_cache_size,
512
+ semantic_cascade_threshold: @semantic_cascade_threshold,
513
+ backend: @backend,
514
+ personal_dictionary: @personal_dictionary
515
+ }
516
+ end
517
+
518
+ # Build a SourceRegistry honoring this configuration's base URL and
519
+ # per-repo pins. Single source of truth for all resource URLs.
520
+ #
521
+ # @return [Kotoshu::SourceRegistry]
522
+ def source_registry
523
+ Kotoshu::SourceRegistry.new(
524
+ base_url: @repos_base_url,
525
+ pins: {
526
+ "dictionaries" => @dictionaries_pin,
527
+ "frequency-list-kelly" => @frequency_pin,
528
+ "models-fasttext-onnx" => @models_pin
529
+ }
530
+ )
531
+ end
532
+
533
+ # Clone the configuration.
534
+ #
535
+ # @return [Configuration] A new configuration with the same settings
536
+ def clone
537
+ self.class.new(to_h)
538
+ end
539
+
540
+ # Get the default configuration.
541
+ #
542
+ # @return [Configuration] Default configuration instance
543
+ #
544
+ # @example
545
+ # config = Configuration.default
546
+ def self.default
547
+ # No explicit settings: passing DEFAULTS here would re-apply them
548
+ # via apply_explicit_settings AFTER the resolver ran, clobbering
549
+ # every ENV-derived value (KOTOSHU_OFFLINE etc.). apply_defaults
550
+ # and the resolver's defaults layer already cover the defaults.
551
+ new
552
+ end
553
+
554
+ # Global configuration instance.
555
+ #
556
+ # The process-default configuration instance.
557
+ #
558
+ # New code should prefer passing a +Configuration+ instance
559
+ # explicitly to `Spellchecker.new` (and friends) rather than
560
+ # reading this singleton — that path is testable without
561
+ # process-wide state and scopes the configuration per-call. This
562
+ # accessor is kept for backward compat with the facade methods
563
+ # in {Kotoshu} (Kotoshu.correct?, Kotoshu.spellchecker, etc.)
564
+ # that don't take a Configuration argument.
565
+ #
566
+ # @return [Configuration] The global configuration
567
+ #
568
+ # @example
569
+ # Configuration.instance.dictionary_path = "/usr/share/dict/words"
570
+ def self.instance
571
+ @instance ||= default
572
+ end
573
+
574
+ # Replace the process-default configuration instance. Type-checked
575
+ # so a typo doesn't silently nil-out the singleton. Clears the
576
+ # cached Spellchecker built off the previous instance so the next
577
+ # Kotoshu.spellchecker rebuilds with the new config.
578
+ #
579
+ # @param config [Configuration]
580
+ # @return [Configuration]
581
+ def self.instance=(config)
582
+ raise ArgumentError, "instance must be a Configuration" unless config.is_a?(Configuration)
583
+
584
+ @instance = config
585
+ end
586
+
587
+ # Reset the global configuration.
588
+ #
589
+ # @return [Configuration] The reset configuration
590
+ def self.reset
591
+ @instance = default
592
+ end
593
+
594
+ private
595
+
596
+ # Apply default values.
597
+ def apply_defaults
598
+ @dictionary_path = DEFAULTS[:dictionary_path]
599
+ @dictionary_type = DEFAULTS[:dictionary_type]
600
+ @language = DEFAULTS[:language]
601
+ @locale = DEFAULTS[:locale]
602
+ @max_suggestions = DEFAULTS[:max_suggestions]
603
+ @case_sensitive = DEFAULTS[:case_sensitive]
604
+ @verbose = DEFAULTS[:verbose]
605
+ @suggestion_algorithms = DEFAULTS[:suggestion_algorithms]
606
+ @custom_words = DEFAULTS[:custom_words].dup
607
+ @encoding = DEFAULTS[:encoding]
608
+ @dictionaries_path = DEFAULTS[:dictionaries_path]
609
+ @dictionary = nil
610
+
611
+ # New cache-related defaults
612
+ @cache_path = self.class.default_cache_path
613
+ @config_path = self.class.default_config_path
614
+ @data_path = self.class.default_data_path
615
+ @dictionaries_url = SCHEMA[:dictionaries_url][:default]
616
+ @models_url = SCHEMA[:models_url][:default]
617
+ default = SCHEMA[:repos_base_url][:default]
618
+ @repos_base_url = default.is_a?(Proc) ? default.call : default
619
+ @dictionaries_pin = SCHEMA[:dictionaries_pin][:default]
620
+ @frequency_pin = SCHEMA[:frequency_pin][:default]
621
+ @models_pin = SCHEMA[:models_pin][:default]
622
+ @model_tier = SCHEMA[:model_tier][:default]
623
+ @download_reporter = nil
624
+ @auto_download = SCHEMA[:auto_download][:default]
625
+ @cache_ttl = SCHEMA[:cache_ttl][:default]
626
+ @max_cache_size = SCHEMA[:max_cache_size][:default]
627
+ @audit_max_bytes = SCHEMA[:audit_max_bytes][:default]
628
+ @audit_rotations = SCHEMA[:audit_rotations][:default]
629
+ @resource_pin = DEFAULTS[:resource_pin]
630
+ @semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
631
+ @backend = DEFAULTS[:backend]
632
+ @personal_dictionary = DEFAULTS[:personal_dictionary]
633
+ end
634
+
635
+ # Apply resolved values from the resolver (ENV, defaults).
636
+ def apply_resolver_values
637
+ # Apply values from ENV and defaults via resolver
638
+ SCHEMA.each_key do |key|
639
+ env_value = @resolver.get(key)
640
+ next if env_value.nil?
641
+
642
+ # Convert boolean strings if needed
643
+ value = convert_schema_value(key, env_value)
644
+ setter = "#{key}="
645
+ public_send(setter, value) if self.class.method_defined?(setter)
646
+ end
647
+ end
648
+
649
+ # Apply explicit settings from the settings hash.
650
+ #
651
+ # Explicit settings have priority over ENV and defaults.
652
+ def apply_explicit_settings(settings)
653
+ settings.each do |key, value|
654
+ next if %i[env cli_options].include?(key)
655
+
656
+ setter = "#{key}="
657
+ public_send(setter, value) if self.class.method_defined?(setter)
658
+ end
659
+ end
660
+
661
+ # Convert a value based on schema type.
662
+ #
663
+ # ENV values arrive as Strings; the schema type declares the target.
664
+ # Class comparisons use ==, not `case`/`===`: `case Integer; when
665
+ # Integer` never matches because Class#=== tests instance-of while
666
+ # the schema stores the class itself.
667
+ #
668
+ # @param key [Symbol] The configuration key
669
+ # @param value [Object] The value to convert
670
+ # @return [Object] The converted value
671
+ def convert_schema_value(key, value)
672
+ schema = SCHEMA[key]
673
+ return value if schema.nil? || value.nil?
674
+
675
+ type = schema[:type]
676
+ if type == :boolean
677
+ parse_boolean(value)
678
+ elsif type == Integer
679
+ value.is_a?(Integer) ? value : value.to_i
680
+ elsif type == Float
681
+ value.is_a?(Float) ? value : value.to_f
682
+ elsif type == Symbol
683
+ value.is_a?(Symbol) ? value : value.to_sym
684
+ else
685
+ value
686
+ end
687
+ end
688
+
689
+ # Parse a boolean value from string.
690
+ #
691
+ # @param value [Object] The value to parse
692
+ # @return [Boolean] The parsed boolean
693
+ def parse_boolean(value)
694
+ return true if value == true || value.to_s =~ /^(true|t|yes|y|1)$/i
695
+ return false if value == false || value.to_s =~ /^(false|f|no|n|0)$/i
696
+
697
+ # Default to false for unrecognized values
698
+ false
699
+ end
700
+
701
+ # Get default cache path.
702
+ #
703
+ # @return [String] The default cache path (see Paths.cache_path)
704
+ def self.default_cache_path
705
+ Paths.cache_path
706
+ end
707
+
708
+ # Get default user config directory.
709
+ #
710
+ # @return [String] The default config path (see Paths.config_path)
711
+ def self.default_config_path
712
+ Paths.config_path
713
+ end
714
+
715
+ # Get default data directory.
716
+ #
717
+ # @return [String] The default data path (see Paths.data_path)
718
+ def self.default_data_path
719
+ Paths.data_path
720
+ end
721
+
722
+ # Load UnixWords dictionary.
723
+ def load_unix_words_dictionary
724
+ # First try configured path or system dictionary
725
+ path = @dictionary_path || Dictionary::UnixWords.detect_system_dictionary
726
+
727
+ if path
728
+ raise DictionaryNotFoundError, path unless File.exist?(path)
729
+
730
+ return Dictionary::UnixWords.new(
731
+ path,
732
+ language_code: @language,
733
+ locale: @locale,
734
+ case_sensitive: @case_sensitive
735
+ )
736
+ end
737
+
738
+ # Try to detect system dictionary
739
+ dict = Dictionary::UnixWords.detect(
740
+ language_code: @language,
741
+ locale: @locale,
742
+ case_sensitive: @case_sensitive
743
+ )
744
+
745
+ return dict if dict
746
+
747
+ # Fall back to local dictionaries directory
748
+ local_paths = [
749
+ File.expand_path("dictionaries/unix_words/words", __dir__),
750
+ File.expand_path("../../dictionaries/unix_words/web2", __dir__),
751
+ File.expand_path("../../dictionaries/unix_words/web2a", __dir__)
752
+ ]
753
+
754
+ local_paths.each do |local_path|
755
+ if File.exist?(local_path)
756
+ return Dictionary::UnixWords.new(
757
+ local_path,
758
+ language_code: @language,
759
+ locale: @locale,
760
+ case_sensitive: @case_sensitive
761
+ )
762
+ end
763
+ end
764
+
765
+ raise DictionaryNotFoundError,
766
+ "no unix_words dictionary found; run `kotoshu setup #{@language}`"
767
+ end
768
+
769
+ # Load PlainText dictionary.
770
+ def load_plain_text_dictionary
771
+ path = @dictionary_path
772
+
773
+ raise ConfigurationError, "dictionary_path is required for plain_text type" unless path
774
+ raise DictionaryNotFoundError, path unless File.exist?(path)
775
+
776
+ Dictionary::PlainText.new(
777
+ path,
778
+ language_code: @language,
779
+ locale: @locale,
780
+ case_sensitive: @case_sensitive
781
+ )
782
+ end
783
+
784
+ # Load Custom dictionary.
785
+ def load_custom_dictionary
786
+ Dictionary::Custom.new(
787
+ words: @custom_words,
788
+ language_code: @language,
789
+ locale: @locale,
790
+ case_sensitive: @case_sensitive
791
+ )
792
+ end
793
+
794
+ # Load Hunspell dictionary.
795
+ def load_hunspell_dictionary
796
+ dic_path = @dictionary_path
797
+
798
+ raise ConfigurationError, "dictionary_path is required for hunspell type" unless dic_path
799
+
800
+ # Replace .dic extension with .aff for affix file
801
+ aff_path = dic_path.sub(/\.dic$/i, ".aff")
802
+
803
+ raise DictionaryNotFoundError, dic_path unless File.exist?(dic_path)
804
+ raise DictionaryNotFoundError, aff_path unless File.exist?(aff_path)
805
+
806
+ Dictionary::Hunspell.new(
807
+ dic_path: dic_path,
808
+ aff_path: aff_path,
809
+ language_code: @language,
810
+ locale: @locale
811
+ )
812
+ end
813
+
814
+ # Load CSpell dictionary.
815
+ def load_cspell_dictionary
816
+ path = @dictionary_path
817
+
818
+ raise ConfigurationError, "dictionary_path is required for cspell type" unless path
819
+ raise DictionaryNotFoundError, path unless File.exist?(path)
820
+
821
+ Dictionary::CSpell.new(
822
+ path,
823
+ language_code: @language,
824
+ locale: @locale,
825
+ case_sensitive: @case_sensitive
826
+ )
827
+ end
828
+ end
829
+ end