kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "kotoshu"
4
+ require "rake"
5
+ require "kotoshu/tasks/check_task"
6
+
7
+ # Wiring for `require "kotoshu/tasks"` (plan 89): installs the
8
+ # default `rake kotoshu:check` over the repository text files using
9
+ # the plan-88 file selection. Configure or add more tasks with
10
+ # Kotoshu::Tasks::CheckTask.new.
11
+ Kotoshu::Tasks::CheckTask.new
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Framework integrations for validation layers (plan 89).
5
+ #
6
+ # Soft dependency: ActiveModel is NOT in kotoshu.gemspec. The
7
+ # validator file soft-requires it exactly like Models::OnnxModel
8
+ # soft-requires onnxruntime (see CLAUDE.md); when it is absent the
9
+ # class still loads but raises a caller-friendly error on use.
10
+ module Validators
11
+ # True when ActiveModel was requireable at load time.
12
+ ACTIVE_MODEL_LOADED = begin
13
+ require "active_model"
14
+ true
15
+ rescue LoadError
16
+ false
17
+ end
18
+
19
+ # Raised when the validator is used without ActiveModel.
20
+ class ActiveModelUnavailable < Kotoshu::Error
21
+ def initialize
22
+ super("Kotoshu spelling validation needs ActiveModel. " \
23
+ "Add gem 'activemodel' (or the full Rails stack) to your Gemfile.")
24
+ end
25
+ end
26
+
27
+ if ACTIVE_MODEL_LOADED
28
+ # ActiveModel EachValidator that fails validation on
29
+ # misspelled content.
30
+ #
31
+ # One error is added per misspelling, with the top suggestion
32
+ # in the message.
33
+ #
34
+ # @example Wired for the validates short syntax
35
+ # # Rails initializer or the model file:
36
+ # SpellingValidator = Kotoshu::Validators::SpellingValidator
37
+ #
38
+ # class Post < ApplicationRecord
39
+ # validates :body, spelling: true
40
+ # validates :summary, spelling: { language: :en,
41
+ # personal_words: %w[kotoshu] }
42
+ # end
43
+ #
44
+ # @example Without the short syntax
45
+ # validates_with Kotoshu::Validators::SpellingValidator,
46
+ # attributes: [:body]
47
+ #
48
+ # Options:
49
+ # - language: language code; defaults to auto-detection.
50
+ # - personal_words: array of words accepted in addition to the
51
+ # dictionary.
52
+ class SpellingValidator < ActiveModel::EachValidator
53
+ def validate_each(record, attribute, value)
54
+ return if value.nil? || value.empty?
55
+
56
+ result = Kotoshu.check(value, language: language)
57
+ errors = result.errors.reject { |error| personal_words.include?(error.word.downcase) }
58
+ errors.each do |error|
59
+ record.errors.add(attribute, :spelling, **error_attributes(error))
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ # The language to check with: the option value, or
66
+ # auto-detection when unset.
67
+ #
68
+ # @return [String, Symbol, nil]
69
+ def language
70
+ options[:language]
71
+ end
72
+
73
+ # Words accepted on top of the dictionary, downcased.
74
+ #
75
+ # @return [Array<String>]
76
+ def personal_words
77
+ @personal_words ||= Array(options[:personal_words]).map(&:downcase)
78
+ end
79
+
80
+ # Build the error details for one misspelling.
81
+ #
82
+ # @param error [Models::Result::WordResult] the misspelling
83
+ # @return [Hash] message and metadata
84
+ def error_attributes(error)
85
+ suggestion = error.top_suggestions(1).first
86
+ message = "'#{error.word}' is misspelled"
87
+ message += " - did you mean '#{suggestion}'?" if suggestion
88
+ { message: message, word: error.word, suggestions: error.top_suggestions(3) }
89
+ end
90
+ end
91
+ else
92
+ # Placeholder raising a friendly error when ActiveModel is
93
+ # missing; keeps the constant resolvable for tooling.
94
+ class SpellingValidator
95
+ def initialize(*)
96
+ raise ActiveModelUnavailable
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Validation-layer integrations (plan 89). Soft dependency on
5
+ # ActiveModel — see Validators::SpellingValidator.
6
+ module Validators
7
+ autoload :SpellingValidator, "kotoshu/validators/spelling_validator"
8
+ end
9
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ VERSION = '1.0.2'
5
+ end
data/lib/kotoshu.rb ADDED
@@ -0,0 +1,538 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Kotoshu is a semantic spell checker for Ruby.
4
+ #
5
+ # Public API entry point. All internal namespaces are autoloaded on first
6
+ # reference; loading this file is cheap and triggers no heavy work.
7
+ #
8
+ # Exception: version.rb is required because autoload only works for module
9
+ # constants, and Bundler/gemspec also reads Kotoshu::VERSION from it.
10
+ require_relative "kotoshu/version"
11
+
12
+ module Kotoshu
13
+ # ---- Namespaces (autoloaded; each namespace file declares its children) ----
14
+ autoload :Algorithms, "kotoshu/algorithms"
15
+ autoload :Analyzers, "kotoshu/analyzers"
16
+ autoload :Baseline, "kotoshu/baseline"
17
+ autoload :Cache, "kotoshu/cache"
18
+ autoload :Cli, "kotoshu/cli"
19
+ autoload :Components, "kotoshu/components"
20
+ autoload :Configuration, "kotoshu/configuration"
21
+ autoload :ConformanceExporter, "kotoshu/conformance_exporter"
22
+ autoload :ConformanceRunner, "kotoshu/conformance_runner"
23
+ autoload :Core, "kotoshu/core"
24
+ autoload :Data, "kotoshu/data"
25
+ autoload :DataStructures, "kotoshu/data_structures"
26
+ autoload :Defaults, "kotoshu/defaults"
27
+ autoload :Dictionaries, "kotoshu/dictionaries"
28
+ autoload :Dictionary, "kotoshu/dictionary"
29
+ autoload :Documents, "kotoshu/documents"
30
+ autoload :Embeddings, "kotoshu/embeddings"
31
+ autoload :FluentChecker, "kotoshu/fluent_checker"
32
+ autoload :Grammar, "kotoshu/grammar"
33
+ autoload :Integrity, "kotoshu/integrity"
34
+ autoload :Keyboard, "kotoshu/keyboard"
35
+ autoload :Language, "kotoshu/language"
36
+ autoload :Languages, "kotoshu/languages"
37
+ autoload :Models, "kotoshu/models"
38
+ autoload :Paths, "kotoshu/paths"
39
+ autoload :PersonalDictionary, "kotoshu/personal_dictionary"
40
+ autoload :Plugins, "kotoshu/plugins"
41
+ autoload :ProjectConfig, "kotoshu/project_config"
42
+ autoload :MultiLanguageChecker, "kotoshu/multi_language_checker"
43
+ # Native engine (Rust extension loader + adapter; plan 66). Required via
44
+ # autoload so a missing extension can never break `require "kotoshu"`.
45
+ autoload :Native, "kotoshu/native"
46
+ autoload :NativeBackend, "kotoshu/native_backend"
47
+ autoload :Readers, "kotoshu/readers"
48
+ autoload :ResourceBundle, "kotoshu/resource_bundle"
49
+ autoload :ResourceManager, "kotoshu/resource_manager"
50
+ autoload :Results, "kotoshu/results"
51
+ autoload :SourceRegistry, "kotoshu/source_registry"
52
+ autoload :Spellchecker, "kotoshu/spellchecker"
53
+ autoload :StringMetrics, "kotoshu/string_metrics"
54
+ autoload :Suggestions, "kotoshu/suggestions"
55
+ # Framework integrations (plan 89): soft dependencies, autoloaded
56
+ # only on reference. The rake task is require-only instead:
57
+ # `require "kotoshu/tasks"` (it defines a task on load, so an
58
+ # autoload would surprise).
59
+ autoload :Validators, "kotoshu/validators"
60
+ autoload :Rspec, "kotoshu/rspec"
61
+ autoload :Jekyll, "kotoshu/jekyll"
62
+
63
+ # ---- Top-level error classes (all defined in core/exceptions.rb) ----
64
+ autoload :Error, "kotoshu/core/exceptions"
65
+ autoload :DictionaryNotFoundError, "kotoshu/core/exceptions"
66
+ autoload :InvalidDictionaryFormatError, "kotoshu/core/exceptions"
67
+ autoload :ConfigurationError, "kotoshu/core/exceptions"
68
+ autoload :SpellcheckError, "kotoshu/core/exceptions"
69
+ autoload :AffixRuleError, "kotoshu/core/exceptions"
70
+ autoload :ResourceNotCachedError, "kotoshu/core/exceptions"
71
+ autoload :ResourceNotSetupError, "kotoshu/core/exceptions"
72
+ autoload :ResourceResolutionError, "kotoshu/core/exceptions"
73
+ autoload :IntegrityError, "kotoshu/core/exceptions"
74
+ autoload :SuikaUnavailable, "kotoshu/core/exceptions"
75
+
76
+ # ---- Lazily-loaded singletons (not in their own namespace file) ----
77
+ autoload :Debug, "kotoshu/debug_mode"
78
+ autoload :DebugLogger, "kotoshu/debug_logger"
79
+ autoload :Metrics, "kotoshu/metrics_module"
80
+ autoload :MetricsCollector, "kotoshu/metrics_collector"
81
+ end
82
+
83
+ module Kotoshu
84
+ # Global configuration instance.
85
+ #
86
+ # @return [Configuration] The global configuration
87
+ #
88
+ # @example
89
+ # Kotoshu.configure do |config|
90
+ # config.dictionary_path = "/usr/share/dict/words"
91
+ # config.language = "en-US"
92
+ # end
93
+ def self.configure
94
+ yield configuration if block_given?
95
+ configuration
96
+ end
97
+
98
+ # Get the global configuration.
99
+ #
100
+ # @return [Configuration] The global configuration
101
+ #
102
+ # @example
103
+ # config = Kotoshu.configuration
104
+ def self.configuration
105
+ Configuration.instance
106
+ end
107
+
108
+ # Replace the global configuration. Useful for tests and for
109
+ # embedding kotoshu in a larger application that wants to share a
110
+ # Configuration across multiple Spellcheckers without each one
111
+ # independently touching the singleton.
112
+ #
113
+ # The argument must be a Configuration instance (type-checked so a
114
+ # typo doesn't silently flip the global to nil and break every
115
+ # downstream lookup).
116
+ #
117
+ # @param config [Configuration]
118
+ # @return [Configuration] the same instance, for chaining
119
+ #
120
+ # @example Inject a test-only configuration
121
+ # Kotoshu.configuration = Configuration.new.tap { |c| c.cache_ttl = 1 }
122
+ def self.configuration=(config)
123
+ raise ArgumentError, "configuration must be a Configuration instance" unless config.is_a?(Configuration)
124
+
125
+ Configuration.instance = config
126
+ end
127
+
128
+ # Default spellchecker (singleton). Uses the configured default language.
129
+ # Cache-only — raises ResourceNotSetupError if the default language hasn't
130
+ # been set up via Kotoshu.setup.
131
+ #
132
+ # @return [Spellchecker] The default spellchecker
133
+ # @raise [ResourceNotSetupError] if no language is set up
134
+ def self.spellchecker
135
+ return @spellchecker if @spellchecker
136
+
137
+ lang = configuration.default_language
138
+ raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil? || lang.to_s.empty?
139
+
140
+ @spellchecker = spellchecker_for(lang)
141
+ end
142
+
143
+ # Get a spellchecker for a specific language (cache-only, raises on miss).
144
+ #
145
+ # @param language [String, Symbol] Language code (e.g., "en", "de", "fr")
146
+ # @return [Spellchecker] Spellchecker using a ResourceManager-resolved bundle
147
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
148
+ #
149
+ # @example
150
+ # Kotoshu.setup(:de)
151
+ # Kotoshu.spellchecker_for("de").correct?("Hallo") # => true
152
+ def self.spellchecker_for(language)
153
+ key = language.to_s
154
+ @spellcheckers ||= {}
155
+ @spellcheckers[key] ||= begin
156
+ bundle = ResourceManager.resolve(language: language)
157
+ Spellchecker.new(resource_bundle: bundle, config: configuration)
158
+ end
159
+ end
160
+
161
+ # Resolve language resources from the cache (no download).
162
+ #
163
+ # @param language [String, Symbol, nil] Language code; if nil, uses default
164
+ # @param want [Array<Symbol>] Resource types (default: [:spelling])
165
+ # @param tier [Symbol, nil] Model tier to resolve (:full default,
166
+ # :fluency, :mini, or :any for "the single cached tier")
167
+ # @return [ResourceBundle] Resolved bundle
168
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
169
+ #
170
+ # @example
171
+ # Kotoshu.setup(:en)
172
+ # bundle = Kotoshu.resolve(language: "en")
173
+ # bundle.dictionary # => #<Dictionary::Hunspell ...>
174
+ def self.resolve(language: nil, want: nil, tier: nil)
175
+ lang = language || configuration.default_language
176
+ raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil?
177
+
178
+ want_param = want || ResourceManager::DEFAULT_WANT
179
+ ResourceManager.resolve(language: lang, want: want_param, tier: tier)
180
+ end
181
+
182
+ # ---- Stage 1: Setup ----
183
+
184
+ # Set up resources for one or more languages (download or register local files).
185
+ # Idempotent: re-running with the same args is a no-op unless `force: true`.
186
+ #
187
+ # @param languages [String, Symbol, Array<String, Symbol>] One or more language codes
188
+ # @param want [Array<Symbol>] Resource types to fetch (default: [:spelling])
189
+ # @param opts [Hash] Additional keyword options, all forwarded to
190
+ # ResourceManager.setup
191
+ # @option opts [Boolean] :force Re-fetch even if already cached
192
+ # @option opts [Boolean] :strict Re-raise on optional-resource failure
193
+ # @option opts [Symbol, String] :tier Model tier when want includes :model
194
+ # (:full default, :fluency, :mini)
195
+ # @option opts [String, nil] :aff Path to local .aff file (single-language only)
196
+ # @option opts [String, nil] :dic Path to local .dic file (single-language only)
197
+ # @option opts [String, nil] :from Directory containing local .aff/.dic
198
+ # (single-language only)
199
+ # @option opts [String, nil] :frequency Path to local frequency.json
200
+ # (single-language only)
201
+ # @return [SetupResult, Array<SetupResult>] Result or results (array if multiple languages)
202
+ #
203
+ # @example Download from kotoshu/dictionaries
204
+ # Kotoshu.setup(:en) # spelling only
205
+ # Kotoshu.setup(:en, want: %i[spelling frequency]) # spelling + frequency
206
+ # Kotoshu.setup(:en, want: %i[model], tier: :mini) # mini-tier model
207
+ # Kotoshu.setup(:en, :de, :fr) # multiple languages
208
+ #
209
+ # @example Register local files (user already has hunspell dicts)
210
+ # Kotoshu.setup(:en, aff: "/usr/share/hunspell/en_US.aff",
211
+ # dic: "/usr/share/hunspell/en_US.dic")
212
+ #
213
+ # @example Register local files from a directory
214
+ # Kotoshu.setup(:en, from: "/usr/share/hunspell/") # looks for en.aff, en.dic
215
+ def self.setup(*languages, want: nil, **opts)
216
+ raise ArgumentError, "Kotoshu.setup requires at least one language" if languages.empty?
217
+
218
+ want_param = want || ResourceManager::DEFAULT_WANT
219
+ if languages.size == 1
220
+ ResourceManager.setup(languages.first, want: want_param, **opts)
221
+ else
222
+ languages.map { |lang| ResourceManager.setup(lang, want: want_param, **opts) }
223
+ end
224
+ end
225
+
226
+ # Check if a language (or a specific resource for that language) is set up.
227
+ #
228
+ # @param language [String, Symbol] Language code
229
+ # @param resource [Symbol, nil] :spelling, :frequency, :model, or nil for any
230
+ # @param tier [Symbol, nil] Model tier to check (:full, :fluency, :mini,
231
+ # or :any for "any tier cached"); only meaningful with resource: :model
232
+ # @return [Boolean] True if the resource is cached and available
233
+ #
234
+ # @example
235
+ # Kotoshu.setup(:en)
236
+ # Kotoshu.setup?(:en) # => true
237
+ # Kotoshu.setup?(:en, :spelling) # => true
238
+ # Kotoshu.setup?(:en, :frequency) # => false (not set up)
239
+ # Kotoshu.setup?(:en, resource: :model, tier: :mini) # => false
240
+ def self.setup?(language, resource = nil, tier: nil, **options)
241
+ # The resource may be given positionally (legacy) or as a keyword.
242
+ resource = options.fetch(:resource, resource)
243
+ ResourceManager.setup?(language, resource: resource, tier: tier)
244
+ end
245
+
246
+ # List languages that have been set up.
247
+ #
248
+ # @return [Array<String>] Sorted array of language codes with cached spelling
249
+ #
250
+ # @example
251
+ # Kotoshu.languages_setup # => ["de", "en", "fr"]
252
+ def self.languages_setup
253
+ ResourceManager.languages_setup
254
+ end
255
+
256
+ # Reset the spellchecker cache. The next call to `spellchecker` or
257
+ # `spellchecker_for` re-resolves from the current configuration.
258
+ #
259
+ # Does NOT eagerly reload — clearing the cache is enough. This makes
260
+ # the method safe to call between tests even when no language is set
261
+ # up yet (the next call will raise ResourceNotSetupError per the
262
+ # strict two-stage contract).
263
+ def self.reset_spellchecker
264
+ @spellchecker = nil
265
+ @spellcheckers = nil
266
+ nil
267
+ end
268
+
269
+ # Check if a word is spelled correctly.
270
+ # Hot path — cache-only, raises if language not set up.
271
+ #
272
+ # @param word [String] The word to check
273
+ # @param language [String, Symbol, nil] Language code; if nil, uses configured default
274
+ # @return [Boolean] True if the word is correct
275
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
276
+ #
277
+ # @example
278
+ # Kotoshu.setup(:en)
279
+ # Kotoshu.correct?("hello") # => true
280
+ # Kotoshu.correct?("Hallo", language: "de") # requires Kotoshu.setup(:de) first
281
+ def self.correct?(word, language: nil)
282
+ checker = language ? spellchecker_for(language) : spellchecker
283
+ checker.correct?(word)
284
+ end
285
+
286
+ # Check if a word is misspelled. Hot path.
287
+ #
288
+ # @param word [String] The word to check
289
+ # @param language [String, Symbol, nil] Language code
290
+ # @return [Boolean] True if the word is misspelled
291
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
292
+ def self.misspelled?(word, language: nil)
293
+ !correct?(word, language: language)
294
+ end
295
+
296
+ # Get spelling suggestions for a word. Hot path.
297
+ #
298
+ # @param word [String] The misspelled word
299
+ # @param language [String, Symbol, nil] Language code
300
+ # @param options [Hash] Options (max_suggestions, etc.)
301
+ # @return [Suggestions::SuggestionSet] Generated suggestions
302
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
303
+ #
304
+ # @example
305
+ # Kotoshu.setup(:en)
306
+ # suggestions = Kotoshu.suggest("helo")
307
+ # suggestions.to_words # => ["hello", "help", "held", ...]
308
+ def self.suggest(word, language: nil, **options)
309
+ checker = language ? spellchecker_for(language) : spellchecker
310
+ checker.suggest(word, **options)
311
+ end
312
+
313
+ # Check text for spelling errors. Hot path.
314
+ #
315
+ # @param text [String] The text to check
316
+ # @param language [String, Symbol, nil] Language code; if nil, uses configured default
317
+ # @param _options [Hash] Accepted and ignored — kept so call sites can
318
+ # pass extra options without breaking
319
+ # @return [Models::Result::DocumentResult] The check result
320
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
321
+ #
322
+ # @example
323
+ # Kotoshu.setup(:en)
324
+ # result = Kotoshu.check("Hello wrold")
325
+ # result.errors.map(&:word) # => ["wrold"]
326
+ def self.check(text, language: nil, suggestions_filter: nil, **_options)
327
+ checker = language ? spellchecker_for(language) : spellchecker
328
+ checker.check(text, suggestions_filter: suggestions_filter)
329
+ end
330
+
331
+ # Check a file for spelling errors. Hot path.
332
+ #
333
+ # @param path [String] The file path
334
+ # @param language [String, Symbol, nil] Language code
335
+ # @param _options [Hash] Accepted and ignored — kept so call sites can
336
+ # pass extra options without breaking
337
+ # @return [Models::Result::DocumentResult] The check result
338
+ # @raise [ResourceNotSetupError] if the language hasn't been set up
339
+ #
340
+ # @example
341
+ # Kotoshu.setup(:en)
342
+ # result = Kotoshu.check_file("README.md")
343
+ # result.success? # => false
344
+ def self.check_file(path, language: nil, **_options)
345
+ checker = language ? spellchecker_for(language) : spellchecker
346
+ checker.check_file(path)
347
+ end
348
+
349
+ # Check multiple files for spelling errors.
350
+ #
351
+ # @param paths [Array<String>] The file paths
352
+ # @param options [Hash] Options
353
+ # @return [Array<Models::Result::DocumentResult>] Results for each file
354
+ #
355
+ # @example
356
+ # results = Kotoshu.check_files(%w[README.md CHANGELOG.md])
357
+ # results.select(&:failed?)
358
+ def self.check_files(paths, **options)
359
+ paths.map { |path| check_file(path, **options) }
360
+ end
361
+
362
+ # Convenience method for creating an indexed dictionary.
363
+ #
364
+ # @param source [Array<String>, Hash, nil] Words or file path
365
+ # @return [Core::IndexedDictionary] New dictionary
366
+ def self.dictionary(source = nil)
367
+ case source
368
+ when Array
369
+ Core::IndexedDictionary.new(source)
370
+ when String
371
+ Core::IndexedDictionary.from_file(source)
372
+ when nil, Hash
373
+ Core::IndexedDictionary.new
374
+ else
375
+ raise ArgumentError, "Invalid dictionary source: #{source.inspect}"
376
+ end
377
+ end
378
+
379
+ # Convenience method for creating a trie.
380
+ #
381
+ # @param source [Array<String>, String, nil] Words or file path
382
+ # @return [Core::Trie::Trie] New trie
383
+ def self.trie(source = nil)
384
+ case source
385
+ when Array
386
+ Core::Trie::Builder.from_array(source)
387
+ when String
388
+ if File.exist?(source)
389
+ Core::Trie::Builder.from_file(source)
390
+ else
391
+ Core::Trie::Builder.from_string(source)
392
+ end
393
+ when nil
394
+ Core::Trie::Trie.new
395
+ else
396
+ raise ArgumentError, "Invalid trie source: #{source.inspect}"
397
+ end
398
+ end
399
+
400
+ # Convenience method for creating a suggestion pipeline.
401
+ #
402
+ # @param strategies [Array] Optional strategies to add
403
+ # @return [Suggestions::Strategies::CompositeStrategy] New pipeline
404
+ def self.suggestion_pipeline(*strategies)
405
+ pipeline = Suggestions::Strategies::CompositeStrategy.new(name: :default)
406
+ strategies.each { |s| pipeline.add(s) }
407
+ pipeline
408
+ end
409
+
410
+ # Register a custom dictionary type.
411
+ #
412
+ # @param type [Symbol] The type key
413
+ # @param klass [Class] The dictionary class
414
+ #
415
+ # @example
416
+ # Kotoshu.register_dictionary_type(:my_custom, MyDictionary)
417
+ def self.register_dictionary_type(type, klass)
418
+ Dictionary.register_type(type, klass)
419
+ end
420
+
421
+ # Register a custom suggestion algorithm.
422
+ #
423
+ # @param name [Symbol] The algorithm name
424
+ # @param klass [Class] The algorithm class
425
+ #
426
+ # @example
427
+ # Kotoshu.register_suggestion_algorithm(:my_custom, MyStrategy)
428
+ def self.register_suggestion_algorithm(name, klass)
429
+ Suggestions::Strategies::BaseStrategy.register_type(name, klass)
430
+ end
431
+
432
+ # Access the language module.
433
+ #
434
+ # @return [Module] The Language module
435
+ #
436
+ # @example
437
+ # Kotoshu::Language.detect("Hello world") # => "en"
438
+ def self.language
439
+ Language
440
+ end
441
+
442
+ # Detect language of text.
443
+ #
444
+ # Native LID when available (plan 106): the Rust extension scoring
445
+ # the `kotoshu://models/lid/lid-176` artifact pair over 176
446
+ # languages, after a one-time {setup_lid}. Falls back to the
447
+ # {Language::Detector} heuristic when the extension is absent,
448
+ # KOTOSHU_BACKEND=ruby, or the model is not set up — so pure-Ruby
449
+ # installs keep the previous behavior.
450
+ #
451
+ # @param text [String] Text to analyze
452
+ # @return [Language::Detection] +code+ (String, or nil when the
453
+ # heuristic is uncertain) and +score+ (Float in [0, 1]);
454
+ # interpolation yields the code
455
+ #
456
+ # @example
457
+ # Kotoshu.setup_lid
458
+ # Kotoshu.detect_language("Bonjour le monde").code # => "fr"
459
+ # Kotoshu.detect_language("こんにちは").score # => 0.99
460
+ def self.detect_language(text)
461
+ Language::LidDetector.detect(text)
462
+ end
463
+
464
+ # Detect language with confidence score.
465
+ #
466
+ # Same engine selection as {detect_language}, returned as the
467
+ # historical [code, confidence] pair.
468
+ #
469
+ # @param text [String] Text to analyze
470
+ # @return [Array<String, Float>] Language code and confidence
471
+ #
472
+ # @example
473
+ # lang, conf = Kotoshu.detect_language_with_confidence("Hello world")
474
+ # lang # => "en"
475
+ # conf # => 0.85
476
+ def self.detect_language_with_confidence(text)
477
+ detection = Language::LidDetector.detect(text)
478
+ [detection.code, detection.score]
479
+ end
480
+
481
+ # Set up the language-identification model: downloads the
482
+ # lid.176.onnx + vocab.json pair from the models registry into the
483
+ # model cache (stage one of the two-stage model — {detect_language}
484
+ # never downloads implicitly). Idempotent; +force+ re-downloads.
485
+ #
486
+ # @param force [Boolean] Re-fetch even if already cached
487
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
488
+ # @raise [Kotoshu::Error] registry unreachable while offline with no
489
+ # cached registry (KOTOSHU_OFFLINE=1), or no registry entry
490
+ # @raise [Kotoshu::IntegrityError] downloaded bytes fail the
491
+ # registry checksum
492
+ #
493
+ # @example
494
+ # Kotoshu.setup_lid
495
+ # Kotoshu.setup_lid? # => true
496
+ def self.setup_lid(force: false)
497
+ Language::LidDetector.setup(config: configuration, force: force)
498
+ end
499
+
500
+ # Whether the language-identification model is cached (cache-only).
501
+ #
502
+ # @return [Boolean]
503
+ def self.setup_lid?
504
+ Language::LidDetector.setup?(config: configuration)
505
+ end
506
+
507
+ # Get language class by code.
508
+ #
509
+ # @param code [String] Language code (e.g., "en-US", "de-DE")
510
+ # @return [Class, nil] Language class or nil
511
+ #
512
+ # @example
513
+ # Kotoshu.get_language("en-US")
514
+ def self.get_language(code)
515
+ Language.get(code)
516
+ end
517
+
518
+ # Check if a language is registered.
519
+ #
520
+ # @param code [String] Language code
521
+ # @return [Boolean] True if registered
522
+ #
523
+ # @example
524
+ # Kotoshu.language_registered?("en-US") # => true or false
525
+ def self.language_registered?(code)
526
+ Language.registered?(code)
527
+ end
528
+
529
+ # Get all supported language codes.
530
+ #
531
+ # @return [Array<String>] List of language codes
532
+ #
533
+ # @example
534
+ # Kotoshu.supported_languages # => ["de-DE", "en-US", "fr-FR", ...]
535
+ def self.supported_languages
536
+ Language.supported_codes
537
+ end
538
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rb_sys/extensiontask"
4
+
5
+ # Load gemspec directly if GEMSPEC constant is not defined
6
+ gemspec = defined?(GEMSPEC) ? GEMSPEC : Gem::Specification.load("kotoshu.gemspec")
7
+
8
+ RbSys::ExtensionTask.new("kotoshu_native", gemspec) do |ext|
9
+ ext.lib_dir = "lib/kotoshu"
10
+ end