kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Single source of truth for where each remote resource lives.
5
+ #
6
+ # Every URL the cache layer fetches is built here. Caches do not
7
+ # construct URL strings inline. Per-repo pins honor that
8
+ # `kotoshu/dictionaries` ships on `v1` while the other repos are on
9
+ # `main`, which previously caused silent 404s on first-use.
10
+ #
11
+ # @example
12
+ # registry = Kotoshu::SourceRegistry.new
13
+ # registry.url_for(:spelling, lang: "en", ext: "aff")
14
+ # # => "https://raw.githubusercontent.com/kotoshu/dictionaries/v1/en/spelling/index.aff"
15
+ class SourceRegistry
16
+ Source = Struct.new(:repo, :default_pin, :template, :lfs, keyword_init: true)
17
+
18
+ DEFAULT_BASE_URL = "https://raw.githubusercontent.com/kotoshu"
19
+
20
+ # Host serving Git-LFS object content. raw.githubusercontent.com serves
21
+ # the 134-byte pointer stub for LFS-tracked files, so LFS sources
22
+ # (*.onnx in models-fasttext-onnx) must be fetched from here.
23
+ MEDIA_BASE_URL = "https://media.githubusercontent.com/media/kotoshu"
24
+
25
+ # @return [Hash<Symbol, Source>]
26
+ SOURCES = {
27
+ spelling: Source.new(repo: "dictionaries", default_pin: "v1",
28
+ template: "dictionaries/%<pin>s/%<lang>s/spelling/index.%<ext>s"),
29
+ # Flat layout: only en ships under the spelling/ sublayout; the
30
+ # staged languages sit at {lang}/index.*. The cache layer tries
31
+ # spelling first and falls back to this.
32
+ spelling_flat: Source.new(repo: "dictionaries", default_pin: "v1",
33
+ template: "dictionaries/%<pin>s/%<lang>s/index.%<ext>s"),
34
+ grammar: Source.new(repo: "dictionaries", default_pin: "v1",
35
+ template: "dictionaries/%<pin>s/%<lang>s/grammar/rules.yaml"),
36
+ dict_manifest: Source.new(repo: "dictionaries", default_pin: "v1",
37
+ template: "dictionaries/%<pin>s/manifest.json"),
38
+ frequency: Source.new(repo: "frequency-list-kelly", default_pin: "main",
39
+ template: "frequency-list-kelly/%<pin>s/data/%<lang>s.json"),
40
+ freq_manifest: Source.new(repo: "frequency-list-kelly", default_pin: "main",
41
+ template: "frequency-list-kelly/%<pin>s/manifest.json"),
42
+ model: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
43
+ template: "models-fasttext-onnx/%<pin>s/models/%<lang>s/fasttext.%<lang>s.onnx",
44
+ lfs: true),
45
+ model_vocab: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
46
+ template: "models-fasttext-onnx/%<pin>s/models/%<lang>s/fasttext.%<lang>s.vocab.json"),
47
+ model_manifest: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
48
+ template: "models-fasttext-onnx/%<pin>s/manifest.json"),
49
+ model_registry: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
50
+ template: "models-fasttext-onnx/%<pin>s/registry.json")
51
+ }.freeze
52
+
53
+ # @param base_url [String] GitHub raw root, no trailing slash.
54
+ # @param pins [Hash<String, String>] Optional per-repo pin overrides
55
+ # keyed by repo name (e.g. `{ "dictionaries" => "v2" }`).
56
+ def initialize(base_url: DEFAULT_BASE_URL, pins: {})
57
+ @base_url = base_url.to_s.chomp("/")
58
+ @pins = pins.transform_keys(&:to_s).freeze
59
+ end
60
+
61
+ # @return [String] Configured GitHub raw root (no trailing slash).
62
+ attr_reader :base_url
63
+
64
+ # @param source_key [Symbol] One of `SOURCES.keys`.
65
+ # @param lang [String, nil] Language code, interpolated into template.
66
+ # @param ext [String, nil] File extension, interpolated into template.
67
+ # @return [String] Fully-qualified URL.
68
+ def url_for(source_key, lang: nil, ext: nil)
69
+ source = SOURCES.fetch(source_key) do
70
+ raise ArgumentError, "unknown source: #{source_key.inspect}"
71
+ end
72
+ path = source.template % { pin: pin_for(source), lang: lang, ext: ext }
73
+ "#{base_for(source)}/#{path}"
74
+ end
75
+
76
+ # @param source_key [Symbol]
77
+ # @return [String] Resolved pin (override or default).
78
+ def pin_for_source(source_key)
79
+ source = SOURCES.fetch(source_key)
80
+ pin_for(source)
81
+ end
82
+
83
+ # @param source_key [Symbol]
84
+ # @return [String] Repo name (e.g. "dictionaries").
85
+ def repo_for(source_key)
86
+ SOURCES.fetch(source_key).repo
87
+ end
88
+
89
+ private
90
+
91
+ def pin_for(source)
92
+ @pins.fetch(source.repo, source.default_pin)
93
+ end
94
+
95
+ # Base URL for a source. LFS-tracked content is only served by the
96
+ # media host — but only when we are on the default GitHub host; a
97
+ # custom mirror base is left alone (the mirror resolves LFS itself).
98
+ def base_for(source)
99
+ return MEDIA_BASE_URL if source.lfs && @base_url == DEFAULT_BASE_URL
100
+
101
+ @base_url
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,90 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ class Spellchecker
5
+ # Parallel file checker for concurrent spellchecking.
6
+ #
7
+ # Uses a thread pool to check multiple files simultaneously,
8
+ # providing significant speedup on multi-core systems.
9
+ #
10
+ # @example Check files in parallel
11
+ # checker = ParallelChecker.new(spellchecker: spellchecker, worker_count: 4)
12
+ # results = checker.check_files_parallel(["file1.txt", "file2.txt"])
13
+ class ParallelChecker
14
+ # Default number of worker threads
15
+ DEFAULT_WORKER_COUNT = 4
16
+
17
+ # @return [Spellchecker] The underlying spellchecker
18
+ attr_reader :spellchecker
19
+
20
+ # @return [Integer] Number of worker threads
21
+ attr_reader :worker_count
22
+
23
+ # Create a new parallel checker.
24
+ #
25
+ # @param spellchecker [Spellchecker] The spellchecker to use
26
+ # @param worker_count [Integer] Number of worker threads (default: 4)
27
+ def initialize(spellchecker:, worker_count: DEFAULT_WORKER_COUNT)
28
+ @spellchecker = spellchecker
29
+ @worker_count = worker_count
30
+ @queue = Queue.new
31
+ @results = []
32
+ @mutex = Mutex.new
33
+ end
34
+
35
+ # Check multiple files in parallel.
36
+ #
37
+ # @param file_paths [Array<String>] Paths to files to check
38
+ # @return [Array<Core::Models::Result::DocumentResult>] Results for each file
39
+ def check_files_parallel(file_paths)
40
+ return [] if file_paths.empty?
41
+
42
+ # Add all files to the queue
43
+ file_paths.each { |path| @queue << path }
44
+
45
+ # Add poison pills to signal workers to stop
46
+ @worker_count.times { @queue << :done }
47
+
48
+ # Create and start workers
49
+ workers = @worker_count.times.map { create_worker }
50
+
51
+ # Wait for all workers to complete
52
+ workers.each(&:join)
53
+
54
+ # Clear queue for reuse
55
+ @queue.clear while @queue.empty? == false
56
+
57
+ @results
58
+ end
59
+
60
+ # Check a single file (convenience method).
61
+ #
62
+ # @param file_path [String] Path to file
63
+ # @return [Core::Models::Result::DocumentResult] Check result
64
+ def check_file(file_path)
65
+ @spellchecker.check_file(file_path)
66
+ end
67
+
68
+ private
69
+
70
+ # Create a worker thread.
71
+ #
72
+ # @return [Thread] Worker thread
73
+ def create_worker
74
+ Thread.new do
75
+ while (path = @queue.pop) != :done
76
+ begin
77
+ result = @spellchecker.check_file(path)
78
+ @mutex.synchronize do
79
+ @results << result
80
+ end
81
+ rescue StandardError => e
82
+ # Log error but continue processing other files
83
+ warn "Error checking file #{path}: #{e.message}"
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,406 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+
5
+ module Kotoshu
6
+ # Main spellchecker class.
7
+ #
8
+ # This is the primary facade for spell checking operations,
9
+ # providing methods to check words, text, and files.
10
+ #
11
+ # @example Creating a spellchecker with a dictionary
12
+ # dict = Kotoshu::Dictionary::UnixWords.new("/usr/share/dict/words", language_code: "en-US")
13
+ # spellchecker = Spellchecker.new(dictionary: dict)
14
+ # spellchecker.correct?("hello") # => true
15
+ #
16
+ # @example Using configuration
17
+ # spellchecker = Spellchecker.new(
18
+ # dictionary_path: "/usr/share/dict/words",
19
+ # language: "en-US"
20
+ # )
21
+ class Spellchecker
22
+ # @return [Suggestions::Generator] The suggestion generator
23
+ attr_reader :generator
24
+
25
+ # @return [Configuration] The configuration
26
+ attr_reader :config
27
+
28
+ # @return [ResourceBundle, nil] The resource bundle if provided
29
+ attr_reader :resource_bundle
30
+
31
+ # @return [NativeBackend, nil] The native engine, when backend
32
+ # selection activated one (see Configuration#backend /
33
+ # KOTOSHU_BACKEND). Nil means every call runs the pure-Ruby engine.
34
+ attr_reader :native_backend
35
+
36
+ # Create a new spellchecker.
37
+ #
38
+ # @param dictionary [Dictionary::Base, nil] The dictionary (optional)
39
+ # @param config [Configuration, Hash] Configuration or settings
40
+ # @param resource_bundle [ResourceBundle, nil] Pre-resolved resource bundle
41
+ # @param kwargs [Hash] Additional configuration options
42
+ #
43
+ # @example With dictionary
44
+ # spellchecker = Spellchecker.new(dictionary: dict)
45
+ #
46
+ # @example With resource bundle (0.2+)
47
+ # bundle = Kotoshu::ResourceManager.resolve(language: "en")
48
+ # spellchecker = Spellchecker.new(resource_bundle: bundle)
49
+ # spellchecker.correct?("hello") # => true
50
+ #
51
+ # @example With configuration hash
52
+ # spellchecker = Spellchecker.new(
53
+ # dictionary_path: "/usr/share/dict/words",
54
+ # language: "en-US"
55
+ # )
56
+ #
57
+ # @example With Configuration object
58
+ # config = Configuration.new(dictionary_path: "words.txt")
59
+ # spellchecker = Spellchecker.new(config: config)
60
+ def initialize(dictionary: nil, config: nil, resource_bundle: nil, **kwargs)
61
+ @resource_bundle = resource_bundle
62
+
63
+ if resource_bundle
64
+ dictionary ||= resource_bundle.dictionary
65
+ kwargs[:language] = resource_bundle.language unless kwargs.key?(:language)
66
+ end
67
+
68
+ if config.is_a?(Configuration)
69
+ @config = config
70
+ else
71
+ settings = kwargs.dup
72
+ settings[:dictionary_path] = dictionary.path if dictionary
73
+ @config = Configuration.new(settings)
74
+ end
75
+
76
+ @config.dictionary = dictionary if dictionary
77
+
78
+ dict = @config.dictionary
79
+ max_suggestions = @config.max_suggestions
80
+
81
+ @generator = Suggestions::Generator.new(
82
+ dict,
83
+ max_suggestions: max_suggestions,
84
+ algorithms: @config.suggestion_algorithms
85
+ )
86
+
87
+ # Backend selection (plan 66): when active, correct?/suggest run on
88
+ # the native engine; everything else keeps using @generator. The
89
+ # generator is always built — the native backend is an accelerator,
90
+ # never a replacement for the Ruby engine's non-hot-path surface.
91
+ @native_backend = NativeBackend.resolve(
92
+ dictionary: dict,
93
+ backend: @config.backend,
94
+ max_suggestions: max_suggestions
95
+ )
96
+
97
+ # Word extraction follows the configured language's script
98
+ # (plan 91): its tokenizer decides which letters form words,
99
+ # so Greek/Cyrillic words become checkable for el/uk while
100
+ # wrong-script words stay invisible. Nil keeps the fallback:
101
+ # ASCII letters and the apostrophe, the pre-plan-91 set (so
102
+ # unknown languages behave as before).
103
+ @language_tokenizer = resolve_language_tokenizer
104
+ @word_char_regex = @language_tokenizer&.spellcheck_word_regex || /[a-zA-Z']/
105
+ end
106
+
107
+ # Check if a word is spelled correctly.
108
+ #
109
+ # @param word [String] The word to check
110
+ # @return [Boolean] True if the word is correct
111
+ #
112
+ # @example
113
+ # spellchecker.correct?("hello") # => true
114
+ # spellchecker.correct?("helo") # => false
115
+ def correct?(word)
116
+ return false if word.nil? || word.empty?
117
+
118
+ return @native_backend.correct?(word) if @native_backend
119
+
120
+ @generator.correct?(word)
121
+ end
122
+
123
+ # Check if a word is misspelled.
124
+ #
125
+ # @param word [String] The word to check
126
+ # @return [Boolean] True if the word is misspelled
127
+ def incorrect?(word)
128
+ !correct?(word)
129
+ end
130
+
131
+ # Get spelling suggestions for a word.
132
+ #
133
+ # @param word [String] The misspelled word
134
+ # @param max_suggestions [Integer] Maximum suggestions (optional)
135
+ # @return [Suggestions::SuggestionSet] Generated suggestions
136
+ #
137
+ # @example
138
+ # suggestions = spellchecker.suggest("helo")
139
+ # suggestions.to_words # => ["hello", "help", "held", ...]
140
+ def suggest(word, max_suggestions: nil)
141
+ return Suggestions::SuggestionSet.empty if word.nil? || word.empty?
142
+
143
+ return @native_backend.suggest(word, max_suggestions: max_suggestions) if @native_backend
144
+
145
+ @generator.generate(word, max_suggestions: max_suggestions)
146
+ end
147
+
148
+ # Check a word and return a result object.
149
+ #
150
+ # @param word [String] The word to check
151
+ # @return [Models::Result::WordResult] The check result
152
+ #
153
+ # @example
154
+ # result = spellchecker.check_word("hello")
155
+ # result.correct? # => true
156
+ #
157
+ # @example With misspelled word
158
+ # result = spellchecker.check_word("helo")
159
+ # result.correct? # => false
160
+ # result.suggestions # => SuggestionSet with suggestions
161
+ def check_word(word)
162
+ if word.nil? || word.empty?
163
+ return Models::Result::WordResult.new(word: "", correct: false,
164
+ suggestions: Suggestions::SuggestionSet.empty)
165
+ end
166
+
167
+ if correct?(word)
168
+ Models::Result::WordResult.correct(word)
169
+ else
170
+ suggestions = suggest(word)
171
+ Models::Result::WordResult.incorrect(word, suggestions: suggestions)
172
+ end
173
+ end
174
+
175
+ # Check text for spelling errors.
176
+ #
177
+ # Words present in the user's personal dictionary
178
+ # (~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override)
179
+ # never surface as errors — the same semantics the LSP applies to
180
+ # its diagnostics (plan 105). They are dropped at result assembly
181
+ # without suppression metadata, still count toward +word_count+,
182
+ # and remain spellcheckable through {#suggest}. Opt out with
183
+ # {Configuration#personal_dictionary} (KOTOSHU_PERSONAL_DICTIONARY,
184
+ # `kotoshu check --no-personal`).
185
+ #
186
+ # @param text [String] The text to check
187
+ # @param suggestions [Boolean] Generate suggestions for errors
188
+ # (default true; false yields errors with empty suggestion
189
+ # sets — positions and counts are unaffected)
190
+ # @param suggestions_filter [#call, nil] Optional word predicate;
191
+ # when it answers false for a misspelled word, that occurrence
192
+ # skips suggestion generation. The CI baseline uses this to
193
+ # skip the sweep for occurrences it is about to absorb
194
+ # (plan 116) while new errors keep their suggestions.
195
+ # @return [Models::Result::DocumentResult] The check result
196
+ #
197
+ # @example
198
+ # result = spellchecker.check("Hello wrold")
199
+ # result.success? # => false
200
+ # result.errors.map(&:word) # => ["wrold"]
201
+ def check(text, suggestions: true, suggestions_filter: nil)
202
+ return Models::Result::DocumentResult.success if text.nil? || text.empty?
203
+
204
+ # Inline ignore directives (plan 82): suppressed errors move into
205
+ # suppressed_errors instead of failing the check. The :auto profile
206
+ # recognizes bare, Markdown HTML-comment, and AsciiDoc // directives
207
+ # because the facade sees raw text without a format parser.
208
+ words = tokenize(text)
209
+ suppressions = Documents::Suppressions.scan(text, format: :auto)
210
+ personal = personal_words
211
+ errors = []
212
+ suppressed_errors = []
213
+ words.each do |word, pos|
214
+ next if personal.include?(word.downcase)
215
+
216
+ correct = correct?(word)
217
+ next if correct
218
+
219
+ want_suggestions =
220
+ suggestions && (!suggestions_filter || suggestions_filter.call(word))
221
+ line = Documents::SourcePosition.line_for_offset(text, pos)
222
+ suppressed = suppressions.any? { |s| s.applies_to?(line, word: word) }
223
+ entry = Models::Result::WordResult.new(
224
+ word: word,
225
+ correct: false,
226
+ suggestions: (suggest(word) if want_suggestions) || Suggestions::SuggestionSet.empty,
227
+ position: pos,
228
+ suppressed: suppressed,
229
+ suppressed_by: (Models::Result::WordResult::SUPPRESSED_BY_INLINE if suppressed)
230
+ )
231
+ (suppressed ? suppressed_errors : errors) << entry
232
+ end
233
+ Models::Result::DocumentResult.new(
234
+ file: nil,
235
+ errors: errors,
236
+ suppressed_errors: suppressed_errors,
237
+ word_count: words.size
238
+ )
239
+ end
240
+
241
+ # Check a file for spelling errors.
242
+ #
243
+ # @param path [String] The file path
244
+ # @return [Models::Result::DocumentResult] The check result
245
+ #
246
+ # @example
247
+ # result = spellchecker.check_file("README.md")
248
+ # result.to_s # => "File 'README.md': 3 spelling error(s) found"
249
+ def check_file(path)
250
+ raise DictionaryNotFoundError, path unless File.exist?(path)
251
+
252
+ text = File.read(path, encoding: @config.encoding)
253
+ result = check(text)
254
+
255
+ # Create a new result with the file path
256
+ Models::Result::DocumentResult.new(
257
+ file: path,
258
+ errors: result.errors,
259
+ word_count: result.word_count
260
+ )
261
+ end
262
+
263
+ # Check a directory for spelling errors.
264
+ #
265
+ # @param path [String] The directory path
266
+ # @param pattern [String] File pattern to match (default: "*.txt")
267
+ # @return [Array<Models::Result::DocumentResult>] Results for each file
268
+ #
269
+ # @example
270
+ # results = spellchecker.check_directory("docs/")
271
+ # results.select(&:failed?).map(&:file)
272
+ def check_directory(path, pattern: "*.txt")
273
+ raise DictionaryNotFoundError, path unless File.exist?(path) && File.directory?(path)
274
+
275
+ files = Dir.glob(File.join(path, pattern))
276
+ files.map { |file| check_file(file) }
277
+ end
278
+
279
+ # Tokenize text into words.
280
+ #
281
+ # @param text [String] The text to tokenize
282
+ # @return [Array<Array>] Array of [word, position] pairs
283
+ #
284
+ # @example
285
+ # spellchecker.tokenize("Hello world!")
286
+ # # => [["Hello", 0], ["world", 6]]
287
+ def tokenize(text)
288
+ return [] if text.nil? || text.empty?
289
+
290
+ words = []
291
+ position = 0
292
+ word_buffer = String.new
293
+ word_start = 0
294
+
295
+ text.each_char.with_index do |char, i|
296
+ if word_char?(char)
297
+ word_buffer << char
298
+ word_start = i if word_buffer.length == 1
299
+ position = i
300
+ elsif !word_buffer.empty?
301
+ words << [word_buffer.dup.freeze, word_start]
302
+ word_buffer.clear
303
+ end
304
+ end
305
+
306
+ # Don't forget the last word
307
+ words << [word_buffer.dup.freeze, word_start] unless word_buffer.empty?
308
+
309
+ words
310
+ end
311
+
312
+ # Get the dictionary being used.
313
+ #
314
+ # @return [Dictionary::Base] The dictionary
315
+ def dictionary
316
+ @generator.dictionary
317
+ end
318
+
319
+ # Reload the dictionary.
320
+ #
321
+ # @return [self] Self for chaining
322
+ def reload_dictionary
323
+ @config.reset_dictionary
324
+
325
+ dict = @config.dictionary
326
+ @generator = Suggestions::Generator.new(
327
+ dict,
328
+ max_suggestions: @config.max_suggestions,
329
+ algorithms: @config.suggestion_algorithms
330
+ )
331
+ @native_backend = NativeBackend.resolve(
332
+ dictionary: dict,
333
+ backend: @config.backend,
334
+ max_suggestions: @config.max_suggestions
335
+ )
336
+
337
+ self
338
+ end
339
+
340
+ private
341
+
342
+ # The personal dictionary as a downcased Set for the check path
343
+ # (plan 105), mirroring the LSP's filter. Loaded once per process:
344
+ # the first {#check} call snapshots the words, and later edits to
345
+ # personal.dic take effect in the next process (the LSP reloads on
346
+ # mtime change; this path deliberately does not — one-shot CLI runs
347
+ # always read fresh, and long-lived embedders rebuild the
348
+ # spellchecker via {Kotoshu.reset_spellchecker}). An empty set when
349
+ # disabled through {Configuration#personal_dictionary}.
350
+ #
351
+ # @return [Set<String>]
352
+ def personal_words
353
+ return @personal_words if instance_variable_defined?(:@personal_words)
354
+
355
+ @personal_words =
356
+ if @config.personal_dictionary
357
+ PersonalDictionary.words.map(&:downcase).to_set
358
+ else
359
+ Set.new
360
+ end
361
+ end
362
+
363
+ # Resolve the tokenizer of the language being checked for word
364
+ # extraction (plan 91).
365
+ #
366
+ # Only Language::Tokenizer::Base tokenizers participate: their
367
+ # script subclasses (Greek, Cyrillic, Latin) declare which letters
368
+ # form words. Anything else — components tokenizers such as
369
+ # English's whitespace tokenizer, or an unknown language — returns
370
+ # nil and the ASCII fallback applies.
371
+ #
372
+ # Module-less staged languages get the script fallback (plan 107):
373
+ # Language::Script maps the code to its writing script and hands
374
+ # back the script-appropriate tokenizer, so basic-tier languages
375
+ # check with real word extraction while their module is pending.
376
+ #
377
+ # A resource bundle pins the language: spellchecker_for passes the
378
+ # shared global Configuration next to the resolved bundle, and the
379
+ # config may still carry the global default language instead of
380
+ # the one actually being checked.
381
+ #
382
+ # @return [Language::Tokenizer::Base, nil] The language tokenizer
383
+ def resolve_language_tokenizer
384
+ code = @resource_bundle&.language || @config.language
385
+ return nil if code.nil? || code.empty?
386
+
387
+ language_class = Language::Registry.get(code)
388
+ return Language::Script.tokenizer_for(code) if language_class.nil?
389
+
390
+ tokenizer = language_class.new.tokenizer
391
+ tokenizer.is_a?(Language::Tokenizer::Base) ? tokenizer : nil
392
+ end
393
+
394
+ # Check if a character is part of a word.
395
+ #
396
+ # Uses the language tokenizer's script-aware set when one is
397
+ # configured; otherwise the historical ASCII letters plus
398
+ # apostrophe.
399
+ #
400
+ # @param char [String] The character
401
+ # @return [Boolean] True if it's a word character
402
+ def word_char?(char)
403
+ @word_char_regex.match?(char)
404
+ end
405
+ end
406
+ end