kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,256 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "json"
5
+
6
+ module Kotoshu
7
+ # Exports conformance vectors for the kotoshu-rs test suite (TODO.impl plan
8
+ # 67, M3).
9
+ #
10
+ # The exporter walks the Hunspell fixture corpora that live in this
11
+ # repository, runs the REAL engine -- one real +Spellchecker+ over one real
12
+ # +Dictionary::Hunspell+ per fixture, no mocks or stubs anywhere -- and emits
13
+ # one JSON object per line (JSONL).
14
+ #
15
+ # == Row shape (v0)
16
+ #
17
+ # Matches kotoshu-rs' +tests/conformance/v0-placeholders.jsonl+, plus a
18
+ # +dictionary+ key that identifies the fixture the row was produced against
19
+ # (without it a consumer cannot reproduce the engine call):
20
+ #
21
+ # {"kind":"correct","language":"en","dictionary":"spec/integrational/fixtures/base",
22
+ # "input":"created","expected":true}
23
+ # {"kind":"suggest","language":"en","dictionary":"spec/integrational/fixtures/base",
24
+ # "input":"hlelo","limit":5,"expected":[{"word":"hello","distance":1,
25
+ # "confidence":1.0,"source":"edit_distance"}]}
26
+ #
27
+ # +expected+ entries for +suggest+ come from the framework serialization of
28
+ # +Suggestions::Suggestion+ (+to_hash+, provided by lutaml-model -- nothing
29
+ # hand-rolled here), reduced to the four keys the kotoshu-rs harness parses.
30
+ #
31
+ # == Semantics of +expected+ -- FROZEN ENGINE BEHAVIOR
32
+ #
33
+ # +expected+ records what the Ruby engine ACTUALLY returns, not what the
34
+ # Hunspell fixture says SHOULD be returned. The point of this file is
35
+ # cross-implementation equality (kotoshu-rs must return byte-identical
36
+ # results), not linguistic correctness. If the Ruby engine changes behavior,
37
+ # the vectors are re-exported and both implementations move together.
38
+ #
39
+ # == Corpora
40
+ #
41
+ # * Spylls corpus: every +spec/integrational/fixtures/*.{aff,dic}+ pair;
42
+ # words are taken from the matching +.good+ / +.wrong+ lists using the
43
+ # Spylls +read_list+ conventions (strip, drop empty lines, drop lines
44
+ # ending in "."). Dictionaries without word lists contribute no rows.
45
+ # * Spec fixture corpus: +spec/fixtures/dictionaries/hunspell/test.{aff,dic}+
46
+ # (its .dic entries plus +spec/fixtures/words.txt+).
47
+ #
48
+ # == Determinism
49
+ #
50
+ # Corpora are iterated sorted by id, words keep fixture file order (.good
51
+ # first, then .wrong, deduplicated), hash keys are built as stable literals,
52
+ # and no timestamps are written. Exporting twice yields byte-identical
53
+ # output.
54
+ #
55
+ # == Sync contract with kotoshu-rs
56
+ #
57
+ # The generated file (default +conformance/vectors.jsonl+ at the gem repo
58
+ # root) IS committed to this repository -- conformance/ is deliberately NOT
59
+ # gitignored. kotoshu-rs copies it into +tests/conformance/+ via its own
60
+ # documented sync step, so its CI consumes the vectors without the gem.
61
+ class ConformanceExporter
62
+ # Fixed suggestion limit used for every "suggest" vector.
63
+ SUGGEST_LIMIT = 5
64
+
65
+ # Language recorded in every row (the engine language_code the exporter
66
+ # passes when loading fixture dictionaries, mirroring SpyllsTestHelper).
67
+ LANGUAGE = "en"
68
+
69
+ # Corpus ids excluded from the default export.
70
+ #
71
+ # "timelimit" is Hunspell's exponential-compounding torture test
72
+ # (COMPOUNDMIN 1 over runs of zeros): +correct?+ survives it, but
73
+ # +suggest+ never terminates in the Ruby engine, so no vector can be
74
+ # frozen. The fixture stays covered by spec/integrational/lookup_spec.rb.
75
+ EXCLUDED_FIXTURES = ["spec/integrational/fixtures/timelimit"].freeze
76
+
77
+ # Keys kept from Suggestions::Suggestion#to_hash (lutaml-model
78
+ # serialization) in "suggest" expected payloads. Mirrors the
79
+ # kotoshu-rs v0 placeholder schema; "metadata" is dropped.
80
+ SUGGESTION_KEYS = %w[word distance confidence source].freeze
81
+
82
+ # One fixture dictionary plus the words to run against it.
83
+ Corpus = Struct.new(:id, :aff_path, :dic_path, :words, keyword_init: true)
84
+
85
+ # Outcome of an export run.
86
+ ExportResult = Struct.new(:path, :row_count, :corpus_count, :skipped_corpora, :skipped_words,
87
+ keyword_init: true)
88
+
89
+ class << self
90
+ # Root of the gem repository (this file lives in lib/kotoshu/).
91
+ #
92
+ # @return [String] Absolute path to the repo root
93
+ def gem_root
94
+ File.expand_path("../..", __dir__)
95
+ end
96
+
97
+ # Default corpora: the Spylls fixtures plus the spec fixture corpus.
98
+ #
99
+ # Applies EXCLUDED_FIXTURES and drops corpora with no words, so the
100
+ # export is stable regardless of which auxiliary dictionaries happen to
101
+ # sit in the fixture directory.
102
+ #
103
+ # @param gem_root [String] Repo root to resolve fixture paths against
104
+ # @return [Array<Corpus>] Corpora sorted by id
105
+ def default_corpora(gem_root: self.gem_root)
106
+ spylls_corpora(gem_root: gem_root) + [spec_fixture_corpus(gem_root: gem_root)]
107
+ end
108
+
109
+ # Scan the Spylls fixture directory for Hunspell dictionaries.
110
+ #
111
+ # @param gem_root [String] Repo root
112
+ # @return [Array<Corpus>] Non-excluded corpora with at least one word,
113
+ # sorted by id
114
+ def spylls_corpora(gem_root: self.gem_root)
115
+ dir = File.join(gem_root, "spec/integrational/fixtures")
116
+ Dir.glob(File.join(dir, "*.dic")).filter_map do |dic_path|
117
+ base = dic_path.delete_suffix(".dic")
118
+ id = "spec/integrational/fixtures/#{File.basename(base)}"
119
+ next if EXCLUDED_FIXTURES.include?(id)
120
+
121
+ words = (read_word_list("#{base}.good") + read_word_list("#{base}.wrong")).uniq
122
+ next if words.empty?
123
+
124
+ Corpus.new(id: id, aff_path: "#{base}.aff", dic_path: dic_path, words: words)
125
+ end
126
+ end
127
+
128
+ # The spec fixture corpus: the small Hunspell test dictionary shipped
129
+ # under spec/fixtures, fed its own .dic entries plus words.txt.
130
+ #
131
+ # @param gem_root [String] Repo root
132
+ # @return [Corpus]
133
+ def spec_fixture_corpus(gem_root: self.gem_root)
134
+ base = File.join(gem_root, "spec/fixtures/dictionaries/hunspell/test")
135
+ words = (dic_headwords("#{base}.dic") + read_word_list(File.join(gem_root, "spec/fixtures/words.txt"))).uniq
136
+ Corpus.new(id: "spec/fixtures/dictionaries/hunspell/test",
137
+ aff_path: "#{base}.aff", dic_path: "#{base}.dic", words: words)
138
+ end
139
+
140
+ # Read a Spylls word list (.good / .wrong / plain list), mirroring
141
+ # SpyllsTestHelper#read_list: strip lines, drop empty lines and lines
142
+ # ending in ".".
143
+ #
144
+ # @param path [String] List file path
145
+ # @return [Array<String>] Words in file order
146
+ def read_word_list(path)
147
+ return [] unless File.file?(path)
148
+
149
+ File.read(path, encoding: "UTF-8").split("\n").filter_map do |line|
150
+ word = line.strip
151
+ next if word.empty?
152
+ next if word.end_with?(".")
153
+
154
+ word
155
+ end
156
+ end
157
+
158
+ # Extract headwords from a Hunspell .dic file: skip the count line,
159
+ # comments and morphological continuation lines; strip affix flags.
160
+ #
161
+ # @param dic_path [String] .dic file path
162
+ # @return [Array<String>] Headwords in file order
163
+ def dic_headwords(dic_path)
164
+ File.read(dic_path, encoding: "UTF-8").split("\n").filter_map do |line|
165
+ entry = line.strip
166
+ next if entry.empty?
167
+ next if entry.start_with?("#")
168
+ next if entry.match?(/\A\d+\z/)
169
+
170
+ entry.split("/", 2).first
171
+ end
172
+ end
173
+
174
+ # Export the default corpora to a JSONL file.
175
+ #
176
+ # @param path [String] Destination file path (directories are created)
177
+ # @param corpora [Array<Corpus>] Corpora to export
178
+ # @return [ExportResult] Row/corpus counts and anything skipped
179
+ def export(path:, corpora: default_corpora)
180
+ new(corpora).export(path)
181
+ end
182
+ end
183
+
184
+ # @param corpora [Array<Corpus>] Corpora to export (sorted by id)
185
+ def initialize(corpora)
186
+ @corpora = corpora.sort_by(&:id)
187
+ @skipped_corpora = {}
188
+ @skipped_words = {}
189
+ end
190
+
191
+ # Build every vector row, deterministically ordered.
192
+ #
193
+ # @return [Array<Hash>] Rows in corpus-id then fixture-word order
194
+ def rows
195
+ @corpora.each_with_object([]) do |corpus, acc|
196
+ checker = spellchecker_for(corpus)
197
+ next unless checker
198
+
199
+ corpus.words.each do |word|
200
+ unless word.dup.force_encoding(Encoding::UTF_8).valid_encoding?
201
+ @skipped_words[[corpus.id, word]] = "not valid UTF-8"
202
+ next
203
+ end
204
+
205
+ acc << { kind: "correct", language: LANGUAGE, dictionary: corpus.id,
206
+ input: word, expected: checker.correct?(word) }
207
+ acc << { kind: "suggest", language: LANGUAGE, dictionary: corpus.id,
208
+ input: word, limit: SUGGEST_LIMIT,
209
+ expected: checker.suggest(word, max_suggestions: SUGGEST_LIMIT)
210
+ .map { |suggestion| suggestion_payload(suggestion) } }
211
+ end
212
+ end
213
+ end
214
+
215
+ # Build the rows and write them as JSONL to +path+.
216
+ #
217
+ # @param path [String] Destination file path (directories are created)
218
+ # @return [ExportResult] Row/corpus counts and anything skipped
219
+ def export(path)
220
+ all_rows = rows
221
+ FileUtils.mkdir_p(File.dirname(path))
222
+ File.write(path, all_rows.map { |row| JSON.generate(row) }.join("\n") << "\n")
223
+
224
+ ExportResult.new(path: path, row_count: all_rows.size,
225
+ corpus_count: @corpora.size - @skipped_corpora.size,
226
+ skipped_corpora: @skipped_corpora, skipped_words: @skipped_words)
227
+ end
228
+
229
+ private
230
+
231
+ # Build the real engine for a corpus: a real Hunspell dictionary wrapped
232
+ # in a real Spellchecker. Corpus load failures are recorded and skipped
233
+ # rather than aborting the whole export.
234
+ #
235
+ # @param corpus [Corpus]
236
+ # @return [Spellchecker, nil] nil when the fixture fails to load
237
+ def spellchecker_for(corpus)
238
+ dictionary = Dictionary::Hunspell.new(dic_path: corpus.dic_path, aff_path: corpus.aff_path,
239
+ language_code: LANGUAGE)
240
+ Spellchecker.new(dictionary: dictionary)
241
+ rescue StandardError => e
242
+ @skipped_corpora[corpus.id] = "#{e.class}: #{e.message.lines.first&.strip}"
243
+ nil
244
+ end
245
+
246
+ # Reduce a Suggestions::Suggestion to the four-key payload the conformance
247
+ # vectors carry. Serialization itself is the framework's (lutaml-model
248
+ # +to_hash+); this only selects keys from its output.
249
+ #
250
+ # @param suggestion [Suggestions::Suggestion]
251
+ # @return [Hash] "word"/"distance"/"confidence"/"source" hash
252
+ def suggestion_payload(suggestion)
253
+ suggestion.to_hash.slice(*SUGGESTION_KEYS)
254
+ end
255
+ end
256
+ end
@@ -0,0 +1,176 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Kotoshu
6
+ # Replays the committed conformance vectors through one backend (or both)
7
+ # and reports differences — the dual-backend suite of plan 66
8
+ # (TODO.impl 05 P4b).
9
+ #
10
+ # The vectors (conformance/vectors.jsonl, exported by
11
+ # {ConformanceExporter} and committed) freeze Ruby engine behavior. This
12
+ # runner replays every vector through a backend and compares the actual
13
+ # against the frozen expectation, so:
14
+ #
15
+ # * rake kotoshu:conformance:ruby — the pure-Ruby engine must reproduce
16
+ # the vectors (guards against accidental behavior drift).
17
+ # * rake kotoshu:conformance:native — the native engine
18
+ # ({NativeBackend} via a real {Spellchecker} with backend "native")
19
+ # must reproduce them too.
20
+ # * rake kotoshu:conformance:compare — both engines run side by side and
21
+ # every row must agree: expected == ruby == native. Any difference is
22
+ # a backend divergence; the task exits nonzero.
23
+ #
24
+ # Everything here runs the real engines over the real fixture
25
+ # dictionaries — no doubles anywhere.
26
+ class ConformanceRunner
27
+ # Default vectors path: the committed file at the gem repo root.
28
+ #
29
+ # @return [String]
30
+ def self.default_path
31
+ File.join(ConformanceExporter.gem_root, "conformance", "vectors.jsonl")
32
+ end
33
+
34
+ # One vector whose replay disagreed with its expectation (or, in a
35
+ # {CompareResult}, with the other backend — then +expected+ is the Ruby
36
+ # actual and +actual+ the native actual).
37
+ Failure = Struct.new(:index, :kind, :dictionary, :input, :expected, :actual, keyword_init: true)
38
+
39
+ # Outcome of replaying the vectors through one backend.
40
+ RunResult = Struct.new(:backend, :row_count, :failures, :actuals, keyword_init: true) do
41
+ def ok?
42
+ failures.empty?
43
+ end
44
+ end
45
+
46
+ # Outcome of {#compare}: both runs plus the rows where the backends
47
+ # disagree with each other.
48
+ CompareResult = Struct.new(:ruby, :native, :divergences, keyword_init: true) do
49
+ def ok?
50
+ ruby.ok? && native.ok? && divergences.empty?
51
+ end
52
+ end
53
+
54
+ # @param path [String] JSONL vectors file to replay
55
+ def initialize(path: self.class.default_path)
56
+ @path = path
57
+ @rows = File.readlines(path, chomp: true).map { |line| JSON.parse(line) }
58
+ end
59
+
60
+ # @return [Integer] vector count
61
+ def row_count
62
+ @rows.size
63
+ end
64
+
65
+ # Replay every vector through one backend, comparing against the frozen
66
+ # expectations.
67
+ #
68
+ # @param backend [String, Symbol] "ruby" or "native"
69
+ # @return [RunResult]
70
+ # @raise [Native::Unavailable] backend "native" and the extension is
71
+ # not built
72
+ def run(backend:)
73
+ backend_name = backend.to_s
74
+ if backend_name == "native" && !Kotoshu::Native.available?
75
+ raise Native::Unavailable,
76
+ "native backend requested but the extension is not built " \
77
+ "(rake compile) — cannot run the native conformance suite"
78
+ end
79
+
80
+ failures = []
81
+ engines = {}
82
+ actuals = @rows.each_with_index.map do |row, index|
83
+ actual = replay(engine_for(row, backend_name, engines), row)
84
+ if actual != row["expected"]
85
+ failures << Failure.new(index: index, kind: row["kind"], dictionary: row["dictionary"],
86
+ input: row["input"], expected: row["expected"], actual: actual)
87
+ end
88
+ actual
89
+ end
90
+ RunResult.new(backend: backend_name, row_count: @rows.size, failures: failures, actuals: actuals)
91
+ end
92
+
93
+ # Run both backends and cross-check every row.
94
+ #
95
+ # @return [CompareResult]
96
+ # @raise [Native::Unavailable] the extension is not built
97
+ def compare
98
+ ruby_result = run(backend: :ruby)
99
+ native_result = run(backend: :native)
100
+
101
+ divergences = @rows.each_index.filter_map do |index|
102
+ ruby_actual = ruby_result.actuals[index]
103
+ native_actual = native_result.actuals[index]
104
+ next if ruby_actual == native_actual
105
+
106
+ row = @rows[index]
107
+ Failure.new(index: index, kind: row["kind"], dictionary: row["dictionary"],
108
+ input: row["input"], expected: ruby_actual, actual: native_actual)
109
+ end
110
+
111
+ CompareResult.new(ruby: ruby_result, native: native_result, divergences: divergences)
112
+ end
113
+
114
+ private
115
+
116
+ # The engine for a row's dictionary, built once per dictionary.
117
+ #
118
+ # @param row [Hash] vector row
119
+ # @param backend [String] "ruby" or "native"
120
+ # @param engines [Hash] cache keyed by dictionary id
121
+ # @return [Spellchecker, String] the engine, or an error marker for
122
+ # dictionaries that fail to load
123
+ def engine_for(row, backend, engines)
124
+ dictionary_id = row["dictionary"]
125
+ engines[dictionary_id] ||= build_engine(dictionary_id, backend)
126
+ end
127
+
128
+ # Build the real engine for a dictionary id from the vectors file: a
129
+ # real {Dictionary::Hunspell} over the fixture files, wrapped in a real
130
+ # {Spellchecker} pinned to the requested backend (pinning overrides any
131
+ # KOTOSHU_BACKEND env so the task measures what it names).
132
+ #
133
+ # @param dictionary_id [String] dictionary path relative to gem root
134
+ # @param backend [String] "ruby" or "native"
135
+ # @return [Spellchecker, String] the engine or a failure description
136
+ def build_engine(dictionary_id, backend)
137
+ base = File.join(ConformanceExporter.gem_root, dictionary_id)
138
+ dictionary = Dictionary::Hunspell.new(
139
+ dic_path: "#{base}.dic", aff_path: "#{base}.aff",
140
+ language_code: ConformanceExporter::LANGUAGE
141
+ )
142
+ Spellchecker.new(dictionary: dictionary, config: Configuration.new(backend: backend))
143
+ rescue StandardError => e
144
+ "#{e.class}: #{e.message.lines.first&.strip}"
145
+ end
146
+
147
+ # Replay one vector through an engine.
148
+ #
149
+ # @param engine [Spellchecker, String] engine or load-failure marker
150
+ # @param row [Hash] vector row
151
+ # @return [Boolean, Array<Hash>] the actual result
152
+ def replay(engine, row)
153
+ return "dictionary failed to load: #{engine}" if engine.is_a?(String)
154
+
155
+ case row["kind"]
156
+ when "correct"
157
+ engine.correct?(row["input"])
158
+ when "suggest"
159
+ engine.suggest(row["input"], max_suggestions: row["limit"])
160
+ .map { |suggestion| suggestion_payload(suggestion) }
161
+ else
162
+ "unknown vector kind: #{row['kind'].inspect}"
163
+ end
164
+ end
165
+
166
+ # Reduce a Suggestion to the four-key conformance payload — the same
167
+ # selection {ConformanceExporter} applies (lutaml-model +to_hash+,
168
+ # slice to SUGGESTION_KEYS).
169
+ #
170
+ # @param suggestion [Suggestions::Suggestion]
171
+ # @return [Hash]
172
+ def suggestion_payload(suggestion)
173
+ suggestion.to_hash.slice(*ConformanceExporter::SUGGESTION_KEYS)
174
+ end
175
+ end
176
+ end
@@ -0,0 +1,186 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Base error class for all Kotoshu exceptions.
5
+ #
6
+ # @example Raising a custom Kotoshu error
7
+ # raise Kotoshu::Error, "Something went wrong"
8
+ class Error < StandardError; end
9
+
10
+ # Error raised when a dictionary file cannot be found.
11
+ #
12
+ # @example Dictionary not found
13
+ # raise DictionaryNotFoundError, "Dictionary not found: /path/to/dic.dic"
14
+ class DictionaryNotFoundError < Error
15
+ # Create a new dictionary not found error.
16
+ #
17
+ # @param path [String] The path that was not found
18
+ # @param message [String] Custom message (optional)
19
+ def initialize(path, message = nil)
20
+ @path = path
21
+ super(message || "Dictionary not found: #{path}")
22
+ end
23
+
24
+ # @return [String] The path that was not found
25
+ attr_reader :path
26
+ end
27
+
28
+ # Error raised when a dictionary file has an invalid format.
29
+ #
30
+ # @example Invalid dictionary format
31
+ # raise InvalidDictionaryFormatError, "Invalid .dic file format"
32
+ class InvalidDictionaryFormatError < Error
33
+ # Create a new invalid format error.
34
+ #
35
+ # @param path [String] The file path
36
+ # @param details [String] Details about the format issue
37
+ def initialize(path, details = nil)
38
+ @path = path
39
+ @details = details
40
+ super("Invalid dictionary format#{": #{details}" if details}: #{path}")
41
+ end
42
+
43
+ # @return [String] The file path
44
+ attr_reader :path
45
+
46
+ # @return [String, nil] Details about the format issue
47
+ attr_reader :details
48
+ end
49
+
50
+ # Error raised when there is a configuration issue.
51
+ #
52
+ # @example Invalid configuration
53
+ # raise ConfigurationError, "Invalid dictionary type: unknown_type"
54
+ class ConfigurationError < Error
55
+ # Create a new configuration error.
56
+ #
57
+ # @param message [String] The error message
58
+ # @param key [String, Symbol] The configuration key (optional)
59
+ def initialize(message, key: nil)
60
+ @key = key
61
+ super(message)
62
+ end
63
+
64
+ # @return [String, Symbol, nil] The configuration key
65
+ attr_reader :key
66
+ end
67
+
68
+ # Error raised during spell checking operations.
69
+ #
70
+ # @example Spell check failure
71
+ # raise SpellcheckError, "Failed to check word: encoding error"
72
+ class SpellcheckError < Error
73
+ # Create a new spellcheck error.
74
+ #
75
+ # @param message [String] The error message
76
+ # @param word [String] The word being checked (optional)
77
+ def initialize(message, word: nil)
78
+ @word = word
79
+ super(message)
80
+ end
81
+
82
+ # @return [String, nil] The word being checked
83
+ attr_reader :word
84
+ end
85
+
86
+ # Error raised when an affix rule cannot be parsed.
87
+ #
88
+ # @example Invalid affix rule
89
+ # raise AffixRuleError, "Invalid affix rule: PFX A Y 1 re"
90
+ class AffixRuleError < Error
91
+ # Create a new affix rule error.
92
+ #
93
+ # @param message [String] The error message
94
+ # @param rule [String] The rule that failed to parse (optional)
95
+ def initialize(message, rule: nil)
96
+ @rule = rule
97
+ super(message)
98
+ end
99
+
100
+ # @return [String, nil] The rule that failed to parse
101
+ attr_reader :rule
102
+ end
103
+
104
+ # Error raised when a required resource is not cached and cannot be
105
+ # downloaded (offline mode or network failure).
106
+ class ResourceNotCachedError < Error
107
+ def initialize(language, resource_type)
108
+ @language = language
109
+ @resource_type = resource_type
110
+ super("Resource not cached: #{language}:#{resource_type}. " \
111
+ "Pre-fetch with `kotoshu cache download language #{language}` " \
112
+ "or disable offline mode (KOTOSHU_OFFLINE=0).")
113
+ end
114
+
115
+ attr_reader :language, :resource_type
116
+ end
117
+
118
+ # Error raised by the hot path (Kotoshu.correct?, .suggest, .check,
119
+ # .check_file, .spellchecker_for) when a language hasn't been set up
120
+ # via Kotoshu.setup / kotoshu setup. The hot path is cache-only and
121
+ # never downloads — explicit setup is required.
122
+ class ResourceNotSetupError < Error
123
+ def initialize(language, resource_type = "spelling")
124
+ @language = language
125
+ @resource_type = resource_type
126
+ super("Language '#{language}' is not set up (missing #{resource_type}). " \
127
+ "Run `kotoshu setup #{language}` or " \
128
+ "`Kotoshu.setup(:#{language})` first.")
129
+ end
130
+
131
+ attr_reader :language, :resource_type
132
+ end
133
+
134
+ # Error raised when a resource cannot be resolved for a language
135
+ # (unsupported language, download failure, etc.).
136
+ class ResourceResolutionError < Error
137
+ def initialize(language, reason)
138
+ @language = language
139
+ super("Cannot resolve resources for '#{language}': #{reason}")
140
+ end
141
+
142
+ attr_reader :language
143
+ end
144
+
145
+ # Error raised when a downloaded resource fails integrity verification
146
+ # (SHA-256 mismatch against manifest, truncated content, etc.).
147
+ #
148
+ # The downloaded bytes are never trusted until verified against a known
149
+ # manifest entry. Mismatch raises this error with both hashes so the
150
+ # caller can surface them in audit logs and CI output.
151
+ #
152
+ # The optional +remediation:+ hint is appended to the message when the
153
+ # caller knows the user-facing fix (e.g. "Run `kotoshu cache download
154
+ # :en --model` to re-download"). Existing callers that omit it see no
155
+ # change in the message format.
156
+ class IntegrityError < Error
157
+ def initialize(resource_id, expected:, actual:, url: nil, remediation: nil)
158
+ @resource_id = resource_id
159
+ @expected = expected
160
+ @actual = actual
161
+ @url = url
162
+ @remediation = remediation
163
+ msg = "Integrity verification failed for #{resource_id}: "
164
+ msg << "expected sha256=#{expected}, got sha256=#{actual}"
165
+ msg << " (url: #{url})" if url
166
+ msg << ". #{remediation}" if remediation
167
+ super(msg)
168
+ end
169
+
170
+ attr_reader :resource_id, :expected, :actual, :url, :remediation
171
+ end
172
+
173
+ # Error raised when Japanese-language features are requested but the
174
+ # suika gem is unavailable. suika is a soft runtime dependency because
175
+ # its native extension (dartsclone) does not build on slim/minimal
176
+ # environments. All non-Japanese code paths are unaffected.
177
+ class SuikaUnavailable < Error
178
+ def initialize(detail = nil)
179
+ message = "suika gem not loaded"
180
+ message += " (#{detail})" if detail
181
+ message += ". Install with: gem install suika"
182
+ message += ". Or set KOTOSHU_NO_SUIKA=1 to silence this in code paths that opt out."
183
+ super(message)
184
+ end
185
+ end
186
+ end