kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,393 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Readers
5
+ # AFF file reader for Hunspell affix files.
6
+ #
7
+ # This class reads .aff files and creates an Aff data structure.
8
+ #
9
+ # @example Reading an aff file
10
+ # reader = AffReader.new('en_US.aff')
11
+ # aff = reader.read
12
+ class AffReader
13
+ # Directives that are single boolean flags
14
+ BOOLEAN_DIRECTIVES = %w[
15
+ COMPLEXPREFIXES FULLSTRIP NOSPLITSUGS CHECKSHARPS
16
+ CHECKCOMPOUNDCASE CHECKCOMPOUNDDUP CHECKCOMPOUNDREP CHECKCOMPOUNDTRIPLE
17
+ SIMPLIFIEDTRIPLE ONLYMAXDIFF COMPOUNDMORESUFFIXES
18
+ ].freeze
19
+
20
+ # Directives that are single string values
21
+ STRING_DIRECTIVES = %w[SET FLAG KEY TRY WORDCHARS LANG].freeze
22
+
23
+ # Directives that are single integer values
24
+ INTEGER_DIRECTIVES = %w[MAXDIFF MAXNGRAMSUGS MAXCPDSUGS COMPOUNDMIN COMPOUNDWORDMAX].freeze
25
+
26
+ # Directives that are single flag values
27
+ FLAG_DIRECTIVES = %w[
28
+ NOSUGGEST KEEPCASE CIRCUMFIX NEEDAFFIX FORBIDDENWORD WARN
29
+ COMPOUNDFLAG COMPOUNDBEGIN COMPOUNDMIDDLE COMPOUNDEND
30
+ ONLYINCOMPOUND COMPOUNDPERMITFLAG COMPOUNDFORBIDFLAG FORCEUCASE
31
+ SUBSTANDARD SYLLABLENUM COMPOUNDROOT
32
+ ].freeze
33
+
34
+ # Outdated directive names and their synonyms
35
+ SYNONYMS = {
36
+ 'PSEUDOROOT' => 'NEEDAFFIX',
37
+ 'COMPOUNDLAST' => 'COMPOUNDEND'
38
+ }.freeze
39
+
40
+ attr_reader :path, :encoding, :flag_format
41
+
42
+ # Create a new AFF reader.
43
+ #
44
+ # @param path [String] Path to the .aff file
45
+ # @param encoding [String] File encoding (default: 'UTF-8');
46
+ # overridden by the file's SET directive when present
47
+ def initialize(path, encoding: 'UTF-8')
48
+ @path = path
49
+ @encoding = detect_encoding(path) || encoding
50
+ @flag_format = 'short'
51
+ @flag_synonyms = {}
52
+ end
53
+
54
+ # Read the aff file and return the aff data structure.
55
+ #
56
+ # @param source [FileReader, nil] Optional file reader to use instead of creating a new one
57
+ # @return [Hash] The aff data structure
58
+ def read(source = nil)
59
+ owned = source.nil?
60
+ reader = source || FileReader.new(@path, @encoding)
61
+
62
+ begin
63
+ data = {
64
+ 'SFX' => {},
65
+ 'PFX' => {},
66
+ 'FLAG' => 'short'
67
+ }
68
+
69
+ reader.each do |_line_no, line|
70
+ dir_value = read_directive(reader, line)
71
+ next unless dir_value
72
+
73
+ directive, value = dir_value
74
+
75
+ # Update flag format when FLAG directive is encountered (BEFORE using it)
76
+ if directive == 'FLAG'
77
+ @flag_format = value
78
+ end
79
+
80
+ # Re-parse FLAG directive value now that @flag_format is updated
81
+ if directive == 'FLAG' && value.is_a?(String)
82
+ # No re-parsing needed for FLAG, just update the format
83
+ end
84
+
85
+ # SFX/PFX have multiple entries
86
+ if %w[SFX PFX].include?(directive)
87
+ data[directive][value.first.flag] = value
88
+ else
89
+ data[directive] = value
90
+ end
91
+
92
+ # Update flag synonyms when AF directive is encountered (AFTER storing it)
93
+ if directive == 'AF'
94
+ @flag_synonyms = value
95
+ end
96
+
97
+ # Note: We don't reset_encoding during iteration because it closes
98
+ # the file and breaks the iteration. The FileReader is initialized
99
+ # with UTF-8 encoding which handles most cases.
100
+ end
101
+
102
+ data
103
+ ensure
104
+ # Close the reader only when we created it. If the caller passed
105
+ # one in, they own its lifecycle. Without this, the underlying
106
+ # File handle leaks (on Windows it also blocks tempfile cleanup).
107
+ reader.close if owned
108
+ end
109
+ end
110
+
111
+ private
112
+
113
+ # Read a directive from a line.
114
+ #
115
+ # @param reader [FileReader] The file reader
116
+ # @param line [String] The line to parse
117
+ # @return [Array, nil] [directive, value] or nil
118
+ def read_directive(reader, line)
119
+ parts = line.split(/\s+/)
120
+ return nil if parts.empty?
121
+
122
+ name = parts[0]
123
+
124
+ # Check if it looks like a directive (all caps)
125
+ return nil unless /^[A-Z]+$/.match?(name)
126
+
127
+ # Handle synonyms
128
+ name = SYNONYMS[name] || name
129
+
130
+ value = read_value(reader, name, parts[1..])
131
+
132
+ return nil if value.nil?
133
+
134
+ [name, value]
135
+ end
136
+
137
+ # Read the value for a directive.
138
+ #
139
+ # @param reader [FileReader] The file reader
140
+ # @param directive [String] The directive name
141
+ # @param values [Array<String>] Values from the line
142
+ # @return [Object] The parsed value
143
+ def read_value(reader, directive, values)
144
+ value = values.first
145
+
146
+ # String directives
147
+ if STRING_DIRECTIVES.include?(directive)
148
+ return value
149
+ end
150
+
151
+ # Integer directives
152
+ if INTEGER_DIRECTIVES.include?(directive)
153
+ return value&.to_i
154
+ end
155
+
156
+ # Flag directives
157
+ if FLAG_DIRECTIVES.include?(directive)
158
+ return parse_flag(value)
159
+ end
160
+
161
+ # Boolean directives
162
+ if BOOLEAN_DIRECTIVES.include?(directive)
163
+ return true
164
+ end
165
+
166
+ # IGNORE directive
167
+ if directive == 'IGNORE'
168
+ return Ignore.new(value || '')
169
+ end
170
+
171
+ # BREAK directive
172
+ if directive == 'BREAK'
173
+ count = value&.to_i || 0
174
+ return read_array(reader, count).map { |parts| BreakPattern.new(parts.first) }
175
+ end
176
+
177
+ # COMPOUNDRULE directive
178
+ if directive == 'COMPOUNDRULE'
179
+ count = value&.to_i || 0
180
+ return read_array(reader, count).map { |parts| CompoundRule.new(parts.first) }
181
+ end
182
+
183
+ # ICONV/OCONV directives
184
+ if %w[ICONV OCONV].include?(directive)
185
+ count = value&.to_i || 0
186
+ pairs = read_array(reader, count).map { |parts| [parts[0], parts[1] || ''] }
187
+ return ConvTable.new(pairs)
188
+ end
189
+
190
+ # REP directive
191
+ if directive == 'REP'
192
+ count = value&.to_i || 0
193
+ return read_array(reader, count).map { |parts| RepPattern.new(parts[0], parts[1] || '') }
194
+ end
195
+
196
+ # MAP directive
197
+ if directive == 'MAP'
198
+ count = value&.to_i || 0
199
+ return read_array(reader, count).map do |parts|
200
+ chars = parts.first || ''
201
+ # Parse MAP format: "aàâä" or "ß(ss)" - split by parentheses or individual chars
202
+ # Parenthesized groups like "(ss)" should be kept as a single string "ss"
203
+ chars.scan(/(\([^()]+\)|[^()])/).flatten.map do |group|
204
+ # Remove parentheses from parenthesized groups, keep as single string
205
+ # For single characters, keep as is
206
+ if group.start_with?('(') && group.end_with?(')')
207
+ group[1..-2] # Remove parentheses, keep content as single string
208
+ else
209
+ group # Keep single character as is
210
+ end
211
+ end
212
+ end
213
+ end
214
+
215
+ # SFX/PFX directives
216
+ if %w[SFX PFX].include?(directive)
217
+ flag = values[0]
218
+ crossproduct = values[1]
219
+ count = values[2]&.to_i || 0
220
+ type = directive == 'PFX' ? :prefix : :suffix
221
+
222
+ affixes = read_array(reader, count).map do |parts|
223
+ # Format: FLAG strip add condition [morph_data]
224
+ # After read_array (which skips directive), parts[0] is FLAG again
225
+ # So we skip parts[0] and use: parts[1]=strip, parts[2]=add, parts[3]=condition
226
+ strip = parts[1] == '0' ? '' : (parts[1] || '')
227
+ add = parts[2] || ''
228
+ condition = parts[3] || '.'
229
+
230
+ # Handle flags in add field: "able/CD" -> add="able", flags=["C", "D"]
231
+ if add.include?('/')
232
+ add_str, _, flags_str = add.rpartition('/')
233
+ else
234
+ add_str = add
235
+ flags_str = ''
236
+ end
237
+ flags = flags_str.empty? ? Set.new : parse_flags(flags_str).to_set
238
+
239
+ Affix.new(
240
+ type:,
241
+ flag:,
242
+ crossproduct: crossproduct == 'Y',
243
+ strip:,
244
+ add: add_str == '0' ? '' : add_str,
245
+ condition:,
246
+ flags:
247
+ )
248
+ end
249
+
250
+ return affixes
251
+ end
252
+
253
+ # CHECKCOMPOUNDPATTERN directive
254
+ if directive == 'CHECKCOMPOUNDPATTERN'
255
+ count = value&.to_i || 0
256
+ return read_array(reader, count).map do |parts|
257
+ CompoundPattern.new(parts[0], parts[1] || '', parts[2])
258
+ end
259
+ end
260
+
261
+ # AF directive (flag synonyms)
262
+ if directive == 'AF'
263
+ count = value&.to_i || 0
264
+ result = {}
265
+ read_array(reader, count).each_with_index do |parts, i|
266
+ # AF entries are decoded with the dictionary's FLAG format
267
+ # (spylls: Context.parse_flags on the entry value). With
268
+ # `FLAG num`, "AF 214,216" defines the two flags 214 and 216;
269
+ # with `FLAG long`, "AF g?1G" defines the two flags g? and 1G.
270
+ result[(i + 1).to_s] = parse_flags(parts.first).to_set
271
+ end
272
+ return result
273
+ end
274
+
275
+ # AM directive
276
+ if directive == 'AM'
277
+ count = value&.to_i || 0
278
+ result = {}
279
+ read_array(reader, count).each_with_index do |parts, i|
280
+ result[(i + 1).to_s] = parts.to_set
281
+ end
282
+ return result
283
+ end
284
+
285
+ # COMPOUNDSYLLABLE directive
286
+ if directive == 'COMPOUNDSYLLABLE'
287
+ return [value&.to_i, values[1]]
288
+ end
289
+
290
+ # PHONE directive
291
+ if directive == 'PHONE'
292
+ count = value&.to_i || 0
293
+ table = read_array(reader, count).map { |parts| [parts[0], parts[1] || '_'] }
294
+ return PhonetTable.new(table)
295
+ end
296
+
297
+ # Unknown directive - return nil
298
+ nil
299
+ end
300
+
301
+ # Read an array of values from the reader.
302
+ #
303
+ # @param reader [FileReader] The file reader
304
+ # @param count [Integer] Number of lines to read
305
+ # @return [Array<Array<String>>] Array of parsed lines
306
+ def read_array(reader, count)
307
+ result = []
308
+ count.times do
309
+ _, line = reader.next
310
+ parts = line.split(/\s+/)
311
+ # Skip the directive name at the beginning
312
+ result << parts[1..] if parts.length > 1
313
+ end
314
+ result
315
+ end
316
+
317
+ # Parse a single flag.
318
+ #
319
+ # @param string [String] Flag string
320
+ # @return [String] Parsed flag
321
+ def parse_flag(string)
322
+ parse_flags(string).first
323
+ end
324
+
325
+ # Parse multiple flags.
326
+ #
327
+ # @param string [String] Flag string
328
+ # @return [Array<String>] Parsed flags
329
+ def parse_flags(string)
330
+ return [] if string.nil? || string.empty?
331
+
332
+ # Check flag synonyms (only if the key exists in @flag_synonyms)
333
+ if @flag_synonyms&.key?(string)
334
+ return @flag_synonyms[string].to_a
335
+ end
336
+
337
+ case @flag_format
338
+ when 'short'
339
+ string.chars
340
+ when 'long'
341
+ string.scan(/../)
342
+ when 'num'
343
+ string.scan(/\d+/)
344
+ when 'UTF-8'
345
+ string.chars
346
+ else
347
+ raise ArgumentError, "Unknown flag format: #{@flag_format}"
348
+ end
349
+ end
350
+
351
+ # Detect the file's encoding.
352
+ #
353
+ # Strategy (matches Hunspell's actual behavior):
354
+ # 1. If the file has a `SET` directive, use that (normalized to Ruby form).
355
+ # 2. Else if the file's bytes are valid UTF-8, treat it as UTF-8.
356
+ # 3. Otherwise default to ISO-8859-1 (Latin-1), which is what Hunspell
357
+ # itself does for non-UTF-8 locales when `SET` is missing.
358
+ #
359
+ # Pre-scans the file in binary mode so we can reopen with the correct
360
+ # encoding before the FileReader consumes it.
361
+ #
362
+ # @param path [String] Path to the .aff file
363
+ # @return [String, nil] Encoding name (e.g., "ISO-8859-1", "UTF-8") or nil
364
+ def detect_encoding(path)
365
+ return nil if path.nil? || path.empty?
366
+ return nil unless File.file?(path)
367
+
368
+ snippet = File.binread(path)
369
+ set_match = snippet.match(/^SET\s+(\S+)/)
370
+ return normalize_encoding_name(set_match[1]) if set_match
371
+
372
+ return "UTF-8" if snippet.force_encoding("UTF-8").valid_encoding?
373
+
374
+ "ISO-8859-1"
375
+ end
376
+
377
+ # Normalize Hunspell encoding names to Ruby encoding names.
378
+ #
379
+ # @param name [String] Hunspell encoding identifier
380
+ # @return [String] Ruby encoding name
381
+ def normalize_encoding_name(name)
382
+ return name if name.upcase == "UTF-8"
383
+
384
+ normalized = name.upcase.delete("-")
385
+ if normalized.start_with?("ISO8859")
386
+ "ISO-8859-#{normalized.sub('ISO8859', '')}"
387
+ else
388
+ name
389
+ end
390
+ end
391
+ end
392
+ end
393
+ end
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Readers
5
+ # Base class for checking affix conditions.
6
+ #
7
+ # Hunspell affix rules specify conditions that the stem must match
8
+ # before an affix can be applied. Different scripts may have different
9
+ # interpretations of these conditions.
10
+ #
11
+ # Conditions are anchored differently for prefixes vs suffixes:
12
+ # - Suffix condition `y` matches stems that END with `y` (e.g. `[^y]$`)
13
+ # - Prefix condition `ij` matches stems that START with `ij`
14
+ # (e.g. `^ij`)
15
+ #
16
+ # @example Latin script condition checking
17
+ # checker = LatinScriptConditionChecker.compile('[^y]', type: :suffix)
18
+ # checker.matches?('try') # => true (doesn't end with 'y')
19
+ # checker.matches?('fly') # => false (ends with 'y')
20
+ #
21
+ # @abstract Subclasses must implement the matches? method
22
+ class ConditionChecker
23
+ # Compile a condition string into a checker.
24
+ #
25
+ # @param condition [String] The condition string from .aff file
26
+ # @param script [Symbol] The script type (:latin, :arabic, :hebrew, etc.)
27
+ # @param type [Symbol] :suffix (anchor at end) or :prefix (anchor at start)
28
+ # @return [ConditionChecker] A checker instance
29
+ def self.compile(condition, script: :latin, type: :suffix)
30
+ case script
31
+ when :latin
32
+ LatinScriptConditionChecker.compile(condition, type: type)
33
+ else
34
+ # For other scripts, create a passthrough checker
35
+ # (condition is not applied)
36
+ PassthroughConditionChecker.new
37
+ end
38
+ end
39
+
40
+ # Check if the given stem matches this condition.
41
+ #
42
+ # @param stem [String] The stem to check
43
+ # @return [Boolean] True if the stem matches
44
+ def matches?(stem)
45
+ raise NotImplementedError, "#{self.class} must implement #matches?"
46
+ end
47
+ end
48
+
49
+ # Passthrough condition checker that always returns true.
50
+ #
51
+ # Used for scripts where Hunspell conditions don't apply or aren't supported.
52
+ class PassthroughConditionChecker < ConditionChecker
53
+ def matches?(_stem)
54
+ true
55
+ end
56
+ end
57
+
58
+ # Condition checker for Latin-script dictionaries.
59
+ #
60
+ # Hunspell conditions are regex-like patterns. Spylls compiles them
61
+ # verbatim (with `-` escaped so it doesn't become a range inside an
62
+ # unintended context) into a regex anchored at the start (for prefixes)
63
+ # or end (for suffixes), and uses `re.search` to test.
64
+ #
65
+ # Examples (suffix unless noted):
66
+ # - '.' matches any stem (regex: /.$/ finds end-of-string)
67
+ # - 'y' matches stems ending with 'y' (regex: /y$/)
68
+ # - '[^y]' matches stems NOT ending with 'y' (regex: /[^y]$/)
69
+ # - '[aeiou]y' matches stems ending with vowel + 'y' (regex: /[aeiou]y$/)
70
+ # - '.[^aeiou]y' matches stems whose last 3 chars are any-non-vowel-y
71
+ # - 'wr.' (prefix) matches stems starting with 'wr' + any char
72
+ #
73
+ # This is NOT suitable for RTL scripts or CJK languages.
74
+ class LatinScriptConditionChecker < ConditionChecker
75
+ attr_reader :condition, :type, :anchor, :pattern
76
+
77
+ # Compile a condition string.
78
+ #
79
+ # @param condition [String] The condition string (e.g., '[^y]', '[abc]', 'y', '.')
80
+ # @param type [Symbol] :suffix (end-anchor) or :prefix (start-anchor)
81
+ # @return [LatinScriptConditionChecker] A checker instance
82
+ def self.compile(condition, type: :suffix)
83
+ anchor = type == :prefix ? :start : :ending
84
+ new(condition: condition, type: type, anchor: anchor)
85
+ end
86
+
87
+ def initialize(condition:, type:, anchor:)
88
+ @condition = condition
89
+ @type = type
90
+ @anchor = anchor
91
+ @pattern = compile_pattern
92
+ end
93
+
94
+ # Check if the stem matches the condition.
95
+ #
96
+ # @param stem [String] The stem to check
97
+ # @return [Boolean] True if the stem matches
98
+ def matches?(stem)
99
+ return true if @condition.nil? || @condition.empty? || @condition == '.'
100
+
101
+ @pattern.match?(stem)
102
+ end
103
+
104
+ private
105
+
106
+ # Compile the condition into a Ruby Regexp.
107
+ #
108
+ # Mirrors Spylls: prefix → `^<condition>`, suffix → `<condition>$`,
109
+ # with `-` escaped so it can appear literally inside the pattern.
110
+ # The check uses `match?` (which searches), so the anchors do the
111
+ # positional work.
112
+ #
113
+ # Returns a match-all regex for nil/empty conditions so the
114
+ # constructor doesn't crash on those inputs (the #matches? method
115
+ # already short-circuits on nil/empty/`.`).
116
+ def compile_pattern
117
+ return // if @condition.nil? || @condition.empty?
118
+
119
+ escaped = @condition.gsub('-', '\\-')
120
+ anchored = @anchor == :start ? "^#{escaped}" : "#{escaped}$"
121
+ Regexp.new(anchored)
122
+ end
123
+ end
124
+ end
125
+ end