kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,530 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Readers
5
+ # Affix data class for Hunspell affix rules.
6
+ #
7
+ # This class represents a prefix or suffix affix rule.
8
+ #
9
+ # @attr flag [String] The flag character identifying this rule
10
+ # @attr crossproduct [Boolean] Whether this is a cross-product rule
11
+ # @attr strip [String] Characters to strip from the word
12
+ # @attr add [String] Characters to add to the word
13
+ # @attr condition [String] Condition for applying this rule
14
+ # @attr flags [Set<String>] Additional flags
15
+ #
16
+ # @example Creating a suffix affix
17
+ # Affix.new(
18
+ # type: :suffix,
19
+ # flag: 'H',
20
+ # crossproduct: false,
21
+ # strip: 'y',
22
+ # add: 'ieth',
23
+ # condition: 'y',
24
+ # flags: Set.new
25
+ # )
26
+ class Affix
27
+ attr_reader :type, :flag, :crossproduct, :strip, :add, :condition, :flags
28
+
29
+ # Create a new affix.
30
+ #
31
+ # @param type [Symbol] :prefix or :suffix
32
+ # @param flag [String] Flag character
33
+ # @param crossproduct [Boolean] Whether cross-product
34
+ # @param strip [String] Characters to strip
35
+ # @param add [String] Characters to add
36
+ # @param condition [String] Condition regex
37
+ # @param flags [Set<String>] Additional flags
38
+ def initialize(type:, flag:, crossproduct:, strip:, add:, condition:, flags: Set.new)
39
+ @type = type
40
+ @flag = flag
41
+ @crossproduct = crossproduct
42
+ @strip = strip
43
+ @add = add
44
+ @condition = condition
45
+ @flags = flags
46
+ end
47
+
48
+ # Check if this is a prefix.
49
+ #
50
+ # @return [Boolean] True if prefix
51
+ def prefix?
52
+ @type == :prefix
53
+ end
54
+
55
+ # Check if this is a suffix.
56
+ #
57
+ # @return [Boolean] True if suffix
58
+ def suffix?
59
+ @type == :suffix
60
+ end
61
+
62
+ # String representation.
63
+ #
64
+ # @return [String] String representation
65
+ def to_s
66
+ type_str = prefix? ? 'Prefix' : 'Suffix'
67
+ "#{type_str}(#{@add}: #{@flag}#{'×' if @crossproduct}/#{@flags.to_a.join(',')}, on #{condition})"
68
+ end
69
+
70
+ # Inspect string.
71
+ #
72
+ # @return [String] Inspect string
73
+ def inspect
74
+ to_s
75
+ end
76
+ end
77
+
78
+ # Break pattern for word splitting.
79
+ #
80
+ # @attr pattern [String] The break pattern
81
+ # @attr matcher [Regexp] Compiled matcher for the pattern
82
+ class BreakPattern
83
+ attr_reader :pattern, :matcher
84
+
85
+ # Create a new break pattern.
86
+ #
87
+ # @param pattern [String] The pattern string
88
+ def initialize(pattern)
89
+ @pattern = pattern
90
+ # Special chars like #, -, * should be escaped, but ^ and $ should be treated as pattern anchors
91
+ regex_pattern = Regexp.escape(pattern).gsub('\\^', '^').gsub('\\$', '$')
92
+ @matcher = if regex_pattern.start_with?('^') || regex_pattern.end_with?('$')
93
+ Regexp.new("(#{regex_pattern})")
94
+ else
95
+ Regexp.new(".(#{regex_pattern}).")
96
+ end
97
+ end
98
+ end
99
+
100
+ # Ignore characters for lookup/suggest.
101
+ #
102
+ # @attr chars [String] Characters to ignore
103
+ # @attr translation_table [Hash] Translation table for removal
104
+ class Ignore
105
+ attr_reader :chars, :translation_table
106
+
107
+ # Create a new ignore set.
108
+ #
109
+ # @param chars [String] Characters to ignore
110
+ def initialize(chars)
111
+ @chars = chars
112
+ # Create translation table that removes these characters
113
+ @translation_table = chars.each_char.each_with_index.to_h
114
+ end
115
+
116
+ # Remove ignored characters from string.
117
+ #
118
+ # @param str [String] Input string
119
+ # @return [String] String with ignored chars removed
120
+ def remove(str)
121
+ str.chars.reject { |c| @translation_table.key?(c) }.join
122
+ end
123
+ end
124
+
125
+ # Replacement pattern for suggestions.
126
+ #
127
+ # @attr pattern [String] The pattern to match
128
+ # @attr replacement [String] The replacement string
129
+ # @attr matcher [Regexp] Compiled matcher for the pattern
130
+ class RepPattern
131
+ attr_reader :pattern, :replacement, :matcher
132
+
133
+ # Create a new replacement pattern.
134
+ #
135
+ # @param pattern [String] The pattern string
136
+ # @param replacement [String] The replacement string
137
+ def initialize(pattern, replacement)
138
+ @pattern = pattern
139
+ @replacement = replacement
140
+ @matcher = Regexp.new(pattern)
141
+ end
142
+ end
143
+
144
+ # Conversion table for ICONV/OCONV.
145
+ #
146
+ # @attr pairs [Array<Array<String>>] Array of [pattern, replacement] pairs
147
+ class ConvTable
148
+ attr_reader :pairs
149
+
150
+ # Create a new conversion table.
151
+ #
152
+ # @param pairs [Array<Array<String>>] Array of [pattern, replacement] pairs
153
+ def initialize(pairs)
154
+ @pairs = pairs
155
+ @table = pairs.map { |pat1, pat2| compile_row(pat1, pat2) }.sort_by { |search, _| search.length }
156
+ end
157
+
158
+ # Apply conversions to word.
159
+ #
160
+ # Note: Python's `re.match(string, pos)` anchors at pos, but Ruby's
161
+ # `Regexp#match?` searches from pos onward. We must check that the
162
+ # match actually begins at pos, otherwise short conversions fire on
163
+ # later positions in the word and produce nonsense like "ÉÉÉÉ" for
164
+ # "bébé".
165
+ #
166
+ # Spylls uses Python's stable `sorted(..., key=lambda r: len(r[0]))`
167
+ # which preserves declaration order for ties. Ruby's `sort_by` is
168
+ # unstable, so we add the table index as a secondary key to mirror
169
+ # Spylls. Without this, Nepali's ICONV reorders the `ZWNJ$ → ZWNJ`
170
+ # no-op rule behind `ZWNJ → U+FFF0`, causing the trailing-ZWNJ word
171
+ # to be normalized to U+FFF0 (and then dropped by IGNORE) so it
172
+ # matches the dictionary.
173
+ #
174
+ # @param word [String] Input word
175
+ # @return [String] Converted word
176
+ def call(word)
177
+ pos = 0
178
+ result = ''
179
+
180
+ while pos < word.length
181
+ matches = @table.each_with_index.filter_map do |(search, pattern, _), idx|
182
+ m = pattern.match(word, pos)
183
+ next unless m && m.begin(0) == pos
184
+
185
+ [search, idx]
186
+ end.sort_by { |s, idx| [-s.length, idx] }
187
+
188
+ if matches.any?
189
+ search, idx = matches.first
190
+ _, _, replacement = @table[idx]
191
+ result += replacement
192
+ pos += search.length
193
+ else
194
+ result += word[pos]
195
+ pos += 1
196
+ end
197
+ end
198
+
199
+ result
200
+ end
201
+
202
+ private
203
+
204
+ def compile_row(pat1, pat2)
205
+ pat1_clean = pat1.gsub('_', '')
206
+ pat1_re = pat1_clean.dup
207
+ pat1_re = "^#{pat1_re}" if pat1.start_with?('_')
208
+ pat1_re = "#{pat1_re}$" if pat1.end_with?('_')
209
+
210
+ [pat1_clean, Regexp.new(pat1_re), pat2.gsub('_', ' ')]
211
+ end
212
+ end
213
+
214
+ # Compound rule pattern.
215
+ #
216
+ # @attr text [String] The rule text
217
+ # @attr flags [Set<String>] Flags in this rule
218
+ # @attr re [Regexp] Compiled regex for full matching
219
+ # @attr partial_re [Regexp] Compiled regex for prefix matching
220
+ class CompoundRule
221
+ attr_reader :text, :flags, :re, :partial_re
222
+
223
+ # Create a new compound rule.
224
+ #
225
+ # @param text [String] The rule text (e.g., "A*B?CD")
226
+ def initialize(text)
227
+ @text = text
228
+ # Parse flags from rule text
229
+ if text.include?('(')
230
+ @flags = text.scan(/\((.+?)\)/).flatten.to_set
231
+ parts = text.scan(/\([^*?]+?\)[*?]?/)
232
+ else
233
+ @flags = text.gsub(/[*?]/, '').chars.to_set
234
+ # A flag character may itself be a regex metacharacter. Escape each
235
+ # flag so it compiles as a literal, keeping a trailing * or ?
236
+ # operator intact. The Swedish dictionary uses ) as a flag
237
+ # (COMPOUNDRULE )k, dictionaries pin 1829a3e).
238
+ parts = text.scan(/[^*?][*?]?/).map do |part|
239
+ "#{Regexp.escape(part[0])}#{part[1..]}"
240
+ end
241
+ end
242
+
243
+ # Full-match regex: the entire flag-combination string must match.
244
+ @re = Regexp.new("\\A#{parts.join}\\z")
245
+
246
+ # Partial-match regex: any prefix of the rule is accepted. Built
247
+ # by making each trailing part optional, from the end backwards:
248
+ # parts ["A","B","C"] → "A(B(C?)?)?" which matches "A", "AB", "ABC".
249
+ @partial_re = if parts.empty?
250
+ Regexp.new('\\A\\z')
251
+ else
252
+ Regexp.new("\\A#{build_partial(parts)}\\z")
253
+ end
254
+ end
255
+
256
+ # Check if flag sets fully match this rule.
257
+ #
258
+ # @param flag_sets [Array<Set<String>>] Array of flag sets
259
+ # @return [Boolean] True if the entire flag-combination matches
260
+ def fullmatch(flag_sets)
261
+ relevant_flags = flag_sets.map { |f| @flags.intersection(f).to_a }
262
+ return false if relevant_flags.empty? || relevant_flags.any?(&:empty?)
263
+
264
+ relevant_flags[0].product(*relevant_flags[1..]).any? do |fc|
265
+ @re.match?(fc.join)
266
+ end
267
+ end
268
+
269
+ # Check if flag sets form a valid prefix of this rule.
270
+ #
271
+ # Used during compounds_by_rules recursion to prune branches that
272
+ # can never lead to a full match.
273
+ #
274
+ # @param flag_sets [Array<Set<String>>] Array of flag sets
275
+ # @return [Boolean] True if a prefix of the rule matches
276
+ def partial_match(flag_sets)
277
+ relevant_flags = flag_sets.map { |f| @flags.intersection(f).to_a }
278
+ return false if relevant_flags.empty? || relevant_flags.any?(&:empty?)
279
+
280
+ relevant_flags[0].product(*relevant_flags[1..]).any? do |fc|
281
+ @partial_re.match?(fc.join)
282
+ end
283
+ end
284
+
285
+ private
286
+
287
+ # Build a regex where each trailing part becomes optional.
288
+ #
289
+ # ["A", "B", "C"] → "A(B(C?)?)?"
290
+ def build_partial(parts)
291
+ parts.reverse.reduce(nil) do |inner, part|
292
+ inner ? "#{part}(#{inner})?" : "#{part}?"
293
+ end.gsub('??', '?')
294
+ end
295
+ end
296
+
297
+ # Compound pattern for checking compound words.
298
+ #
299
+ # @attr left [String] Left side pattern
300
+ # @attr right [String] Right side pattern
301
+ # @attr replacement [String, nil] Optional replacement
302
+ class CompoundPattern
303
+ attr_reader :left, :right, :replacement, :left_stem, :left_flag, :right_stem, :right_flag,
304
+ :left_no_affix
305
+
306
+ # Create a new compound pattern.
307
+ #
308
+ # Mirrors Hunspell's parse_checkcpdtable: each side is the literal text
309
+ # up to an optional "/flag", and neither side is rewritten at parse
310
+ # time. In particular a "0" stays a "0" — cpdpat_check is what gives
311
+ # the left side's leading zero its meaning, and the right side never
312
+ # gets one.
313
+ #
314
+ # @param left [String] Left side pattern
315
+ # @param right [String] Right side pattern
316
+ # @param replacement [String, nil] Optional replacement
317
+ def initialize(left, right, replacement = nil)
318
+ @left = left
319
+ @right = right
320
+ @replacement = replacement
321
+
322
+ # The separator from partition('/') distinguishes "no slash" (no flag
323
+ # specified → nil, so the matcher skips the flag check) from a slash
324
+ # with an empty flag.
325
+ @left_stem, sep, @left_flag = left.partition('/')
326
+ @left_flag = nil if sep.empty?
327
+ # Hunspell tests only the first character (`pattern[0] == '0'`).
328
+ @left_no_affix = @left_stem.start_with?('0')
329
+
330
+ @right_stem, sep, @right_flag = right.partition('/')
331
+ @right_flag = nil if sep.empty?
332
+ # isSubset treats "." as a single-character wildcard. Precomputed so
333
+ # the overwhelmingly common literal case stays a plain start_with?.
334
+ @right_wildcard = @right_stem.include?('.')
335
+ end
336
+
337
+ # Whether this pattern also spells out a simplified compound form.
338
+ #
339
+ # @return [Boolean] True if a non-empty replacement was given
340
+ def replacement?
341
+ !@replacement.nil? && !@replacement.empty?
342
+ end
343
+
344
+ # How the two members are actually written when this pattern's
345
+ # replacement stands in for the junction between them.
346
+ #
347
+ # `foo` and `bar` under `CHECKCOMPOUNDPATTERN o b z` are written
348
+ # `fozar`, not `foobar`. Checks that read the word as the user typed it
349
+ # need this rather than the concatenation of the members.
350
+ #
351
+ # Without a replacement the members are simply written one after the
352
+ # other, so that is what this returns.
353
+ #
354
+ # @param left_text [String] Surface text of the left member
355
+ # @param right_text [String] Surface text of the right member
356
+ # @return [String] The simplified spelling
357
+ def surface(left_text, right_text)
358
+ return left_text + right_text unless replacement?
359
+
360
+ # Trim by length, not by content. Hunspell builds the members by
361
+ # splicing the two stems in at fixed offsets (affixmgr.cxx:1684), so
362
+ # undoing it is a matter of position. Matching on the text instead
363
+ # fails whenever a stem is not literal — a "." wildcard on the right,
364
+ # or a leading "0" on the left, would trim nothing or trim a real
365
+ # character that happened to look like the stem.
366
+ left_text[0...(left_text.length - @left_stem.length)].to_s +
367
+ @replacement + right_text[@right_stem.length..].to_s
368
+ end
369
+
370
+ # Check if this pattern matches the given left and right parts.
371
+ #
372
+ # Ported from Hunspell's cpdpat_check. Both sides are matched against
373
+ # the members as written, not against their stems: the pattern
374
+ # describes the seam the reader sees.
375
+ #
376
+ # The left side has three readings, in Hunspell's order:
377
+ # empty means "only the flags matter"; a leading "0" means "the text
378
+ # running into the seam is the bare dictionary root", which a
379
+ # zero-width affix still satisfies; anything else is literal text the
380
+ # left member must end with.
381
+ #
382
+ # @param left_part [AffixForm] Left part with text, stem, flags
383
+ # @param right_part [AffixForm] Right part with text, flags
384
+ # @return [Boolean] True if matches
385
+ def match?(left_part, right_part)
386
+ return false unless right_matches?(right_part.text)
387
+ return false if @left_flag && !left_part.flags.include?(@left_flag)
388
+ return false if @right_flag && !right_part.flags.include?(@right_flag)
389
+
390
+ left_matches?(left_part)
391
+ end
392
+
393
+ private
394
+
395
+ # Hunspell matches the right side with isSubset, a shared affix-key
396
+ # helper in which "." stands for any one character. The behaviour is
397
+ # undocumented for this directive and absent from Spylls, but it is
398
+ # what the reference implementation does.
399
+ #
400
+ # @param text [String] Surface text of the right member
401
+ # @return [Boolean] Whether the right side's text condition holds
402
+ def right_matches?(text)
403
+ return text.start_with?(@right_stem) unless @right_wildcard
404
+ return false if text.length < @right_stem.length
405
+
406
+ @right_stem.each_char.with_index.all? { |char, i| char == '.' || char == text[i] }
407
+ end
408
+
409
+ # @param left_part [AffixForm] Left part of the junction
410
+ # @return [Boolean] Whether the left side's text condition holds
411
+ def left_matches?(left_part)
412
+ return true if @left_stem.empty?
413
+ return left_part.text.end_with?(left_part.stem) if @left_no_affix
414
+
415
+ left_part.text.end_with?(@left_stem)
416
+ end
417
+ end
418
+
419
+ # Phonetic table for PHONE directive.
420
+ #
421
+ # Domain object wrapping the parsed PHONE rules. Rules are indexed by
422
+ # their first letter so the metaphone algorithm can look up candidates
423
+ # in O(1) per character position.
424
+ #
425
+ # @attr rules [Hash<String, Array<Rule>>] First-char → rule list
426
+ class PhonetTable
427
+ attr_reader :table, :rules
428
+
429
+ # Pattern for matching phonetic rules.
430
+ # Updated to support extended ASCII (Latin-1) characters like É, À, etc.
431
+ RULE_PATTERN = /^(?<letters>[[:alpha:]]+)(\((?<optional>[[:alpha:]]+)\))?(?<lookahead>-+)?(?<flags>[\^$<]*)(?<priority>\d)?$/
432
+
433
+ # Rule class for phonetic transformations.
434
+ #
435
+ # @attr search [Regexp] Search pattern
436
+ # @attr replacement [String] Replacement string
437
+ # @attr start [Boolean] Match at start
438
+ # @attr end [Boolean] Match at end
439
+ # @attr priority [Integer] Rule priority
440
+ # @attr followup [Boolean] Follow-up rule
441
+ Rule = Struct.new(:search, :replacement, :start, :end, :priority, :followup, keyword_init: true) do
442
+ # Length of the match if this rule fires at +pos+ in +word+,
443
+ # or nil if the rule doesn't match. The metaphone walker uses
444
+ # the length to advance its position past the consumed input.
445
+ #
446
+ # @param word [String] Word to match against
447
+ # @param pos [Integer] Position in word
448
+ # @return [Integer, nil] Length of matched substring, or nil
449
+ def match_length(word, pos)
450
+ return nil if start && pos.positive?
451
+
452
+ match_data = if self[:end]
453
+ # Anchor at end of word: strip the prefix so
454
+ # the regex can match without considering the
455
+ # preceding characters.
456
+ search.match(word[pos..])
457
+ else
458
+ search.match(word, pos)
459
+ end
460
+ return nil unless match_data
461
+
462
+ match_data.to_s.length
463
+ end
464
+
465
+ # Check if rule matches at position.
466
+ #
467
+ # @param word [String] Word to check
468
+ # @param pos [Integer] Position in word
469
+ # @return [Boolean] True if matches
470
+ def match?(word, pos)
471
+ !match_length(word, pos).nil?
472
+ end
473
+ end
474
+
475
+ # Create a new phonetic table.
476
+ #
477
+ # @param table [Array<Array<String>>] Array of [pattern, replacement] pairs
478
+ def initialize(table)
479
+ @table = table
480
+ @rules = Hash.new { |h, k| h[k] = [] }
481
+
482
+ table.each do |search, replacement|
483
+ @rules[search[0]] << parse_rule(search, replacement)
484
+ end
485
+ end
486
+
487
+ # Whether this table contains any rules.
488
+ #
489
+ # @return [Boolean]
490
+ def empty?
491
+ @table.empty?
492
+ end
493
+
494
+ # Parse a phonetic rule.
495
+ #
496
+ # @param search [String] Search pattern
497
+ # @param replacement [String] Replacement string ("_" means silent/empty,
498
+ # per aspell's phonetic convention which Hunspell inherited)
499
+ # @return [Rule] Parsed rule
500
+ def parse_rule(search, replacement)
501
+ match = RULE_PATTERN.match(search)
502
+ raise ArgumentError, "Not a proper rule: #{search.inspect}" unless match
503
+
504
+ text = match['letters'].chars
505
+ text << "[#{match['optional']}]" if match['optional']
506
+
507
+ if match['lookahead']
508
+ lookahead_len = match['lookahead'].length
509
+ regex = text[0...-lookahead_len].join + "(?=#{text[-lookahead_len..].join})"
510
+ else
511
+ regex = text.join
512
+ end
513
+
514
+ # Aspell/Hunspell phonetic convention: "_" in the replacement column
515
+ # means "produce nothing" (silent). Normalizing here means downstream
516
+ # code can append the replacement verbatim.
517
+ normalized_replacement = replacement == '_' ? '' : replacement
518
+
519
+ Rule.new(
520
+ search: Regexp.new(regex),
521
+ replacement: normalized_replacement,
522
+ start: match['flags']&.include?('^'),
523
+ end: match['flags']&.include?('$'),
524
+ priority: match['priority']&.to_i || 5,
525
+ followup: !match['lookahead'].nil?
526
+ )
527
+ end
528
+ end
529
+ end
530
+ end