kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,841 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Main "is this word correct?" algorithm implementation.
6
+ #
7
+ # Ported from Spylls (Python) lookup.py
8
+ #
9
+ # On a bird's-eye view level:
10
+ # * Word correctness check is an attempt to analyze word form
11
+ # (maybe it has this suffix? maybe it has this prefix? maybe it
12
+ # consists of several words?)
13
+ # * The word is considered correct if at least one form is found that
14
+ # has valid suffixes/prefixes from .aff file and valid stem from
15
+ # .dic file, and they are all compatible with each other.
16
+ #
17
+ # To follow algorithm details, start reading from Lookup.call method.
18
+ module Lookup
19
+ autoload :CompoundChecks, "kotoshu/algorithms/lookup/compound_checks"
20
+ autoload :AffixForm, "kotoshu/algorithms/lookup/affix_form"
21
+ autoload :CompoundForm, "kotoshu/algorithms/lookup/compound_form"
22
+
23
+ NUMBER_REGEXP = /^\d+(\.\d+)?$/
24
+
25
+ # Position of word part in compound word.
26
+ #
27
+ # Used when checking whether a word could be part of a compound
28
+ # (specifically its begin/middle/end).
29
+ module CompoundPos
30
+ BEGIN_POS = :begin
31
+ MIDDLE = :middle
32
+ END_POS = :end
33
+ end
34
+
35
+ # Main word correctness lookup class.
36
+ #
37
+ # Typically, you would not use this directly.
38
+ #
39
+ # Example:
40
+ # dictionary = Kotoshu::Dictionary.load('en_US')
41
+ # lookuper = dictionary.lookuper
42
+ #
43
+ # lookuper.call('spylls') # => false
44
+ # lookuper.call('spells') # => true
45
+ #
46
+ # lookuper.good_forms('spells') do |form|
47
+ # puts form
48
+ # end
49
+ # # AffixForm(spells = spells)
50
+ # # AffixForm(spells = spell + Suffix(s: S×, on [[^sxzhy]]$))
51
+ class Lookuper
52
+ include CompoundChecks
53
+
54
+ # @return [Hash] Aff data structure (from aff file)
55
+ attr_reader :aff
56
+
57
+ # @return [Hash] Dic data structure (from dic file)
58
+ attr_reader :dic
59
+
60
+ def initialize(aff, dic)
61
+ @aff = aff
62
+ @dic = dic
63
+ end
64
+
65
+ # The outermost word correctness check.
66
+ #
67
+ # Basically, prepares word for check (converting/removing chars), and
68
+ # then checks whether any good word form can be produced with good_forms.
69
+ # If there is none, also tries to break word by break-points.
70
+ #
71
+ # @param word [String] Word to check
72
+ # @param capitalization [Boolean] If false, check only exact capitalization
73
+ # @param allow_nosuggest [Boolean] If false, don't consider NOSUGGEST words as correct
74
+ # @return [Boolean] Whether word is correct
75
+ def call(word, capitalization: true, allow_nosuggest: true)
76
+ # Check if word is correct
77
+ is_correct = ->(w) do
78
+ good_forms(w, capitalization: capitalization, allow_nosuggest: allow_nosuggest).any?
79
+ end
80
+
81
+ # If all entries matching the word have FORBIDDENWORD flag, word can't be correct
82
+ if @aff[:FORBIDDENWORD] && @dic[:has_flag]&.call(word, @aff[:FORBIDDENWORD], for_all: true)
83
+ return false
84
+ end
85
+
86
+ # Convert word with ICONV table
87
+ word_to_check = @aff[:ICONV] ? @aff[:ICONV].call(word) : word
88
+
89
+ # Remove ignored characters
90
+ if @aff[:IGNORE]
91
+ ignore_chars = @aff[:IGNORE]
92
+ word_to_check = word_to_check.chars.reject { |c| ignore_chars.include?(c) }.join
93
+ end
94
+
95
+ # Numbers are always good
96
+ return true if NUMBER_REGEXP.match?(word_to_check)
97
+
98
+ # Try breaking word by break patterns
99
+ break_word(word_to_check).each do |parts|
100
+ if parts.all? { |part| part.empty? || is_correct.call(part) }
101
+ return true
102
+ end
103
+ end
104
+
105
+ false
106
+ end
107
+
108
+ # Recursively produce all possible lists of word breaking by break
109
+ # patterns (like dashes).
110
+ #
111
+ # Example: "pre-processed-meat" would produce:
112
+ # ["pre-processed-meat"]
113
+ # ["pre", "processed-meat"]
114
+ # ["pre", "processed", "meat"]
115
+ # ["pre-processed", "meat"]
116
+ #
117
+ # This is necessary because dictionary might contain "pre-processed"
118
+ # as a separate entry.
119
+ #
120
+ # @param text [String] Text to break
121
+ # @param depth [Integer] Current recursion depth
122
+ # @yield [Array<String>] Each possible breaking
123
+ # @return [Enumerator] If no block given
124
+ def break_word(text, depth = 0)
125
+ return enum_for(:break_word, text, depth) unless block_given?
126
+ return if depth > 10
127
+
128
+ # Return whole text as first option
129
+ yield [text]
130
+
131
+ break_patterns = @aff[:BREAK] || []
132
+ break_patterns.each do |pattern|
133
+ str = text.to_s
134
+ pos = 0
135
+
136
+ while (match_data = pattern[:matcher].match(str, pos))
137
+ start = str[0...match_data.begin(1)]
138
+ rest = str[match_data.end(1)..]
139
+
140
+ break_word(rest, depth + 1) do |breaking|
141
+ yield [start, *breaking]
142
+ end
143
+
144
+ pos = match_data.end(0)
145
+ break if pos >= str.length
146
+ end
147
+ end
148
+ end
149
+
150
+ # The main producer of correct word forms.
151
+ #
152
+ # Produces all ways the proposed string might correspond to dictionary/
153
+ # affixes. If there is at least one, the word is correctly spelled.
154
+ #
155
+ # Example:
156
+ # lookuper.good_forms('building') do |form|
157
+ # puts form
158
+ # end
159
+ # # AffixForm(building = building) # noun
160
+ # # AffixForm(building = build + Suffix(ing: G×, on [[^e]]$)) # verb
161
+ #
162
+ # @param word [String] Word to check
163
+ # @param capitalization [Boolean] If false, use only exact capitalization
164
+ # @param allow_nosuggest [Boolean] If false, exclude NOSUGGEST words
165
+ # @param affix_forms [Boolean] If false, only return compound forms
166
+ # @param compound_forms [Boolean] If false, only return affix forms
167
+ # @yield [AffixForm, CompoundForm] Each valid word form
168
+ def good_forms(word,
169
+ capitalization: true,
170
+ allow_nosuggest: true,
171
+ affix_forms: true,
172
+ compound_forms: true, &block)
173
+ unless block
174
+ return enum_for(:good_forms, word,
175
+ capitalization: capitalization,
176
+ allow_nosuggest: allow_nosuggest,
177
+ affix_forms: affix_forms,
178
+ compound_forms: compound_forms)
179
+ end
180
+
181
+ # Get capitalization variants
182
+ if capitalization
183
+ captype, variants = @aff[:casing].variants(word)
184
+ else
185
+ captype = @aff[:casing].guess(word)
186
+ variants = [word]
187
+ end
188
+
189
+ # Check each variant
190
+ variants.each do |variant|
191
+ if affix_forms
192
+ affix_forms_internal(variant, captype: captype, allow_nosuggest: allow_nosuggest) do |form|
193
+ # Special German ß handling
194
+ if @aff[:CHECKSHARPS] && @aff[:KEEPCASE]
195
+ stem = form.in_dictionary ? form.in_dictionary[:stem] : form.stem
196
+ if stem.include?('ß') &&
197
+ captype == Capitalization::Type::ALL &&
198
+ word.include?('ß') &&
199
+ form.flags.include?(@aff[:KEEPCASE])
200
+ next
201
+ end
202
+ end
203
+
204
+ yield form
205
+ end
206
+ end
207
+
208
+ if compound_forms
209
+ compound_forms_internal(variant, captype: captype, allow_nosuggest: allow_nosuggest, &block)
210
+ end
211
+ end
212
+ end
213
+
214
+ # Check if the word is correct without yielding forms.
215
+ #
216
+ # Convenience method for simple correctness checks.
217
+ #
218
+ # @param word [String] Word to check
219
+ # @param capitalization [Boolean] Check capitalization variants
220
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
221
+ # @param affix_forms [Boolean] Check affix forms
222
+ # @param compound_forms [Boolean] Check compound forms
223
+ # @return [Boolean] Whether word is correct
224
+ def correct?(word,
225
+ capitalization: true,
226
+ allow_nosuggest: true,
227
+ affix_forms: true,
228
+ compound_forms: true)
229
+ good_forms(word,
230
+ capitalization: capitalization,
231
+ allow_nosuggest: allow_nosuggest,
232
+ affix_forms: affix_forms,
233
+ compound_forms: compound_forms).any?
234
+ end
235
+
236
+ # Alias for better readability
237
+ alias is_correct? correct?
238
+
239
+ private
240
+
241
+ # Internal affix forms generator.
242
+ #
243
+ # @param word [String] Word to process
244
+ # @param captype [Symbol] Capitalization type
245
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
246
+ # @param with_forbidden [Boolean] When true, also yield forms whose
247
+ # homonym carries FORBIDDENWORD (used by compound_forms_internal to
248
+ # detect forbidden base words). When false (default), forbidden
249
+ # homonyms are skipped per-homonym — not by aborting the whole
250
+ # search, so a non-forbidden homonym of the same stem can still
251
+ # match.
252
+ # @param compoundpos [Symbol, nil] When called from compounds_by_flags,
253
+ # the position in compound (BEGIN_POS / MIDDLE / END_POS). Drives
254
+ # suffix/prefix allowance in produce_affix_forms and compound
255
+ # position flag checks in is_good_form.
256
+ # @param prefix_flags [Array<String>] Flags a prefix must carry to be
257
+ # valid inside a compound (COMPOUNDPERMITFLAG).
258
+ # @param suffix_flags [Array<String>] Flags a suffix must carry to be
259
+ # valid inside a compound (COMPOUNDPERMITFLAG).
260
+ # @param forbidden_flags [Array<String>] Flags that disqualify an
261
+ # affix inside a compound (COMPOUNDFORBIDFLAG).
262
+ # @yield [AffixForm] Each valid affix form
263
+ def affix_forms_internal(word, captype:, allow_nosuggest:, with_forbidden: false,
264
+ compoundpos: nil, prefix_flags: [], suffix_flags: [],
265
+ forbidden_flags: [])
266
+ unless block_given?
267
+ return enum_for(:affix_forms_internal, word,
268
+ captype: captype,
269
+ allow_nosuggest: allow_nosuggest,
270
+ with_forbidden: with_forbidden,
271
+ compoundpos: compoundpos,
272
+ prefix_flags: prefix_flags,
273
+ suffix_flags: suffix_flags,
274
+ forbidden_flags: forbidden_flags)
275
+ end
276
+
277
+ produce_affix_forms(word, compoundpos: compoundpos,
278
+ prefix_flags: prefix_flags,
279
+ suffix_flags: suffix_flags,
280
+ forbidden_flags: forbidden_flags).each do |form|
281
+ found = false
282
+ homonyms = @dic[:homonyms]&.call(form.stem) || []
283
+
284
+ # FORBIDDENWORD: in compound context (compoundpos set) OR when the
285
+ # form has affixes, if ANY homonym of the stem carries
286
+ # FORBIDDENWORD, the entire form is rejected. This mirrors
287
+ # Spylls lookup.py — a forbidden stem must not appear as a
288
+ # compound part (even without affixes) nor as an affixed form.
289
+ if !with_forbidden && @aff[:FORBIDDENWORD] &&
290
+ (compoundpos || form.has_affixes?) &&
291
+ homonyms.any? { |h| (h[:flags] || []).include?(@aff[:FORBIDDENWORD]) }
292
+ next
293
+ end
294
+
295
+ homonyms.each do |homonym|
296
+ candidate = form.replace(in_dictionary: homonym)
297
+ if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
298
+ compoundpos: compoundpos)
299
+ found = true
300
+ yield candidate
301
+ end
302
+ end
303
+
304
+ # FORCEUCASE: when checking the beginning of a compound and the
305
+ # original word is capitalized, also try lowercased stem homonyms
306
+ # so that compound parts that must be uppercased can still match.
307
+ if compoundpos == CompoundPos::BEGIN_POS && @aff[:FORCEUCASE] &&
308
+ captype == Capitalization::Type::INIT
309
+ lower_homonyms = @dic[:homonyms]&.call(form.stem.downcase) || []
310
+ lower_homonyms.each do |homonym|
311
+ candidate = form.replace(in_dictionary: homonym)
312
+ if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
313
+ compoundpos: compoundpos)
314
+ found = true
315
+ yield candidate
316
+ end
317
+ end
318
+ end
319
+
320
+ # Skip the case-insensitive fallback when any path already matched
321
+ # for this form, when the form lives in a compound slot (compound
322
+ # parts go through their own dispatch), or when the original
323
+ # word's captype is not ALL — the lowercase index is only meant
324
+ # for ALLCAPS queries whose dictionary stem differs in case.
325
+ next if found
326
+ next if compoundpos
327
+ next if captype != Capitalization::Type::ALL
328
+ next unless @aff[:casing].guess(word) == Capitalization::Type::NO
329
+
330
+ # ALLCAPS case-insensitive fallback: when the original input was
331
+ # ALL CAPS but the dictionary stem has a different case (e.g.
332
+ # user typed "UNICEF'S" / captype=ALL but the variant being
333
+ # checked is "unicef's" / captype=NO), look up homonyms in the
334
+ # lowercase index. Mirrors Spylls lookup.py:423-436.
335
+ ignorecase_homonyms = @dic[:homonyms]&.call(form.stem, ignorecase: true) || []
336
+ ignorecase_homonyms.each do |homonym|
337
+ forbidden = @aff[:FORBIDDENWORD] &&
338
+ form.has_affixes? &&
339
+ (homonym[:flags] || []).include?(@aff[:FORBIDDENWORD])
340
+ next if forbidden && !with_forbidden
341
+
342
+ candidate = form.replace(in_dictionary: homonym)
343
+ if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
344
+ compoundpos: compoundpos)
345
+ yield candidate
346
+ end
347
+ end
348
+ end
349
+ end
350
+
351
+ # Internal compound forms generator.
352
+ #
353
+ # @param word [String] Word to process
354
+ # @param captype [Symbol] Capitalization type
355
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
356
+ # @yield [CompoundForm] Each valid compound form
357
+ def compound_forms_internal(word, captype:, allow_nosuggest:)
358
+ unless block_given?
359
+ return enum_for(:compound_forms_internal, word, captype: captype,
360
+ allow_nosuggest: allow_nosuggest)
361
+ end
362
+
363
+ # Check if any affix form has FORBIDDENWORD
364
+ if @aff[:FORBIDDENWORD]
365
+ forbidden_found = false
366
+ affix_forms_internal(word, captype: captype, allow_nosuggest: allow_nosuggest,
367
+ with_forbidden: true) do |form|
368
+ if form.flags.include?(@aff[:FORBIDDENWORD])
369
+ forbidden_found = true
370
+ break
371
+ end
372
+ end
373
+ return if forbidden_found
374
+ end
375
+
376
+ # Try compounds by flags
377
+ if @aff[:COMPOUNDBEGIN] || @aff[:COMPOUNDFLAG]
378
+ compounds_by_flags(word, captype: captype, allow_nosuggest: allow_nosuggest) do |compound|
379
+ yield compound unless is_bad_compound(compound, captype)
380
+ end
381
+ end
382
+
383
+ # Try compounds by rules
384
+ if @aff[:COMPOUNDRULE]
385
+ compounds_by_rules(word, allow_nosuggest: allow_nosuggest) do |compound|
386
+ yield compound unless is_bad_compound(compound, captype)
387
+ end
388
+ end
389
+ end
390
+
391
+ # Produce all possible affix forms for a word.
392
+ #
393
+ # @param word [String] Word to process
394
+ # @param compoundpos [Symbol, nil] Position in compound
395
+ # @param prefix_flags [Array<String>] Required prefix flags
396
+ # @param suffix_flags [Array<String>] Required suffix flags
397
+ # @param forbidden_flags [Array<String>] Forbidden affix flags
398
+ # @yield [AffixForm] Each possible affix form
399
+ def produce_affix_forms(word,
400
+ compoundpos: nil,
401
+ prefix_flags: [],
402
+ suffix_flags: [],
403
+ forbidden_flags: [], &block)
404
+ unless block
405
+ return enum_for(:produce_affix_forms, word,
406
+ compoundpos: compoundpos,
407
+ prefix_flags: prefix_flags,
408
+ suffix_flags: suffix_flags,
409
+ forbidden_flags: forbidden_flags)
410
+ end
411
+
412
+ # "Whole word" is always an option
413
+ yield AffixForm.new(word, word)
414
+
415
+ # Check if suffixes/prefixes are allowed
416
+ suffix_allowed = compoundpos.nil? || compoundpos == CompoundPos::END_POS || !suffix_flags.empty?
417
+ prefix_allowed = compoundpos.nil? || compoundpos == CompoundPos::BEGIN_POS || !prefix_flags.empty?
418
+
419
+ # Generate suffix forms
420
+ if suffix_allowed
421
+ desuffix(word, required_flags: suffix_flags, forbidden_flags: forbidden_flags, &block)
422
+ end
423
+
424
+ # Generate prefix forms
425
+ if prefix_allowed
426
+ deprefix(word, required_flags: prefix_flags, forbidden_flags: forbidden_flags) do |form|
427
+ yield form
428
+
429
+ # Try prefix + suffix if allowed
430
+ if suffix_allowed && form.prefix && form.prefix[:crossproduct]
431
+ desuffix(form.stem,
432
+ required_flags: suffix_flags,
433
+ forbidden_flags: forbidden_flags,
434
+ crossproduct: true) do |form2|
435
+ yield form2.replace(text: form.text, prefix: form.prefix)
436
+ end
437
+ end
438
+ end
439
+ end
440
+ end
441
+
442
+ # Remove suffixes from word.
443
+ #
444
+ # @param word [String] Word to process
445
+ # @param required_flags [Array<String>] Required suffix flags
446
+ # @param forbidden_flags [Array<String>] Forbidden suffix flags
447
+ # @param nested [Boolean] Whether this is a nested call
448
+ # @param crossproduct [Boolean] Whether suffix must have crossproduct
449
+ # @yield [AffixForm] Each form with suffix removed
450
+ def desuffix(word, required_flags: [], forbidden_flags: [], nested: false, crossproduct: false)
451
+ unless block_given?
452
+ return enum_for(:desuffix, word,
453
+ required_flags: required_flags,
454
+ forbidden_flags: forbidden_flags,
455
+ nested: nested,
456
+ crossproduct: crossproduct)
457
+ end
458
+
459
+ suffixes_index = @aff[:suffixes_index] || {}
460
+ word_reversed = word.reverse
461
+
462
+ # Spylls's Trie.lookup yields root payloads (empty-add suffixes)
463
+ # before walking the path, so suffixes with add="" are always
464
+ # considered. The hash index drops them unless we explicitly
465
+ # include the "" bucket.
466
+ candidates = (suffixes_index[''] || []) + (suffixes_index[word_reversed[0]] || [])
467
+ candidates.each do |suffix|
468
+ # Check if suffix is valid
469
+ next if crossproduct && !suffix[:crossproduct]
470
+ next unless (required_flags - (suffix[:flags] || [])).empty?
471
+ next unless (forbidden_flags & (suffix[:flags] || [])).empty?
472
+
473
+ # Check if suffix matches
474
+ if word.end_with?(suffix[:affix])
475
+ # Remove suffix and add strip value. Note: when affix is "",
476
+ # `word[0...-0]` would be `word[0...0]` = "" — so handle the
477
+ # empty case explicitly to keep the whole word as the base.
478
+ base = suffix[:affix].empty? ? word : word[0...-suffix[:affix].length]
479
+ strip = suffix[:affix_data] ? suffix[:affix_data][:strip] : ''
480
+ stem = base + strip
481
+
482
+ # Check condition (only if condition_checker is present)
483
+ next if suffix[:condition_checker] && !suffix[:condition_checker].matches?(stem)
484
+
485
+ yield AffixForm.new(word, stem, suffix: suffix)
486
+
487
+ # Try removing another suffix (one level only)
488
+ unless nested
489
+ desuffix(stem,
490
+ required_flags: [suffix[:flag], *required_flags],
491
+ forbidden_flags: forbidden_flags,
492
+ nested: true,
493
+ crossproduct: crossproduct) do |form2|
494
+ yield form2.replace(suffix2: suffix, text: word)
495
+ end
496
+ end
497
+ end
498
+ end
499
+ end
500
+
501
+ # Remove prefixes from word.
502
+ #
503
+ # @param word [String] Word to process
504
+ # @param required_flags [Array<String>] Required prefix flags
505
+ # @param forbidden_flags [Array<String>] Forbidden prefix flags
506
+ # @param nested [Boolean] Whether this is a nested call
507
+ # @yield [AffixForm] Each form with prefix removed
508
+ def deprefix(word, required_flags: [], forbidden_flags: [], nested: false)
509
+ unless block_given?
510
+ return enum_for(:deprefix, word,
511
+ required_flags: required_flags,
512
+ forbidden_flags: forbidden_flags,
513
+ nested: nested)
514
+ end
515
+
516
+ prefixes_index = @aff[:prefixes_index] || {}
517
+
518
+ # Mirror the suffix side: prefixes with add="" live under the ""
519
+ # bucket and must always be considered.
520
+ candidates = (prefixes_index[''] || []) + (prefixes_index[word[0]] || [])
521
+ candidates.each do |prefix|
522
+ # Check if prefix is valid
523
+ next unless (required_flags - (prefix[:flags] || [])).empty?
524
+ next unless (forbidden_flags & (prefix[:flags] || [])).empty?
525
+
526
+ # Check if prefix matches
527
+ if word.start_with?(prefix[:affix])
528
+ # Remove prefix and re-add the strip value at the START.
529
+ # (For suffixes the strip is appended; for prefixes it's
530
+ # prepended — the strip/affix are mirrors of each other
531
+ # and the strip lives on the same edge of the stem as the
532
+ # affix does on the candidate.)
533
+ strip = prefix[:affix_data] ? prefix[:affix_data][:strip] : ''
534
+ stem = strip + word[prefix[:affix].length..]
535
+
536
+ # Check condition (only if condition_checker is present)
537
+ next if prefix[:condition_checker] && !prefix[:condition_checker].matches?(stem)
538
+
539
+ yield AffixForm.new(word, stem, prefix: prefix)
540
+
541
+ # Try removing another prefix if COMPLEXPREFIXES is set
542
+ unless nested || !@aff[:COMPLEXPREFIXES]
543
+ deprefix(stem,
544
+ required_flags: [prefix[:flag], *required_flags],
545
+ forbidden_flags: forbidden_flags,
546
+ nested: true) do |form2|
547
+ yield form2.replace(prefix2: prefix, text: word)
548
+ end
549
+ end
550
+ end
551
+ end
552
+ end
553
+
554
+ # Check if an affix form is valid.
555
+ #
556
+ # When compoundpos is nil (non-compound check), the form must not
557
+ # carry ONLYINCOMPOUND. When compoundpos is set (compound part
558
+ # check), ONLYINCOMPOUND is allowed and instead the form must carry
559
+ # COMPOUNDFLAG or the position-specific flag (COMPOUNDBEGIN /
560
+ # COMPOUNDMIDDLE / COMPOUNDEND). This mirrors Spylls lookup.py
561
+ # is_good_form.
562
+ #
563
+ # @param form [AffixForm] Form to check
564
+ # @param captype [Symbol] Original word's capitalization type
565
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
566
+ # @param compoundpos [Symbol, nil] Position in compound, or nil
567
+ # @return [Boolean] Whether form is valid
568
+ def is_good_form(form, captype:, allow_nosuggest:, compoundpos: nil)
569
+ return false unless form.in_dictionary
570
+
571
+ root_flags = form.in_dictionary[:flags] || []
572
+ all_flags = form.flags
573
+
574
+ # Check NOSUGGEST
575
+ if !allow_nosuggest && @aff[:NOSUGGEST] && root_flags.include?(@aff[:NOSUGGEST])
576
+ return false
577
+ end
578
+
579
+ # Check KEEPCASE
580
+ if @aff[:KEEPCASE] && root_flags.include?(@aff[:KEEPCASE])
581
+ stem_captype = @aff[:casing].guess(form.in_dictionary[:stem])
582
+ return false if captype != stem_captype && !(@aff[:CHECKSHARPS] && form.in_dictionary[:stem].include?('ß'))
583
+ end
584
+
585
+ # Check NEEDAFFIX
586
+ if @aff[:NEEDAFFIX]
587
+ if root_flags.include?(@aff[:NEEDAFFIX]) && !form.has_affixes?
588
+ return false
589
+ end
590
+ if form.has_affixes? && form.all_affixes.all? { |a| (a[:flags] || []).include?(@aff[:NEEDAFFIX]) }
591
+ return false
592
+ end
593
+ end
594
+
595
+ # Check prefix flag compatibility
596
+ if form.prefix && !all_flags.include?(form.prefix[:flag])
597
+ return false
598
+ end
599
+
600
+ # Check suffix flag compatibility
601
+ if form.suffix && !all_flags.include?(form.suffix[:flag])
602
+ return false
603
+ end
604
+
605
+ # Check CIRCUMFIX
606
+ if @aff[:CIRCUMFIX]
607
+ suffix_has = form.suffix ? (form.suffix[:flags] || []).include?(@aff[:CIRCUMFIX]) : false
608
+ prefix_has = form.prefix ? (form.prefix[:flags] || []).include?(@aff[:CIRCUMFIX]) : false
609
+ return false if suffix_has != prefix_has
610
+ end
611
+
612
+ # Compound position checks
613
+ if compoundpos.nil?
614
+ # Non-compound: reject ONLYINCOMPOUND words
615
+ return !all_flags.include?(@aff[:ONLYINCOMPOUND])
616
+ end
617
+
618
+ return false if compoundpos == CompoundPos::END_POS && barred_from_compound_end?(form)
619
+
620
+ # Compound: must carry COMPOUNDFLAG or the position-specific flag.
621
+ # ONLYINCOMPOUND is allowed here (it just means "not valid outside
622
+ # compounds") — the compound position flag is what authorizes the
623
+ # part to appear at this slot.
624
+ return true if @aff[:COMPOUNDFLAG] && all_flags.include?(@aff[:COMPOUNDFLAG])
625
+ return true if compoundpos == CompoundPos::BEGIN_POS && @aff[:COMPOUNDBEGIN] &&
626
+ all_flags.include?(@aff[:COMPOUNDBEGIN])
627
+ return true if compoundpos == CompoundPos::MIDDLE && @aff[:COMPOUNDMIDDLE] &&
628
+ all_flags.include?(@aff[:COMPOUNDMIDDLE])
629
+ return true if compoundpos == CompoundPos::END_POS && @aff[:COMPOUNDEND] &&
630
+ all_flags.include?(@aff[:COMPOUNDEND])
631
+
632
+ false
633
+ end
634
+
635
+ # Whether a suffix stops this form from closing a compound.
636
+ #
637
+ # Ported from Hunspell's suffix_check (affixmgr.cxx:2832): a suffix
638
+ # is barred at the compound end only when it carries ONLYINCOMPOUND
639
+ # and nothing else rescues it. Two things do. A zero-width suffix is
640
+ # handled by a separate branch upstream that has no such guard, and
641
+ # a prefix on the form is an explicit exemption (`ppfx` upstream).
642
+ # That is how
643
+ # German reaches "Arbeitscomputern", through its decapitalising
644
+ # prefix rather than through any compound-position flag.
645
+ #
646
+ # @param form [AffixForm] Form being placed at the compound end
647
+ # @return [Boolean] Whether a suffix bars it from ending the compound
648
+ def barred_from_compound_end?(form)
649
+ only_in_compound = @aff[:ONLYINCOMPOUND]
650
+ return false unless only_in_compound
651
+ return false if form.prefix
652
+
653
+ form.suffixes.any? do |suffix|
654
+ !suffix[:affix].to_s.empty? && (suffix[:flags] || []).include?(only_in_compound)
655
+ end
656
+ end
657
+
658
+ # Generate compound forms by flags.
659
+ #
660
+ # @param word_rest [String] Remaining word to process
661
+ # @param captype [Symbol] Capitalization type
662
+ # @param depth [Integer] Current recursion depth
663
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
664
+ # @yield [CompoundForm] Each valid compound form
665
+ def compounds_by_flags(word_rest, captype:, depth: 0, allow_nosuggest: true)
666
+ unless block_given?
667
+ return enum_for(:compounds_by_flags, word_rest,
668
+ captype: captype,
669
+ depth: depth,
670
+ allow_nosuggest: allow_nosuggest)
671
+ end
672
+
673
+ aff = @aff
674
+ compound_min = aff[:COMPOUNDMIN] || 3
675
+ compound_word_max = aff[:COMPOUNDWORDMAX]
676
+ compound_permit_flag = aff[:COMPOUNDPERMITFLAG]
677
+ compound_forbid_flag = aff[:COMPOUNDFORBIDFLAG]
678
+
679
+ forbidden_flags = compound_forbid_flag ? [compound_forbid_flag] : []
680
+ permit_flags = compound_permit_flag ? [compound_permit_flag] : []
681
+
682
+ # Check if rest can be compound end. At END position, suffixes are
683
+ # always allowed (compoundpos=END_POS in produce_affix_forms), but
684
+ # prefixes still need COMPOUNDPERMITFLAG.
685
+ if depth.positive?
686
+ affix_forms_internal(word_rest, captype: captype, allow_nosuggest: allow_nosuggest,
687
+ compoundpos: CompoundPos::END_POS,
688
+ prefix_flags: permit_flags,
689
+ forbidden_flags: forbidden_flags) do |form|
690
+ yield CompoundForm.new([form])
691
+ end
692
+ end
693
+
694
+ # Check compounding limits
695
+ return if word_rest.length < compound_min * 2
696
+ return if compound_word_max && depth >= compound_word_max
697
+
698
+ compoundpos = depth.zero? ? CompoundPos::BEGIN_POS : CompoundPos::MIDDLE
699
+ # At BEGIN_POS, prefixes are allowed by default (no permit flag
700
+ # needed); at MIDDLE, prefixes need the permit flag. Suffixes at
701
+ # both BEGIN and MIDDLE need the permit flag — when there is no
702
+ # COMPOUNDPERMITFLAG in the .aff, suffix_flags is empty and
703
+ # produce_affix_forms blocks all suffixes inside compounds.
704
+ prefix_flags = compoundpos == CompoundPos::BEGIN_POS ? [] : permit_flags
705
+
706
+ each_compound_junction(word_rest, compound_min) do |left, right, left_text, pattern|
707
+ affix_forms_internal(left, captype: captype, allow_nosuggest: allow_nosuggest,
708
+ compoundpos: compoundpos,
709
+ prefix_flags: prefix_flags,
710
+ suffix_flags: permit_flags,
711
+ forbidden_flags: forbidden_flags) do |form|
712
+ compounds_by_flags(right, captype: captype, depth: depth + 1,
713
+ allow_nosuggest: allow_nosuggest) do |partial|
714
+ next if pattern && !pattern.match?(form, partial.parts.first)
715
+
716
+ part = left_text ? form.replace(text: left_text) : form
717
+ yield CompoundForm.new([part, *partial.parts],
718
+ [pattern, *partial.junction_patterns])
719
+ end
720
+ end
721
+ end
722
+ end
723
+
724
+ # CHECKCOMPOUNDPATTERN entries that spell a simplified compound form.
725
+ #
726
+ # @return [Array<Readers::CompoundPattern>] Patterns with a replacement
727
+ def replacement_patterns
728
+ @replacement_patterns ||= (@aff[:CHECKCOMPOUNDPATTERN] || []).select(&:replacement?)
729
+ end
730
+
731
+ # Every cut of the word worth trying as a compound boundary.
732
+ #
733
+ # Yields `(left, right, left_text, pattern)` rather than an object:
734
+ # this is the innermost loop of the whole checker, and one allocation
735
+ # per split position is measurable on deeply compounding languages.
736
+ #
737
+ # `left_text` is the text to record on the left member when it
738
+ # differs from the text looked up, and `pattern` is set only for a
739
+ # cut a CHECKCOMPOUNDPATTERN replacement rebuilt. Both are nil for
740
+ # the ordinary case, which is nearly all of them.
741
+ #
742
+ # The window is COMPOUNDMIN either side of the cut, measured on the
743
+ # word as written. Hunspell fixes it the same way, before it tries
744
+ # any replacement (setcminmax at affixmgr.cxx:1644, the replacement
745
+ # loop nested inside at :1660). A replacement can therefore stand for
746
+ # members longer than the text it replaced, which is why hunspell.5
747
+ # warns that COMPOUNDMIN "doesn't work correctly with the compound
748
+ # word alternation, so it may need to set COMPOUNDMIN to lower
749
+ # value". Matching that quirk beats being quietly more permissive
750
+ # than the dictionaries were written against.
751
+ #
752
+ # @param word_rest [String] Remaining word being split
753
+ # @param compound_min [Integer] COMPOUNDMIN
754
+ # @yield [String, String, String, Readers::CompoundPattern] Left
755
+ # member, remaining text, surface text or nil, pattern or nil
756
+ def each_compound_junction(word_rest, compound_min)
757
+ simplified_triple = @aff[:SIMPLIFIEDTRIPLE]
758
+ patterns = replacement_patterns
759
+
760
+ (compound_min...(word_rest.length - compound_min + 1)).each do |pos|
761
+ beg = word_rest[0...pos]
762
+ rest = word_rest[pos..]
763
+ yield beg, rest, nil, nil
764
+
765
+ if simplified_triple && !beg.empty? && !rest.empty? && beg[-1] == rest[0]
766
+ yield beg + beg[-1], rest, beg, nil
767
+ end
768
+
769
+ patterns.each do |pattern|
770
+ replacement = pattern.replacement
771
+ next unless word_rest[pos, replacement.length] == replacement
772
+
773
+ yield beg + pattern.left_stem,
774
+ pattern.right_stem + word_rest[(pos + replacement.length)..],
775
+ nil, pattern
776
+ end
777
+ end
778
+ end
779
+
780
+ # Generate compound forms by rules.
781
+ #
782
+ # @param word_rest [String] Remaining word to process
783
+ # @param prev_parts [Array<Hash>] Previously processed parts
784
+ # @param rules [Array<Hash>] Valid compound rules
785
+ # @param allow_nosuggest [Boolean] Include NOSUGGEST words
786
+ # @yield [CompoundForm] Each valid compound form
787
+ def compounds_by_rules(word_rest, prev_parts: [], rules: nil, allow_nosuggest: true)
788
+ unless block_given?
789
+ return enum_for(:compounds_by_rules, word_rest,
790
+ prev_parts: prev_parts,
791
+ rules: rules,
792
+ allow_nosuggest: allow_nosuggest)
793
+ end
794
+
795
+ aff = @aff
796
+ compound_min = aff[:COMPOUNDMIN] || 3
797
+ compound_word_max = aff[:COMPOUNDWORDMAX]
798
+ compound_rules = aff[:COMPOUNDRULE] || []
799
+
800
+ rules ||= compound_rules
801
+
802
+ # Check if rest can be a complete compound
803
+ if prev_parts.any?
804
+ homonyms = @dic[:homonyms]&.call(word_rest) || []
805
+ homonyms.each do |homonym|
806
+ parts = [*prev_parts, homonym]
807
+ flag_sets = parts.map { |p| p[:flags] || [] }
808
+
809
+ if compound_rules.any? { |rule| rule[:full_match]&.call(flag_sets) }
810
+ yield CompoundForm.new([AffixForm.new(word_rest, word_rest)])
811
+ end
812
+ end
813
+ end
814
+
815
+ # Check limits
816
+ return if word_rest.length < compound_min * 2
817
+ return if compound_word_max && prev_parts.length >= compound_word_max
818
+
819
+ # Try all possible split positions
820
+ (compound_min...(word_rest.length - compound_min + 1)).each do |pos|
821
+ beg = word_rest[0...pos]
822
+ homonyms = @dic[:homonyms]&.call(beg) || []
823
+
824
+ homonyms.each do |homonym|
825
+ parts = [*prev_parts, homonym]
826
+ flag_sets = parts.map { |p| p[:flags] || [] }
827
+
828
+ matching_rules = compound_rules.select { |rule| rule[:partial_match]&.call(flag_sets) }
829
+ next if matching_rules.empty?
830
+
831
+ compounds_by_rules(word_rest[pos..], prev_parts: parts, rules: matching_rules,
832
+ allow_nosuggest: allow_nosuggest) do |partial|
833
+ yield CompoundForm.new([AffixForm.new(beg, beg), *partial.parts])
834
+ end
835
+ end
836
+ end
837
+ end
838
+ end
839
+ end
840
+ end
841
+ end