kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,520 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open-uri"
4
+
5
+ module Kotoshu
6
+ module Dictionary
7
+ # Hunspell dictionary backend.
8
+ #
9
+ # This dictionary reads Hunspell-formatted dictionary files (.dic and .aff).
10
+ # Hunspell is the spell checker used by LibreOffice, Firefox, Chrome, and many
11
+ # other applications.
12
+ #
13
+ # File format:
14
+ # - .dic: Dictionary file with word count on first line, words with optional flags
15
+ # - .aff: Affix file with prefix/suffix rules and configuration
16
+ #
17
+ # @example Creating a Hunspell dictionary
18
+ # dict = Hunspell.new(
19
+ # dic_path: "en_US.dic",
20
+ # aff_path: "en_US.aff",
21
+ # language_code: "en-US"
22
+ # )
23
+ # dict.lookup?("hello") # => true
24
+ #
25
+ # @example Creating from GitHub cache
26
+ # dict = Hunspell.from_github("de")
27
+ # dict.lookup?("über") # => true
28
+ #
29
+ # @see https://hunspell.github.io/ Hunspell documentation
30
+ class Hunspell < Base
31
+ # @return [String] Path to the .dic file
32
+ attr_reader :dic_path
33
+
34
+ # @return [String] Path to the .aff file
35
+ attr_reader :aff_path
36
+
37
+ # @return [Hash] Affix rules (flag => array of rules)
38
+ attr_reader :affix_rules
39
+
40
+ # @return [Hash] Configuration options from affix file
41
+ attr_reader :aff_config
42
+
43
+ # @return [Hash] Raw aff data from AffReader (cached for Lookuper)
44
+ attr_reader :aff_data
45
+
46
+ # @return [Array] Raw words from DicReader (cached for Lookuper)
47
+ attr_reader :dic_words
48
+
49
+ # @return [Algorithms::Lookup::Lookuper] The lookup algorithm instance
50
+ def lookuper
51
+ @lookuper ||= Readers::LookupBuilder.from_data(@aff_data, @dic_words).build
52
+ end
53
+
54
+ # The TRY string from the .aff file, driving substitution and
55
+ # insertion candidate generation (see {Base#try_string}).
56
+ #
57
+ # @return [String, nil] TRY characters, or nil when the aff has no TRY
58
+ def try_string
59
+ aff_data['TRY']
60
+ end
61
+
62
+ # @return [Algorithms::Suggest::Suggester] The suggestion algorithm instance
63
+ def suggester
64
+ @suggester ||= Algorithms::Suggest::Suggester.new(
65
+ lookuper.aff, lookuper.dic, lookuper
66
+ )
67
+ end
68
+
69
+ class << self
70
+ # Load Hunspell dictionary from GitHub cache, downloading if necessary.
71
+ #
72
+ # This class method provides automatic dictionary management by:
73
+ # 1. Checking the local cache for existing dictionaries
74
+ # 2. Downloading from GitHub if not cached or expired
75
+ # 3. Managing cache metadata and TTL
76
+ #
77
+ # @example Load English dictionary
78
+ # dict = Hunspell.from_github("en")
79
+ # dict.lookup?("hello") # => true
80
+ #
81
+ # @example Load German dictionary
82
+ # dict = Hunspell.from_github("de")
83
+ # dict.lookup?("über") # => true
84
+ #
85
+ # @example Force re-download
86
+ # dict = Hunspell.from_github("fr", force_download: true)
87
+ #
88
+ # @param language_code [String] ISO 639-1 language code (e.g., 'en', 'de', 'fr')
89
+ # @param cache [Cache::LanguageCache, nil] Custom cache instance (optional)
90
+ # @param force_download [Boolean] Force re-download even if cached
91
+ # @return [Hunspell] Configured Hunspell dictionary instance
92
+ # @raise [ArgumentError] If language_code is not supported
93
+ def from_github(language_code, cache: nil, force_download: false)
94
+ cache ||= Cache::LanguageCache.new
95
+ cached = cache.get_dictionary(language_code, force_download: force_download)
96
+
97
+ new(
98
+ dic_path: cached[:dic_path],
99
+ aff_path: cached[:aff_path],
100
+ language_code: language_code,
101
+ metadata: {
102
+ source: 'github',
103
+ github_url: cached[:metadata]['url'],
104
+ checksum: cached[:metadata]['checksum'],
105
+ downloaded_at: cached[:metadata]['downloaded_at']
106
+ }
107
+ )
108
+ end
109
+
110
+ # Check if a language is available on GitHub.
111
+ #
112
+ # @param language_code [String] ISO 639-1 language code
113
+ # @param cache [Cache::LanguageCache, nil] Custom cache instance (optional)
114
+ # @return [Boolean] True if language is supported
115
+ def available_on_github?(language_code, cache: nil)
116
+ cache ||= Cache::LanguageCache.new
117
+ cache.available_languages.include?(language_code)
118
+ end
119
+
120
+ # Get list of available languages on GitHub.
121
+ #
122
+ # @param cache [Cache::LanguageCache, nil] Custom cache instance (optional)
123
+ # @return [Array<String>] List of supported language codes
124
+ def available_github_languages(cache: nil)
125
+ cache ||= Cache::LanguageCache.new
126
+ cache.available_languages
127
+ end
128
+
129
+ # Get information about a language from GitHub.
130
+ #
131
+ # @param language_code [String] ISO 639-1 language code
132
+ # @param cache [Cache::LanguageCache, nil] Custom cache instance (optional)
133
+ # @return [Hash] Language information
134
+ def language_info(language_code, cache: nil)
135
+ cache ||= Cache::LanguageCache.new
136
+ cache.get_language_info(language_code)
137
+ end
138
+ end
139
+
140
+ # Create a new Hunspell dictionary.
141
+ #
142
+ # @param dic_path [String] Path or URL to the .dic file
143
+ # @param aff_path [String] Path or URL to the .aff file
144
+ # @param language_code [String] The language code
145
+ # @param locale [String, nil] The locale (optional)
146
+ # @param metadata [Hash] Additional metadata (optional)
147
+ def initialize(dic_path:, aff_path:, language_code:, locale: nil, metadata: {})
148
+ super(language_code, locale: locale, metadata: metadata)
149
+
150
+ @dic_path = resolve_path(dic_path)
151
+ @aff_path = resolve_path(aff_path)
152
+
153
+ raise DictionaryNotFoundError, @aff_path unless File.exist?(@aff_path)
154
+ raise DictionaryNotFoundError, @dic_path unless File.exist?(@dic_path)
155
+
156
+ # Read aff file using AffReader and cache the data
157
+ aff_reader = Readers::AffReader.new(@aff_path)
158
+ @aff_data = aff_reader.read
159
+ @aff_config = @aff_data # For backward compatibility
160
+
161
+ # Read dic file using DicReader with the same encoding as the aff file
162
+ dic_reader = Readers::DicReader.new(@dic_path,
163
+ encoding: aff_reader.encoding,
164
+ flag_format: @aff_data['FLAG'] || 'short',
165
+ flag_synonyms: @aff_data['AF'] || {})
166
+ @dic_words = dic_reader.read
167
+
168
+ # Build legacy structures for backward compatibility
169
+ @word_index = build_word_index(@dic_words)
170
+ @affix_rules = parse_affix_rules(@aff_config)
171
+
172
+ # Lazy initialization of Lookuper (only created when needed)
173
+ @lookuper = nil
174
+
175
+ # Register this dictionary type
176
+ self.class.register_type(:hunspell) unless Dictionary.registry.key?(:hunspell)
177
+ end
178
+
179
+ private
180
+
181
+ # Build word index from DicReader words.
182
+ #
183
+ # @param words [Array<Readers::Word>] Words from DicReader
184
+ # @return [Hash] Word index (word => flags)
185
+ def build_word_index(words)
186
+ index = {}
187
+ words.each do |word|
188
+ index[word.stem.downcase] = word.flags.to_a
189
+ end
190
+ index
191
+ end
192
+
193
+ # Parse affix rules from AffReader data.
194
+ #
195
+ # @param aff_data [Hash] Aff data from AffReader
196
+ # @return [Hash] Affix rules by type
197
+ def parse_affix_rules(aff_data)
198
+ rules = {
199
+ prefix: Hash.new { |h, k| h[k] = [] },
200
+ suffix: Hash.new { |h, k| h[k] = [] }
201
+ }
202
+
203
+ # Convert AffReader's SFX/PFX data to legacy format
204
+ # AffReader returns: 'SFX' => { flag => [Affix, ...] }
205
+ # We need to convert each Affix to Models::AffixRule
206
+
207
+ aff_data['SFX']&.each do |flag, affix_list|
208
+ rules[:suffix][flag] = affix_list.map do |affix|
209
+ convert_to_affix_rule(affix, :suffix)
210
+ end
211
+ end
212
+
213
+ aff_data['PFX']&.each do |flag, affix_list|
214
+ rules[:prefix][flag] = affix_list.map do |affix|
215
+ convert_to_affix_rule(affix, :prefix)
216
+ end
217
+ end
218
+
219
+ rules
220
+ end
221
+
222
+ # Convert AffReader Affix to Models::AffixRule.
223
+ #
224
+ # @param affix [Readers::Affix] The affix to convert
225
+ # @param type [Symbol] :prefix or :suffix
226
+ # @return [Models::AffixRule] The converted rule
227
+ def convert_to_affix_rule(affix, type)
228
+ # Create a simple string representation for from_hunspell
229
+ # Format: PFX/SFX FLAG crossproduct strip add condition
230
+ cross_str = affix.crossproduct ? 'Y' : 'N'
231
+ strip_str = affix.strip.empty? ? '0' : affix.strip
232
+ add_str = affix.add.empty? ? '0' : affix.add
233
+ condition_str = affix.condition || '.'
234
+
235
+ type_str = type == :prefix ? 'PFX' : 'SFX'
236
+ rule_line = "#{type_str} #{affix.flag} #{cross_str} #{strip_str} #{add_str} #{condition_str}"
237
+
238
+ Models::AffixRule.from_hunspell(rule_line, type)
239
+ end
240
+
241
+ # Check if path is a URL
242
+ # @param path [String] Path to check
243
+ # @return [Boolean] True if path is a URL
244
+ def url?(path)
245
+ path.start_with?("http://", "https://")
246
+ end
247
+
248
+ # Resolve path to local file path (downloading if URL)
249
+ # @param path [String] Path or URL
250
+ # @return [String] Local file path
251
+ def resolve_path(path)
252
+ return File.expand_path(path) unless url?(path)
253
+
254
+ download_to_temp(path)
255
+ end
256
+
257
+ # Download URL to temporary file
258
+ # @param url [String] URL to download
259
+ # @return [String] Temporary file path
260
+ def download_to_temp(url)
261
+ require "tempfile"
262
+
263
+ uri = URI.parse(url)
264
+ filename = File.basename(uri.path)
265
+
266
+ temp = Tempfile.new([filename, ""], encoding: "UTF-8")
267
+ temp.binmode
268
+
269
+ URI.open(uri, "rb") do |remote_file|
270
+ IO.copy_stream(remote_file, temp)
271
+ end
272
+
273
+ temp.close
274
+ temp.path
275
+ end
276
+
277
+ public
278
+
279
+ # Check if a word exists in the dictionary.
280
+ #
281
+ # Uses the Lookup::Lookuper algorithm for full affix and compound support.
282
+ #
283
+ # @param word [String] The word to look up
284
+ # @return [Boolean] True if the word exists
285
+ def lookup(word)
286
+ return false if word.nil? || word.empty?
287
+
288
+ # Use the Lookuper for full Hunspell algorithm support
289
+ lookuper.call(word)
290
+ end
291
+
292
+ # Generate spelling suggestions.
293
+ #
294
+ # Uses Algorithms::Suggest::Suggester for full Hunspell-compatible
295
+ # suggestion generation (edits, REP, MAP, KEY, TRY, ngram, phonetic).
296
+ #
297
+ # @param word [String] The misspelled word
298
+ # @param max_suggestions [Integer] Maximum suggestions
299
+ # @return [Array<String>] List of suggested words
300
+ def suggest(word, max_suggestions: 10)
301
+ return [] if word.nil? || word.empty?
302
+
303
+ suggester.call(word).first(max_suggestions)
304
+ end
305
+
306
+ # Add a word to the dictionary.
307
+ #
308
+ # @param word [String] The word to add
309
+ # @param flags [Array<String>] Morphological flags
310
+ # @return [Boolean] True if added
311
+ def add_word(word, flags: [])
312
+ return false if word.nil? || word.empty?
313
+
314
+ word_key = word.downcase
315
+ @word_index[word_key] = flags
316
+ reset_sweep_index
317
+
318
+ true
319
+ end
320
+
321
+ # Remove a word from the dictionary.
322
+ #
323
+ # @param word [String] The word to remove
324
+ # @return [Boolean] True if the word was removed
325
+ def remove_word(word)
326
+ return false if word.nil? || word.empty?
327
+
328
+ word_key = word.downcase
329
+ removed = @word_index.delete(word_key)
330
+ reset_sweep_index unless removed.nil?
331
+ !removed.nil?
332
+ end
333
+
334
+ # Get all words in the dictionary.
335
+ #
336
+ # @return [Array<String>] All words
337
+ def words
338
+ @word_index.keys.dup
339
+ end
340
+
341
+ # Get word variants using affix rules.
342
+ #
343
+ # @param word [String] The word
344
+ # @return [Array<String>] Word variants
345
+ def word_variants(word)
346
+ return [] if word.nil? || word.empty?
347
+
348
+ variants = []
349
+
350
+ # Get flags for this word (if any)
351
+ word_key = word.downcase
352
+ flags = @word_index[word_key] || []
353
+
354
+ # Generate prefix variants
355
+ @affix_rules[:prefix].each do |flag, rules|
356
+ next unless flags.include?(flag)
357
+
358
+ rules.each do |rule|
359
+ variant = rule.apply(word)
360
+ variants << variant if variant
361
+ end
362
+ end
363
+
364
+ # Generate suffix variants
365
+ @affix_rules[:suffix].each do |flag, rules|
366
+ next unless flags.include?(flag)
367
+
368
+ rules.each do |rule|
369
+ variant = rule.apply(word)
370
+ variants << variant if variant
371
+ end
372
+ end
373
+
374
+ variants
375
+ end
376
+
377
+ private
378
+
379
+ # Load the dictionary file.
380
+ #
381
+ # @param path [String] Path to .dic file
382
+ # @return [Hash] Word index (word => flags)
383
+ def load_dic_file(path)
384
+ index = {}
385
+ lines = File.readlines(path, chomp: true)
386
+
387
+ # First line is word count
388
+ return index if lines.empty?
389
+
390
+ # Parse remaining lines
391
+ lines[1..].each do |line|
392
+ next if line.empty? || line.start_with?("#") || line.strip.empty?
393
+
394
+ parts = line.split("/")
395
+ word = parts[0]
396
+
397
+ # Skip if word is nil or empty after stripping
398
+ next if word.nil? || word.strip.empty?
399
+
400
+ word = word.strip
401
+ flags = parts[1] ? parts[1].split("") : []
402
+
403
+ index[word.downcase] = flags
404
+ end
405
+
406
+ index
407
+ end
408
+
409
+ # Load the affix file.
410
+ #
411
+ # @param path [String] Path to .aff file
412
+ # @return [Hash] Configuration options
413
+ def load_aff_file(path)
414
+ config = {
415
+ set: "UTF-8",
416
+ try: "",
417
+ flag: "char", # or "long" or "num"
418
+ affix_rules: []
419
+ }
420
+
421
+ File.foreach(path, chomp: true) do |line|
422
+ next if line.empty? || line.start_with?("#")
423
+
424
+ parts = line.split
425
+ next if parts.empty?
426
+
427
+ keyword = parts[0].upcase
428
+
429
+ case keyword
430
+ when "SET"
431
+ config[:set] = parts[1]
432
+ when "TRY"
433
+ config[:try] = parts[1]
434
+ when "FLAG"
435
+ config[:flag] = parts[1]
436
+ when "PFX", "SFX"
437
+ config[:affix_rules] << line
438
+ when "REP", "MAP", "COMPOUNDRULE", "COMPOUNDWORDMIN", "COMPOUNDFLAG"
439
+ # Store for future use
440
+ config[keyword.downcase.to_sym] ||= []
441
+ config[keyword.downcase.to_sym] << line
442
+ end
443
+ end
444
+
445
+ config
446
+ end
447
+
448
+ # Direct lookup without affix processing.
449
+ #
450
+ # @param word [String] The word
451
+ # @return [Boolean] True if word exists
452
+ def direct_lookup?(word)
453
+ word_key = word.downcase
454
+ @word_index.key?(word_key)
455
+ end
456
+
457
+ # Generate all possible affix variants.
458
+ #
459
+ # @return [Array<String>] All variants
460
+ def generate_affix_variants
461
+ variants = []
462
+
463
+ @affix_rules[:prefix].each do |flag, rules|
464
+ rules.each do |rule|
465
+ @word_index.each do |word, flags|
466
+ next unless flags.include?(flag)
467
+
468
+ variant = rule.apply(word)
469
+ variants << variant if variant
470
+ end
471
+ end
472
+ end
473
+
474
+ @affix_rules[:suffix].each do |flag, rules|
475
+ rules.each do |rule|
476
+ @word_index.each do |word, flags|
477
+ next unless flags.include?(flag)
478
+
479
+ variant = rule.apply(word)
480
+ variants << variant if variant
481
+ end
482
+ end
483
+ end
484
+
485
+ variants.uniq
486
+ end
487
+
488
+ # Calculate Levenshtein edit distance.
489
+ #
490
+ # @param str1 [String] First string
491
+ # @param str2 [String] Second string
492
+ # @return [Integer] Edit distance
493
+ def edit_distance(str1, str2)
494
+ return str2.length if str1.empty?
495
+ return str1.length if str2.empty?
496
+
497
+ # Use smaller string for inner loop
498
+ str1, str2 = str2, str1 if str1.length > str2.length
499
+
500
+ previous = (0..str1.length).to_a
501
+
502
+ str2.each_char.with_index do |char2, j|
503
+ current = [j + 1]
504
+
505
+ str1.each_char.with_index do |char1, i|
506
+ insert_cost = current[i] + 1
507
+ delete_cost = previous[i + 1] + 1
508
+ substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
509
+
510
+ current << [insert_cost, delete_cost, substitute_cost].min
511
+ end
512
+
513
+ previous = current
514
+ end
515
+
516
+ previous.last
517
+ end
518
+ end
519
+ end
520
+ end