kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,584 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Readers
5
+ # Builder for creating Lookup::Lookuper instances from Hunspell data.
6
+ #
7
+ # This class can either read from files or accept pre-read aff/dic data.
8
+ #
9
+ # @example Building a lookuper from files
10
+ # builder = LookupBuilder.new('en_US.aff', 'en_US.dic')
11
+ # lookuper = builder.build
12
+ #
13
+ # @example Building a lookuper from pre-read data
14
+ # aff_reader = AffReader.new('en_US.aff')
15
+ # aff_data = aff_reader.read
16
+ # dic_reader = DicReader.new('en_US.dic')
17
+ # words = dic_reader.read
18
+ # builder = LookupBuilder.from_data(aff_data, words)
19
+ # lookuper = builder.build
20
+ class LookupBuilder
21
+ # Hunspell upstream defaults for suggester directives. These apply
22
+ # when the .aff file is silent on the directive. See the Hunspell
23
+ # documentation at https://hunspell.github.io/ and the spylls
24
+ # `Aff` defaults for reference.
25
+ DEFAULT_MAX_NGRAM_SUGS = 4
26
+ DEFAULT_MAX_CPD_SUGS = 3
27
+ DEFAULT_MAX_DIFF = 5
28
+
29
+ attr_reader :aff_path, :dic_path, :encoding, :aff_data, :words, :script
30
+
31
+ # Create a new LookupBuilder from file paths.
32
+ #
33
+ # @param aff_path [String, nil] Path to the .aff file (nil when
34
+ # constructing via .from_data with pre-read aff/dic data)
35
+ # @param dic_path [String, nil] Path to the .dic file
36
+ # @param encoding [String] File encoding (default: 'UTF-8')
37
+ # @param script [Symbol] The script type for condition checking (default: :latin)
38
+ # @param aff_data [Hash, nil] Pre-read aff data (skips file read
39
+ # in #build when provided)
40
+ # @param words [Array<Word>, nil] Pre-read word entries
41
+ def initialize(aff_path, dic_path, encoding: 'UTF-8', script: :latin,
42
+ aff_data: nil, words: nil)
43
+ @aff_path = aff_path
44
+ @dic_path = dic_path
45
+ @encoding = encoding
46
+ @script = script
47
+ @aff_data = aff_data
48
+ @words = words
49
+ end
50
+
51
+ # Create a new LookupBuilder from pre-read data.
52
+ #
53
+ # @param aff_data [Hash] Raw aff data from AffReader
54
+ # @param words [Array<Word>] Word entries from DicReader
55
+ # @return [LookupBuilder] A new builder instance
56
+ def self.from_data(aff_data, words)
57
+ new(nil, nil, aff_data: aff_data, words: words)
58
+ end
59
+
60
+ # Build the Lookuper instance.
61
+ #
62
+ # @return [Algorithms::Lookup::Lookuper] The lookuper instance
63
+ def build
64
+ # Read files if data not already provided
65
+ aff_data_to_use = @aff_data || read_aff_data
66
+ words_to_use = @words || read_dic_data(aff_data_to_use)
67
+
68
+ # Build the aff structure for Lookuper
69
+ aff = build_aff_structure(aff_data_to_use)
70
+
71
+ # Build the dic structure for Lookuper. Dictionary `ph:` morph data
72
+ # is a source of REP entries (Hunspell 1.7+), so build_dic_structure
73
+ # also enriches aff[:REP] — see PhRepExtractor.
74
+ dic = build_dic_structure(words_to_use, aff: aff)
75
+
76
+ # Create and return the Lookuper
77
+ Algorithms::Lookup::Lookuper.new(aff, dic)
78
+ end
79
+
80
+ private
81
+
82
+ # Read aff data from file.
83
+ #
84
+ # @return [Hash] Raw aff data
85
+ def read_aff_data
86
+ aff_reader = AffReader.new(@aff_path, encoding: @encoding)
87
+ aff_reader.read
88
+ end
89
+
90
+ # Read dic data from file.
91
+ #
92
+ # @param aff_data [Hash] Aff data for flag format info
93
+ # @return [Array<Word>] Word entries
94
+ def read_dic_data(aff_data)
95
+ dic_reader = DicReader.new(@dic_path,
96
+ encoding: @encoding,
97
+ flag_format: aff_data['FLAG'] || 'short',
98
+ flag_synonyms: aff_data['AF'] || {})
99
+ dic_reader.read
100
+ end
101
+
102
+ # Select the appropriate Casing class based on the aff file's directives.
103
+ #
104
+ # Hunspell associates a casing strategy with the language declared via
105
+ # LANG or with specific directives like CHECKSHARPS (German ß↔SS) and
106
+ # Turkic languages (i↔İ / ı↔I). We mirror that here.
107
+ def select_casing(aff_data)
108
+ lang = aff_data['LANG'].to_s.downcase
109
+
110
+ if aff_data['CHECKSHARPS']
111
+ Algorithms::Capitalization::GermanCasing.new
112
+ elsif lang.start_with?('tr', 'az', 'crh', 'tt', 'krc', 'kaa')
113
+ Algorithms::Capitalization::TurkicCasing.new
114
+ elsif lang.start_with?('de')
115
+ Algorithms::Capitalization::GermanCasing.new
116
+ else
117
+ Algorithms::Capitalization::Casing.new
118
+ end
119
+ end
120
+
121
+ # Build the aff data structure for Lookuper.
122
+ #
123
+ # @param aff_data [Hash] Raw aff data from AffReader
124
+ # @return [Hash] Aff structure for Lookuper
125
+ def build_aff_structure(aff_data)
126
+ aff = {}
127
+
128
+ # Capitalization handler — German when CHECKSHARPS is set (the
129
+ # German ß↔SS rule needs special handling: lower("STRASSE") yields
130
+ # both "straße" and "strasse"), Turkic when LANG starts with az/tr,
131
+ # otherwise the standard Casing.
132
+ aff[:casing] = select_casing(aff_data)
133
+
134
+ # IGNORE chars (as Array<String>) — applied to dictionary stems
135
+ # and affix `add` strings at read time so input lookups (which
136
+ # strip IGNORE from the query) match. Without this, prefix "re" +
137
+ # stem "expression" could never match input "reexpression" once
138
+ # the input is reduced to "rxprssn".
139
+ ignore_chars = aff_data['IGNORE']&.chars&.chars || []
140
+
141
+ # Build suffixes index (indexed by first character of reversed suffix)
142
+ suffixes_index = {}
143
+ suffixes_by_flag = Hash.new { |h, k| h[k] = [] }
144
+ aff_data['SFX'].each do |_flag, affix_list|
145
+ affix_list.each do |affix|
146
+ stripped_add = strip_ignore(affix.add, ignore_chars)
147
+ # For suffixes, we need to index by the first char of the REVERSED suffix
148
+ # because the lookup code reverses the word to check suffixes
149
+ reversed_suffix = stripped_add.reverse
150
+ first_char = reversed_suffix[0] || ''
151
+ suffixes_index[first_char] ||= []
152
+ affix_hash = build_affix_hash(affix, script: @script || :latin,
153
+ add: stripped_add)
154
+ suffixes_index[first_char] << affix_hash
155
+ suffixes_by_flag[affix.flag] << affix_hash
156
+ end
157
+ end
158
+ aff[:suffixes_index] = suffixes_index
159
+ aff[:suffixes_by_flag] = suffixes_by_flag
160
+
161
+ # Build prefixes index (indexed by first character of prefix)
162
+ prefixes_index = {}
163
+ prefixes_by_flag = Hash.new { |h, k| h[k] = [] }
164
+ aff_data['PFX'].each do |_flag, affix_list|
165
+ affix_list.each do |affix|
166
+ stripped_add = strip_ignore(affix.add, ignore_chars)
167
+ first_char = stripped_add[0] || ''
168
+ prefixes_index[first_char] ||= []
169
+ affix_hash = build_affix_hash(affix, script: @script || :latin,
170
+ add: stripped_add)
171
+ prefixes_index[first_char] << affix_hash
172
+ prefixes_by_flag[affix.flag] << affix_hash
173
+ end
174
+ end
175
+ aff[:prefixes_index] = prefixes_index
176
+ aff[:prefixes_by_flag] = prefixes_by_flag
177
+
178
+ # Single-value flags
179
+ aff[:COMPOUNDMIN] = aff_data['COMPOUNDMIN']
180
+ aff[:COMPOUNDWORDMAX] = aff_data['COMPOUNDWORDMAX']
181
+ aff[:COMPOUNDBEGIN] = aff_data['COMPOUNDBEGIN']
182
+ aff[:COMPOUNDMIDDLE] = aff_data['COMPOUNDMIDDLE']
183
+ aff[:COMPOUNDEND] = aff_data['COMPOUNDEND']
184
+ aff[:COMPOUNDFLAG] = aff_data['COMPOUNDFLAG']
185
+ aff[:COMPOUNDPERMITFLAG] = aff_data['COMPOUNDPERMITFLAG']
186
+ aff[:COMPOUNDFORBIDFLAG] = aff_data['COMPOUNDFORBIDFLAG']
187
+ aff[:COMPOUNDRULE] = build_compound_rules(aff_data['COMPOUNDRULE'])
188
+ aff[:ONLYINCOMPOUND] = aff_data['ONLYINCOMPOUND']
189
+ aff[:COMPLEXPREFIXES] = aff_data['COMPLEXPREFIXES']
190
+ aff[:FORCEUCASE] = aff_data['FORCEUCASE']
191
+
192
+ # Special flags
193
+ aff[:FORBIDDENWORD] = aff_data['FORBIDDENWORD']
194
+ aff[:NOSUGGEST] = aff_data['NOSUGGEST']
195
+ aff[:KEEPCASE] = aff_data['KEEPCASE']
196
+ aff[:NEEDAFFIX] = aff_data['NEEDAFFIX']
197
+ aff[:CIRCUMFIX] = aff_data['CIRCUMFIX']
198
+ aff[:WARN] = aff_data['WARN']
199
+
200
+ # Compound checking flags
201
+ aff[:CHECKCOMPOUNDCASE] = aff_data['CHECKCOMPOUNDCASE']
202
+ aff[:CHECKCOMPOUNDDUP] = aff_data['CHECKCOMPOUNDDUP']
203
+ aff[:CHECKCOMPOUNDREP] = aff_data['CHECKCOMPOUNDREP']
204
+ aff[:CHECKCOMPOUNDTRIPLE] = aff_data['CHECKCOMPOUNDTRIPLE']
205
+ aff[:CHECKCOMPOUNDPATTERN] = aff_data['CHECKCOMPOUNDPATTERN'] || []
206
+ aff[:SIMPLIFIEDTRIPLE] = aff_data['SIMPLIFIEDTRIPLE']
207
+
208
+ # Other directives
209
+ aff[:IGNORE] = aff_data['IGNORE']&.chars&.chars || []
210
+ aff[:BREAK] = build_break_patterns(aff_data['BREAK'])
211
+ aff[:ICONV] = aff_data['ICONV']
212
+ aff[:OCONV] = aff_data['OCONV']
213
+ aff[:REP] = build_rep_table(aff_data['REP'])
214
+ aff[:MAP] = aff_data['MAP'] || []
215
+ aff[:CHECKSHARPS] = aff_data['CHECKSHARPS']
216
+
217
+ # Suggester directives (Hunspell defaults applied where the .aff
218
+ # is silent — these defaults come from the upstream Hunspell
219
+ # manual and match spylls' Aff.objects defaults).
220
+ aff[:TRY] = aff_data['TRY']
221
+ aff[:KEY] = aff_data['KEY']
222
+ aff[:WORDCHARS] = aff_data['WORDCHARS']
223
+ aff[:PHONE] = aff_data['PHONE']
224
+ aff[:NOSPLITSUGS] = aff_data['NOSPLITSUGS']
225
+ aff[:SUGSWITHDOTS] = aff_data['SUGSWITHDOTS']
226
+ aff[:FULLSTRIP] = aff_data['FULLSTRIP']
227
+ aff[:MAXNGRAMSUGS] = aff_data['MAXNGRAMSUGS'] || DEFAULT_MAX_NGRAM_SUGS
228
+ aff[:MAXCPDSUGS] = aff_data['MAXCPDSUGS'] || DEFAULT_MAX_CPD_SUGS
229
+ aff[:MAXDIFF] = aff_data['MAXDIFF'] || DEFAULT_MAX_DIFF
230
+ aff[:ONLYMAXDIFF] = aff_data['ONLYMAXDIFF']
231
+
232
+ aff
233
+ end
234
+
235
+ # Build the dic data structure for Lookuper.
236
+ #
237
+ # Two stem indexes are maintained, mirroring Spylls's `Dic.index` and
238
+ # `Dic.lowercase_index`:
239
+ #
240
+ # * `word_index` — keyed by the exact stem as it appears in the .dic file.
241
+ # * `lowercase_word_index` — keyed by the lowercased stem. Hunspell's
242
+ # ALLCAPS lookup path uses this to find dictionary entries whose stem
243
+ # casing doesn't match (e.g. querying "unicef" against a "UNICEF" entry).
244
+ #
245
+ # As a side effect, dictionary `ph:` morph data is folded into the
246
+ # aff[:REP] table — Hunspell 1.7+ exposes `ph:` as additional REP
247
+ # patterns so the existing replchars permutation surfaces them. See
248
+ # {PhRepExtractor} for the three supported forms.
249
+ #
250
+ # @param words [Array<Word>] List of word entries
251
+ # @param aff [Hash] Aff structure being built (mutated to add REP)
252
+ # @return [Hash] Dic structure for Lookuper
253
+ def build_dic_structure(words, aff:)
254
+ word_index = Hash.new { |h, k| h[k] = [] }
255
+ lowercase_index = Hash.new { |h, k| h[k] = [] }
256
+ word_list = []
257
+ casing = aff[:casing]
258
+ ignore_chars = aff[:IGNORE] || []
259
+
260
+ words.each do |word|
261
+ ph_tokens = ph_tokens_from(word.morph_data)
262
+ # Hunspell applies IGNORE at read time: any char listed in IGNORE
263
+ # is stripped from dictionary stems so that input words with the
264
+ # same char removed (done in Lookuper#call) match. Without this,
265
+ # the `ignore` fixture's "expression" would never match a dict
266
+ # entry of "expression" — both need to be reduced to "exprssn".
267
+ stem = ignore_chars.any? ? strip_ignore(word.stem, ignore_chars) : word.stem
268
+ entry = {
269
+ stem: stem,
270
+ flags: word.flags.to_a,
271
+ alt_spellings: PhRepExtractor.simple_alt_spellings(stem, ph_tokens)
272
+ }
273
+ word_index[stem] << entry
274
+ word_list << entry
275
+
276
+ # The lowercase index mirrors Spylls's `Dic.lowercase_index`:
277
+ # only words whose casing is not NO (i.e. INIT/ALL/HUH/HUHINIT)
278
+ # contribute a lowercased key, because a NO-cased word's stem is
279
+ # already its own lowercase form and would just duplicate the
280
+ # exact-stem index entry.
281
+ captype = casing.guess(stem)
282
+ if captype != Algorithms::Capitalization::Type::NO
283
+ casing.lower(stem).each do |lowered|
284
+ lowercase_index[lowered] << entry
285
+ end
286
+ end
287
+
288
+ PhRepExtractor.append_to_aff(aff, stem, ph_tokens)
289
+ end
290
+
291
+ {
292
+ words: word_list,
293
+ homonyms: ->(w, ignorecase: false) {
294
+ if ignorecase
295
+ lowercase_index[w] || []
296
+ else
297
+ word_index[w] || []
298
+ end
299
+ },
300
+ has_flag: ->(w, flag, for_all: false) {
301
+ entries = word_index[w] || []
302
+ # Spylls's `Dic.has_flag` returns False when there are no homonyms
303
+ # at all — a vacuous `for_all: true` would otherwise block the
304
+ # caller (e.g. the FORBIDDENWORD guard in Lookuper#call) on
305
+ # words whose case doesn't match the dictionary entry.
306
+ return false if entries.empty?
307
+
308
+ if for_all
309
+ entries.all? { |e| (e[:flags] || []).include?(flag) }
310
+ else
311
+ entries.any? { |e| (e[:flags] || []).include?(flag) }
312
+ end
313
+ }
314
+ }
315
+ end
316
+
317
+ # Select only the `ph:` morphological tokens.
318
+ #
319
+ # @param morph_data [Array<String>, nil] Morphological tokens
320
+ # @return [Array<String>] Raw `ph:` payloads (with the `ph:` prefix stripped)
321
+ def ph_tokens_from(morph_data)
322
+ return [] if morph_data.nil? || morph_data.empty?
323
+
324
+ morph_data.filter_map { |token| token[3..] if token.start_with?('ph:') }
325
+ end
326
+
327
+ # Remove IGNORE chars from a string. Mirrors Spylls's
328
+ # `word.translate(ignore.tr)` — used at read time for both stems and
329
+ # affix `add` strings so that lookups (which strip IGNORE from the
330
+ # input) match.
331
+ #
332
+ # @param str [String] Input string
333
+ # @param ignore_chars [Array<String>] Chars to remove
334
+ # @return [String] String with ignored chars removed
335
+ def strip_ignore(str, ignore_chars)
336
+ return str if ignore_chars.empty?
337
+
338
+ str.chars.reject { |c| ignore_chars.include?(c) }.join
339
+ end
340
+
341
+ # Build an affix hash for Lookuper.
342
+ #
343
+ # @param affix [Affix] The affix object
344
+ # @param script [Symbol] The script type for condition checking
345
+ # @param add [String, nil] Pre-stripped `add` string (IGNORE chars
346
+ # removed). When nil, the affix's raw `add` is used.
347
+ # @return [Hash] Affix hash for Lookuper
348
+ def build_affix_hash(affix, script: :latin, add: nil)
349
+ add ||= affix.add
350
+ {
351
+ flag: affix.flag,
352
+ crossproduct: affix.crossproduct,
353
+ strip: affix.strip,
354
+ affix: add,
355
+ condition_checker: compile_condition_matcher(affix.condition,
356
+ script: script,
357
+ type: affix.type),
358
+ affix_data: build_affix_transform(affix.strip, add, type: affix.type),
359
+ flags: affix.flags.to_a
360
+ }
361
+ end
362
+
363
+ # Compile a condition checker.
364
+ #
365
+ # Hunspell conditions are anchored: for suffixes the matcher checks
366
+ # the END of the stem (e.g. `[^y]$`), for prefixes the START
367
+ # (e.g. `^ij`). Without this distinction, prefix conditions silently
368
+ # never match — Dutch "ijs" with PFX condition `ij` would never get
369
+ # its prefix applied, so "IJs" wouldn't surface.
370
+ #
371
+ # @param condition [String] Condition string from .aff file
372
+ # @param script [Symbol] The script type (:latin, :arabic, etc.)
373
+ # @param type [Symbol] :prefix or :suffix (anchor direction)
374
+ # @return [ConditionChecker, nil] Compiled checker or nil
375
+ def compile_condition_matcher(condition, script: :latin, type: :suffix)
376
+ return nil if condition.nil? || condition.empty?
377
+
378
+ ConditionChecker.compile(condition, script: script, type: type)
379
+ end
380
+
381
+ # Build affix stripping data.
382
+ #
383
+ # Build affix transformation data.
384
+ #
385
+ # @param strip [String] Characters to strip
386
+ # @param add [String] Characters to add
387
+ # @param type [Symbol] :prefix or :suffix
388
+ # @return [Hash] Hash with affix data for transformation
389
+ def build_affix_transform(strip, add, type:)
390
+ return nil if strip.empty? && add.empty?
391
+
392
+ {
393
+ add: add,
394
+ strip: strip || '',
395
+ type: type
396
+ }
397
+ end
398
+
399
+ # Build compound rules array.
400
+ #
401
+ # @param rules [Array<CompoundRule>] List of compound rules
402
+ # @return [Array<Hash>] Array of compound rule hashes
403
+ def build_compound_rules(rules)
404
+ return [] if rules.nil? || rules.empty?
405
+
406
+ rules.map do |rule|
407
+ {
408
+ text: rule.text,
409
+ flags: rule.flags,
410
+ full_match: ->(flag_sets) { rule.fullmatch(flag_sets) },
411
+ partial_match: ->(flag_sets) { rule.partial_match(flag_sets) }
412
+ }
413
+ end
414
+ end
415
+
416
+ # Build break patterns array.
417
+ #
418
+ # Spylls's default BREAK is `['-', '^-', '-$']` (applied when the
419
+ # directive is absent). If the .aff file says `BREAK 0` explicitly,
420
+ # the result must be EMPTY — `BREAK 0` is the documented way to
421
+ # disable hyphen breaking. The reader signals "absent" with nil and
422
+ # "explicit zero" with [], so we branch on nil here.
423
+ #
424
+ # @param break_patterns [Array<BreakPattern>, nil] List of break patterns
425
+ # @return [Array<Hash>] Array of break pattern hashes
426
+ def build_break_patterns(break_patterns)
427
+ if break_patterns.nil?
428
+ return [
429
+ { pattern: '-', matcher: BreakPattern.new('-').matcher },
430
+ { pattern: '^-', matcher: BreakPattern.new('^-').matcher },
431
+ { pattern: '-$', matcher: BreakPattern.new('-$').matcher }
432
+ ]
433
+ end
434
+
435
+ break_patterns.map do |bp|
436
+ {
437
+ pattern: bp.pattern,
438
+ matcher: bp.matcher
439
+ }
440
+ end
441
+ end
442
+
443
+ # Build REP table for the Suggester.
444
+ #
445
+ # The Suggester's `replchars` permutation expects each entry as a
446
+ # Hash with `:regexp` (a Regexp) and `:replacement` (a String).
447
+ # AffReader returns `Readers::RepPattern` value objects; this
448
+ # method adapts that representation without poking at the
449
+ # RepPattern's internals — it uses the public accessors.
450
+ #
451
+ # @param rep_patterns [Array<RepPattern>, nil] REP entries
452
+ # @return [Array<Hash{Symbol=>Object}>] REP table for Suggester
453
+ def build_rep_table(rep_patterns)
454
+ return [] if rep_patterns.nil? || rep_patterns.empty?
455
+
456
+ rep_patterns.map do |rp|
457
+ {
458
+ regexp: rp.matcher,
459
+ pattern: rp.pattern,
460
+ replacement: rp.replacement
461
+ }
462
+ end
463
+ end
464
+ end
465
+
466
+ # Extracts REP entries from dictionary `ph:` morphological data.
467
+ #
468
+ # Hunspell 1.7+ treats `ph:` as a phonetic misspelling hint and reuses
469
+ # the existing replchars permutation to surface corrections. To make
470
+ # that work, `ph:` payloads are converted to REP entries at dic-read
471
+ # time. Three payload forms are recognised (mirroring Spylls/Hunspell):
472
+ #
473
+ # * simple — `ph:wich` against stem "which" yields REP(`wich`,
474
+ # `which`) and registers `wich` as an alt_spelling for ngram
475
+ # scoring.
476
+ # * star — `ph:prity*` against stem "pretty" yields REP(`prit`,
477
+ # `prett`): the trailing char of the pattern (and the `*`) is
478
+ # stripped, and the trailing char of the stem is stripped, so the
479
+ # REP entry can attach suffixes on either side (matches "prity" →
480
+ # "pretty" and "pritiest" → "prettiest").
481
+ # * arrow — `ph:hepi->happi` is an explicit pair: REP(`hepi`,
482
+ # `happi`). The stem is irrelevant for arrow form.
483
+ #
484
+ # The class is a pure transformer; side effects on the aff hash are
485
+ # performed by the caller via {append_to_aff}.
486
+ module PhRepExtractor
487
+ module_function
488
+
489
+ # Return only the *simple* payloads — the ones that double as
490
+ # alt_spellings for ngram scoring. Star and arrow forms are excluded
491
+ # because their REP entries don't map 1:1 to the stem and would
492
+ # mislead ngram.
493
+ #
494
+ # @param _stem [String] Dictionary stem (unused, present for symmetry)
495
+ # @param ph_tokens [Array<String>] Raw `ph:` payloads
496
+ # @return [Array<String>] Simple alt spellings
497
+ def simple_alt_spellings(_stem, ph_tokens)
498
+ ph_tokens.reject { |token| token.end_with?('*') || token.include?('->') }
499
+ end
500
+
501
+ # Convert each `ph:` payload into a {RepPattern} and append it to
502
+ # `aff[:REP]`. Mutates the aff hash.
503
+ #
504
+ # @param aff [Hash] Aff structure (mutated)
505
+ # @param stem [String] Dictionary stem
506
+ # @param ph_tokens [Array<String>] Raw `ph:` payloads
507
+ # @return [void]
508
+ def append_to_aff(aff, stem, ph_tokens)
509
+ return if ph_tokens.nil? || ph_tokens.empty?
510
+
511
+ aff[:REP] ||= []
512
+ ph_tokens.each do |token|
513
+ rep = build_rep(stem, token)
514
+ aff[:REP] << rep_hash(rep) if rep
515
+ end
516
+ end
517
+
518
+ # Build a RepPattern from a single `ph:` payload.
519
+ #
520
+ # @param stem [String] Dictionary stem
521
+ # @param token [String] Raw `ph:` payload
522
+ # @return [RepPattern, nil]
523
+ def build_rep(stem, token)
524
+ if token.end_with?('*')
525
+ star_rep(stem, token)
526
+ elsif token.include?('->')
527
+ arrow_rep(token)
528
+ else
529
+ simple_rep(stem, token)
530
+ end
531
+ end
532
+
533
+ # Convert a RepPattern to the Hash shape used by Permutations.replchars.
534
+ #
535
+ # @param rep [RepPattern]
536
+ # @return [Hash{Symbol=>Object}]
537
+ def rep_hash(rep)
538
+ {
539
+ regexp: rep.matcher,
540
+ pattern: rep.pattern,
541
+ replacement: rep.replacement
542
+ }
543
+ end
544
+
545
+ # `ph:prity*` (against stem "pretty") → REP(`prit`, `prett`).
546
+ #
547
+ # Strip the last two chars of the pattern (`y*`) and the last char
548
+ # of the stem (`y`), so affixed forms on either side can match.
549
+ #
550
+ # @param stem [String] Dictionary stem
551
+ # @param token [String] Star-form payload (e.g. `prity*`)
552
+ # @return [RepPattern, nil] `nil` if pattern or stem too short
553
+ def star_rep(stem, token)
554
+ pattern = token[0...-2]
555
+ replacement = stem[0...-1]
556
+ return nil if pattern.empty? || replacement.empty?
557
+
558
+ RepPattern.new(pattern, replacement)
559
+ end
560
+
561
+ # `ph:hepi->happi` → REP(`hepi`, `happi`).
562
+ #
563
+ # @param token [String] Arrow-form payload
564
+ # @return [RepPattern, nil] `nil` if either side is empty
565
+ def arrow_rep(token)
566
+ from, to = token.split('->', 2)
567
+ return nil if from.nil? || to.nil? || from.empty? || to.empty?
568
+
569
+ RepPattern.new(from, to)
570
+ end
571
+
572
+ # `ph:wich` (against stem "which") → REP(`wich`, `which`).
573
+ #
574
+ # @param stem [String] Dictionary stem
575
+ # @param token [String] Simple payload
576
+ # @return [RepPattern, nil] `nil` if pattern is empty
577
+ def simple_rep(stem, token)
578
+ return nil if token.empty?
579
+
580
+ RepPattern.new(token, stem)
581
+ end
582
+ end
583
+ end
584
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is intentionally a no-op stub.
4
+ #
5
+ # History: an earlier draft of the readers layer used `require_relative`
6
+ # here to eagerly load every reader file. The current design uses
7
+ # autoload declared in `lib/kotoshu/readers.rb` (the parent namespace
8
+ # file), so this nested file is redundant — and the `require_relative`
9
+ # calls violated the project rule against `require_relative` in lib/.
10
+ #
11
+ # Loading the original version of this file re-required already-loaded
12
+ # files (harmless but useless) and bypassed the autoload lazy-loading
13
+ # design. Rather than delete the file (project policy preserves files
14
+ # contributors may have local context for), it is reduced to this
15
+ # stub so any future `require` of it succeeds harmlessly.
16
+ #
17
+ # For the actual reader autoloads see: lib/kotoshu/readers.rb
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Hunspell readers module for reading dictionary and affix files.
5
+ module Readers
6
+ autoload :FileReader, "kotoshu/readers/file_reader"
7
+ autoload :StringReader, "kotoshu/readers/file_reader"
8
+ autoload :ZipReader, "kotoshu/readers/file_reader"
9
+ autoload :Affix, "kotoshu/readers/aff_data"
10
+ autoload :BreakPattern, "kotoshu/readers/aff_data"
11
+ autoload :Ignore, "kotoshu/readers/aff_data"
12
+ autoload :RepPattern, "kotoshu/readers/aff_data"
13
+ autoload :ConvTable, "kotoshu/readers/aff_data"
14
+ autoload :CompoundRule, "kotoshu/readers/aff_data"
15
+ autoload :CompoundPattern, "kotoshu/readers/aff_data"
16
+ autoload :PhonetTable, "kotoshu/readers/aff_data"
17
+ autoload :AffReader, "kotoshu/readers/aff_reader"
18
+ autoload :DicReader, "kotoshu/readers/dic_reader"
19
+ autoload :ConditionChecker, "kotoshu/readers/condition_checker"
20
+ autoload :PassthroughConditionChecker, "kotoshu/readers/condition_checker"
21
+ autoload :LatinScriptConditionChecker, "kotoshu/readers/condition_checker"
22
+ autoload :LookupBuilder, "kotoshu/readers/lookup_builder"
23
+ autoload :PhRepExtractor, "kotoshu/readers/lookup_builder"
24
+ end
25
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # The Stage-2 result: every resource {ResourceManager.resolve} found
5
+ # for one language, ready to hand to `Spellchecker.new`.
6
+ #
7
+ # A member is nil for any resource that was not requested in the
8
+ # `want:` list or is not set up. The bundle is cache-only data —
9
+ # building one never touches the network.
10
+ #
11
+ # @example
12
+ # Kotoshu.setup(:en)
13
+ # bundle = Kotoshu::ResourceManager.resolve(language: "en")
14
+ # bundle.dictionary # => #<Kotoshu::Dictionary::Hunspell ...>
15
+ # bundle.has_model? # => false (model not requested)
16
+ ResourceBundle = Struct.new(
17
+ :language, # [String] Language code the bundle was resolved for
18
+ :dictionary, # [Dictionary::Hunspell, nil] Spelling dictionary
19
+ :frequency, # [Object, nil] Frequency data (Kelly tiers)
20
+ :model, # [Object, nil] ONNX embedding model
21
+ :rules, # [Object, nil] Grammar rules (reserved; nil today)
22
+ :cached, # [Boolean] True — resolve is cache-only by contract
23
+ :source_urls, # [Array<String>] URLs the resources came from
24
+ keyword_init: true
25
+ ) do
26
+ # Whether the bundle's data came from the cache (always true for
27
+ # ResourceManager-resolved bundles; kept for forward compatibility).
28
+ #
29
+ # @return [Boolean]
30
+ def cached?
31
+ cached ? true : false
32
+ end
33
+
34
+ # Whether a frequency list is included.
35
+ #
36
+ # @return [Boolean]
37
+ def has_frequency?
38
+ !frequency.nil?
39
+ end
40
+
41
+ # Whether an ONNX model is included.
42
+ #
43
+ # @return [Boolean]
44
+ def has_model?
45
+ !model.nil?
46
+ end
47
+
48
+ # Whether grammar rules are included (reserved; false today).
49
+ #
50
+ # @return [Boolean]
51
+ def has_rules?
52
+ !rules.nil?
53
+ end
54
+ end
55
+ end