kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,276 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Capitalization handling for different languages.
6
+ #
7
+ # Ported from Spylls (Python) capitalization.py
8
+ #
9
+ # This module provides capitalization detection and conversion for different
10
+ # language casing rules, including special handling for Turkic and German languages.
11
+ module Capitalization
12
+ # Type of capitalization detected by Casing.guess.
13
+ #
14
+ # NO:: all lowercase ("foo")
15
+ # INIT:: titlecase, only initial letter is capitalized ("Foo")
16
+ # ALL:: all uppercase ("FOO")
17
+ # HUH:: mixed capitalization ("fooBar")
18
+ # HUHINIT:: mixed capitalization, first letter is capitalized ("FooBar")
19
+ module Type
20
+ NO = :no
21
+ INIT = :init
22
+ ALL = :all
23
+ HUH = :huh
24
+ HUHINIT = :huhinit
25
+ end
26
+
27
+ # Base class for casing-related algorithms specific for dictionary's language.
28
+ #
29
+ # This is a class (not a set of functions) because it needs to have
30
+ # subclasses for specific language casing, which have only some aspects
31
+ # different from generic one.
32
+ class Casing
33
+ # Guess word's capitalization. Redefined in GermanCasing.
34
+ #
35
+ # @param word [String] The word to analyze
36
+ # @return [Symbol] One of the Type constants
37
+ def guess(word)
38
+ return Type::NO if word.downcase == word
39
+ return Type::ALL if word.upcase == word
40
+ return Type::INIT if word[0].upcase == word[0] && word[1..].downcase == word[1..]
41
+
42
+ if word[0].upcase == word[0]
43
+ Type::HUHINIT
44
+ else
45
+ Type::HUH
46
+ end
47
+ end
48
+
49
+ # Lowercases the word. Returns list of possible lowercasings for all
50
+ # casing classes to behave consistently.
51
+ #
52
+ # In GermanCasing (and only there), lowercasing word like "STRASSE"
53
+ # produces two possibilities: "strasse" and "ße" (ß is most of the time
54
+ # upcased to SS, so we can't decide which of downcased words is "right"
55
+ # and need to check both).
56
+ #
57
+ # Also redefined in TurkicCasing, because in Turkic languages lowercase
58
+ # "i" is uppercased as "İ", and uppercase "I" is downcased as "ı".
59
+ #
60
+ # @param word [String] The word to lowercase
61
+ # @return [Array<String>] List of possible lowercasings
62
+ def lower(word)
63
+ # Can't be properly lowercased in non-Turkic collation
64
+ return [] if word.nil? || word.empty? || word[0] == 'İ'
65
+
66
+ # Turkic "lowercase dot i" to latinic "i", just in case
67
+ [word.downcase.gsub('i̇', 'i')]
68
+ end
69
+
70
+ # Uppercase the word. Redefined in TurkicCasing, because in Turkic
71
+ # languages lowercase "i" is uppercased as "İ", and uppercase "I"
72
+ # is downcased as "ı".
73
+ #
74
+ # @param word [String] The word to uppercase
75
+ # @return [String] Uppercased word
76
+ def upper(word)
77
+ word.upcase
78
+ end
79
+
80
+ # Capitalize (convert word to all lowercase and first letter uppercase).
81
+ # Returns a list of results for same reasons as lower.
82
+ #
83
+ # @param word [String] The word to capitalize
84
+ # @return [Enumerator<String>] Enum of capitalized variants
85
+ def capitalize(word)
86
+ return enum_for(:capitalize, word) unless block_given?
87
+
88
+ if word.length == 1
89
+ yield upper(word[0])
90
+ else
91
+ upper_first = upper(word[0])
92
+ lower(word[1..]).each do |lowered|
93
+ yield upper_first + lowered
94
+ end
95
+ end
96
+ end
97
+
98
+ # Just change the case of the first letter to lower.
99
+ # Returns a list of results for same reasons as lower.
100
+ #
101
+ # @param word [String] The word to process
102
+ # @return [Enumerator<String>] Enum of variants with lowercased first letter
103
+ def lowerfirst(word)
104
+ return enum_for(:lowerfirst, word) unless block_given?
105
+
106
+ lower(word[0]).each do |lowered|
107
+ yield lowered + word[1..]
108
+ end
109
+ end
110
+
111
+ # Returns hypotheses of how the word might have been cased (in dictionary),
112
+ # if we consider it is spelled correctly.
113
+ #
114
+ # Example: If word is "Kitten", hypotheses are "kitten", "Kitten".
115
+ #
116
+ # @param word [String] The word to analyze
117
+ # @return [Array<Symbol, Array<String>>] Pair of [captype, variants]
118
+ def variants(word)
119
+ captype = guess(word)
120
+
121
+ result = case captype
122
+ when Type::NO
123
+ [word]
124
+ when Type::INIT
125
+ [word, *lower(word)]
126
+ when Type::HUHINIT
127
+ [word, *lowerfirst(word).to_a]
128
+ when Type::HUH
129
+ [word]
130
+ when Type::ALL
131
+ [word, *lower(word), *capitalize(word).to_a]
132
+ end
133
+
134
+ [captype, result]
135
+ end
136
+
137
+ # Returns hypotheses of how the word might have been cased if it is a
138
+ # misspelling.
139
+ #
140
+ # Example: "DiCtionary" (HUHINIT capitalization) produces hypotheses
141
+ # "DiCtionary", "diCtionary", "dictionary", "Dictionary", and all of
142
+ # them are checked by Suggest.
143
+ #
144
+ # @param word [String] The word to analyze
145
+ # @return [Array<Symbol, Array<String>>] Pair of [captype, variants]
146
+ def corrections(word)
147
+ captype = guess(word)
148
+
149
+ result = case captype
150
+ when Type::NO
151
+ [word]
152
+ when Type::INIT
153
+ [word, *lower(word)]
154
+ when Type::HUHINIT
155
+ [word, *lowerfirst(word).to_a, *lower(word), *capitalize(word).to_a]
156
+ when Type::HUH
157
+ [word, *lower(word)]
158
+ when Type::ALL
159
+ [word, *lower(word), *capitalize(word).to_a]
160
+ end
161
+
162
+ [captype, result]
163
+ end
164
+
165
+ # Used by suggest: by known (valid) suggestion, and initial word's
166
+ # capitalization, produce proper suggestion capitalization.
167
+ #
168
+ # Example: If misspelling was "Kiten" (INIT capitalization),
169
+ # found suggestion "kitten", then this method makes it "Kitten".
170
+ #
171
+ # @param word [String] The valid suggestion word
172
+ # @param cap [Symbol] Original word's capitalization type
173
+ # @return [String] Properly capitalized suggestion
174
+ def coerce(word, cap)
175
+ case cap
176
+ when Type::INIT, Type::HUHINIT
177
+ upper(word[0]) + word[1..]
178
+ when Type::ALL
179
+ upper(word)
180
+ else
181
+ word
182
+ end
183
+ end
184
+ end
185
+
186
+ # Redefines upper and lower, because in Turkic languages lowercase "i"
187
+ # is uppercased as "İ", and uppercase "I" is downcased as "ı".
188
+ #
189
+ # Example:
190
+ # turkic = Kotoshu::Algorithms::Capitalization::TurkicCasing.new
191
+ # turkic.lower('Izmir') # => ['ızmir']
192
+ # turkic.upper('Izmir') # => 'IZMİR'
193
+ class TurkicCasing < Casing
194
+ U2L = {
195
+ 'İ' => 'i',
196
+ 'I' => 'ı'
197
+ }.freeze
198
+
199
+ L2U = {
200
+ 'i' => 'İ',
201
+ 'ı' => 'I'
202
+ }.freeze
203
+
204
+ # Translate uppercase Turkic characters to lowercase.
205
+ #
206
+ # @param word [String] The word to lowercase
207
+ # @return [Array<String>] List of lowercased variants
208
+ def lower(word)
209
+ translated = word.chars.map { |c| U2L[c] || c }.join
210
+ super(translated)
211
+ end
212
+
213
+ # Translate lowercase Turkic characters to uppercase.
214
+ #
215
+ # @param word [String] The word to uppercase
216
+ # @return [String] Uppercased word
217
+ def upper(word)
218
+ translated = word.chars.map { |c| L2U[c] || c }.join
219
+ super(translated)
220
+ end
221
+ end
222
+
223
+ # Redefines lower because in German "SS" can be lowercased both as "ss" and "ß".
224
+ #
225
+ # Example:
226
+ # german = Kotoshu::Algorithms::Capitalization::GermanCasing.new
227
+ # german.lower('STRASSE') # => ['straße', 'strasse']
228
+ class GermanCasing < Casing
229
+ # Generate sharp S (ß) variants for all "ss" occurrences.
230
+ #
231
+ # @param text [String] The text to process
232
+ # @param start [Integer] Starting position for search
233
+ # @return [Array<String>] All variants with ß replacements
234
+ def sharp_s_variants(text, start = 0)
235
+ pos = text.index('ss', start)
236
+ return [] unless pos
237
+
238
+ replaced = text[0...pos] + 'ß' + text[(pos + 2)..]
239
+ [replaced,
240
+ *sharp_s_variants(replaced, pos + 1),
241
+ *sharp_s_variants(text, pos + 2)]
242
+ end
243
+
244
+ # Lowercase word, generating both "ss" and "ß" variants where applicable.
245
+ #
246
+ # @param word [String] The word to lowercase
247
+ # @return [Array<String>] List of lowercased variants
248
+ def lower(word)
249
+ lowered = super.first
250
+ return [lowered] unless word.include?('SS')
251
+
252
+ [*sharp_s_variants(lowered), lowered]
253
+ end
254
+
255
+ # Guess word's capitalization, accounting for German ß handling.
256
+ #
257
+ # In German uppercased words, ß (which is lowercase, and usually uppercased
258
+ # as SS) is allowed: "straße" => "STRAßE"
259
+ #
260
+ # @param word [String] The word to analyze
261
+ # @return [Symbol] One of the Type constants
262
+ def guess(word)
263
+ result = super
264
+
265
+ # Check if removing ß makes it ALL caps
266
+ if word.include?('ß')
267
+ word_without_ss = word.gsub('ß', '')
268
+ return Type::ALL if super(word_without_ss) == Type::ALL
269
+ end
270
+
271
+ result
272
+ end
273
+ end
274
+ end
275
+ end
276
+ end
@@ -0,0 +1,128 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Damerau-Levenshtein edit distance.
6
+ #
7
+ # Counts the minimum number of operations (insertion, deletion,
8
+ # substitution, or transposition of adjacent characters) needed
9
+ # to transform one string into another. The transposition
10
+ # extension distinguishes this from plain Levenshtein — a
11
+ # transposition (e.g. "teh" → "the") costs 1 instead of 2.
12
+ #
13
+ # Extracted from EditDistanceStrategy so that the algorithm is
14
+ # reusable independent of the strategy pipeline and testable
15
+ # without send-to-private.
16
+ module EditDistance
17
+ module_function
18
+
19
+ # Compute the Damerau-Levenshtein distance between two strings.
20
+ #
21
+ # @param str1 [String] First string
22
+ # @param str2 [String] Second string
23
+ # @return [Integer] Edit distance (0 when str1 == str2)
24
+ def distance(str1, str2)
25
+ return str2.length if str1.empty?
26
+ return str1.length if str2.empty?
27
+
28
+ len1 = str1.length
29
+ len2 = str2.length
30
+
31
+ d = Array.new(len1 + 1) { Array.new(len2 + 1, 0) }
32
+
33
+ (0..len1).each { |i| d[i][0] = i }
34
+ (0..len2).each { |j| d[0][j] = j }
35
+
36
+ (1..len1).each do |i|
37
+ (1..len2).each do |j|
38
+ cost = str1[i - 1] == str2[j - 1] ? 0 : 1
39
+
40
+ d[i][j] = [
41
+ d[i - 1][j] + 1, # deletion
42
+ d[i][j - 1] + 1, # insertion
43
+ d[i - 1][j - 1] + cost # substitution
44
+ ].min
45
+
46
+ next unless i > 1 && j > 1 &&
47
+ str1[i - 1] == str2[j - 2] &&
48
+ str1[i - 2] == str2[j - 1]
49
+
50
+ d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
51
+ end
52
+ end
53
+
54
+ d[len1][len2]
55
+ end
56
+
57
+ # Compute edit distance with early-exit threshold.
58
+ #
59
+ # Returns nil when the true distance exceeds +threshold+. Uses
60
+ # the row-minimum early-termination technique: after each DP
61
+ # row is computed, if every cell in the row exceeds threshold,
62
+ # the final distance must exceed threshold (the last-row cell
63
+ # is bounded below by the row minimum) — so we bail without
64
+ # computing the rest.
65
+ #
66
+ # Combined with the length pre-filter (|len1-len2| > threshold
67
+ # implies distance > threshold), this prunes clearly-different
68
+ # pairs in O(threshold * min(len1, len2)) instead of the full
69
+ # O(len1 * len2) Damerau-Levenshtein DP.
70
+ #
71
+ # Note: the full matrix is kept (not 2-row DP) because the
72
+ # Damerau transposition step needs d[i-2][j-2] which a 2-row
73
+ # implementation doesn't retain. The early termination is the
74
+ # primary win, not memory reduction.
75
+ #
76
+ # @param str1 [String] First string
77
+ # @param str2 [String] Second string
78
+ # @param threshold [Integer] Maximum distance to report (assumed >= 0)
79
+ # @return [Integer, nil] Distance if ≤ threshold, else nil
80
+ def distance_with_threshold(str1, str2, threshold)
81
+ return 0 if str1 == str2
82
+ # The length filter must run before the empty-string shortcuts:
83
+ # an empty str1 gives distance str2.length, which still has to
84
+ # honor the nil-above-threshold contract.
85
+ return nil if (str1.length - str2.length).abs > threshold
86
+ return str2.length if str1.empty?
87
+ return str1.length if str2.empty?
88
+
89
+ len1 = str1.length
90
+ len2 = str2.length
91
+
92
+ d = Array.new(len1 + 1) { Array.new(len2 + 1, 0) }
93
+ (0..len1).each { |i| d[i][0] = i }
94
+ (0..len2).each { |j| d[0][j] = j }
95
+
96
+ (1..len1).each do |i|
97
+ row_min = Float::INFINITY
98
+
99
+ (1..len2).each do |j|
100
+ cost = str1[i - 1] == str2[j - 1] ? 0 : 1
101
+
102
+ d[i][j] = [
103
+ d[i - 1][j] + 1, # deletion
104
+ d[i][j - 1] + 1, # insertion
105
+ d[i - 1][j - 1] + cost # substitution
106
+ ].min
107
+
108
+ # Damerau transposition (needs d[i-2][j-2] which is why
109
+ # we keep the full matrix).
110
+ if i > 1 && j > 1 &&
111
+ str1[i - 1] == str2[j - 2] &&
112
+ str1[i - 2] == str2[j - 1]
113
+ d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
114
+ end
115
+
116
+ row_min = d[i][j] if d[i][j] < row_min
117
+ end
118
+
119
+ # Row minimum > threshold ⇒ final answer > threshold.
120
+ return nil if row_min > threshold
121
+ end
122
+
123
+ result = d[len1][len2]
124
+ result <= threshold ? result : nil
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ module Lookup
6
+ # AffixForm is a hypothesis of how some word might be split into
7
+ # stem, suffixes and prefixes.
8
+ #
9
+ # It always has full text and stem, and may have up to two suffixes
10
+ # and up to two prefixes.
11
+ #
12
+ # The following is always true (considering absent affixes as empty):
13
+ # prefix + prefix2 + stem + suffix2 + suffix = text
14
+ #
15
+ # prefix2/suffix2 are "secondary", so if the word has only one suffix,
16
+ # it is stored in suffix and suffix2 is nil.
17
+ class AffixForm
18
+ # @return [String] Full word text
19
+ attr_reader :text
20
+
21
+ # @return [String] Stem (word without affixes)
22
+ attr_reader :stem
23
+
24
+ # @return [Hash, nil] Prefix affix data
25
+ attr_reader :prefix
26
+
27
+ # @return [Hash, nil] Suffix affix data
28
+ attr_reader :suffix
29
+
30
+ # @return [Hash, nil] Secondary prefix affix data
31
+ attr_reader :prefix2
32
+
33
+ # @return [Hash, nil] Secondary suffix affix data
34
+ attr_reader :suffix2
35
+
36
+ # @return [Hash, nil] Dictionary entry for stem
37
+ attr_reader :in_dictionary
38
+
39
+ def initialize(text, stem,
40
+ prefix: nil, suffix: nil,
41
+ prefix2: nil, suffix2: nil,
42
+ in_dictionary: nil)
43
+ @text = text
44
+ @stem = stem
45
+ @prefix = prefix
46
+ @suffix = suffix
47
+ @prefix2 = prefix2
48
+ @suffix2 = suffix2
49
+ @in_dictionary = in_dictionary
50
+ end
51
+
52
+ # Create a copy with changes.
53
+ #
54
+ # @param changes [Hash] Changes to apply
55
+ # @return [AffixForm] New affix form with changes applied
56
+ def replace(**changes)
57
+ self.class.new(
58
+ changes.fetch(:text, @text),
59
+ changes.fetch(:stem, @stem),
60
+ prefix: changes.fetch(:prefix, @prefix),
61
+ suffix: changes.fetch(:suffix, @suffix),
62
+ prefix2: changes.fetch(:prefix2, @prefix2),
63
+ suffix2: changes.fetch(:suffix2, @suffix2),
64
+ in_dictionary: changes.fetch(:in_dictionary, @in_dictionary)
65
+ )
66
+ end
67
+
68
+ # Check if this form has any affixes.
69
+ #
70
+ # @return [Boolean]
71
+ def has_affixes?
72
+ !@suffix.nil? || !@prefix.nil?
73
+ end
74
+
75
+ # Check if this is a base form (no affixes).
76
+ #
77
+ # @return [Boolean]
78
+ def is_base?
79
+ !has_affixes?
80
+ end
81
+
82
+ # Suffixes applied to this form, outermost first.
83
+ #
84
+ # @return [Array<Hash>] Suffix affix data
85
+ def suffixes
86
+ [@suffix, @suffix2].compact
87
+ end
88
+
89
+ # Flags the form carries: the dictionary entry's, plus those of the
90
+ # outermost prefix and suffix.
91
+ #
92
+ # Secondary affixes are deliberately excluded, matching Spylls. A
93
+ # secondary affix sits between the stem and the affix that was
94
+ # stripped last; its flags gate that stripping (see #desuffix's
95
+ # required_flags) rather than describing the finished word.
96
+ #
97
+ # @return [Set<String>] Combined flags
98
+ def flags
99
+ flags = @in_dictionary ? Set.new(@in_dictionary[:flags] || []) : Set.new
100
+ flags.merge(@prefix[:flags] || []) if @prefix
101
+ flags.merge(@suffix[:flags] || []) if @suffix
102
+ flags
103
+ end
104
+
105
+ # Get all affixes (excluding nils).
106
+ #
107
+ # @return [Array<Hash>] List of affix data
108
+ def all_affixes
109
+ [@prefix2, @prefix, @suffix, @suffix2].compact
110
+ end
111
+
112
+ # String representation.
113
+ #
114
+ # @return [String]
115
+ def to_s
116
+ return @text if is_base?
117
+
118
+ parts = []
119
+ parts << @prefix.inspect if @prefix
120
+ parts << @prefix2.inspect if @prefix2
121
+ parts << @stem
122
+ parts << @suffix2.inspect if @suffix2
123
+ parts << @suffix.inspect if @suffix
124
+
125
+ "AffixForm(#{@text} = #{parts.join(' + ')})"
126
+ end
127
+
128
+ alias inspect to_s
129
+ end
130
+ end
131
+ end
132
+ end