kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,315 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # Base class for suggestion strategies.
7
+ #
8
+ # Subclasses must implement the {#generate} method.
9
+ #
10
+ # @example Implementing a custom strategy
11
+ # class MyStrategy < BaseStrategy
12
+ # def generate(context)
13
+ # # Return suggestions based on context.word
14
+ # SuggestionSet.from_words(%w[word1 word2], source: :my_strategy)
15
+ # end
16
+ # end
17
+ class BaseStrategy
18
+ # @return [Symbol] Strategy name
19
+ attr_reader :name
20
+
21
+ # @return [Hash] Strategy configuration
22
+ attr_reader :config
23
+
24
+ # Create a new base strategy.
25
+ #
26
+ # @param name [String, Symbol] Strategy name
27
+ # @param config [Hash] Configuration options
28
+ # @option config [Integer] max_results Maximum results to return
29
+ # @option config [Boolean] enabled Whether strategy is enabled
30
+ def initialize(name: :base, **config)
31
+ @name = name.to_sym
32
+ @config = config
33
+ @enabled = config.fetch(:enabled, true)
34
+ @max_results = config.fetch(:max_results, 10)
35
+ end
36
+
37
+ # Generate suggestions for a word.
38
+ #
39
+ # @abstract Subclasses must implement this method.
40
+ # @param context [Context] The suggestion context
41
+ # @return [SuggestionSet] Generated suggestions
42
+ # @raise [NotImplementedError] Subclass must implement
43
+ def generate(context)
44
+ raise NotImplementedError, "#{self.class} must implement #generate"
45
+ end
46
+
47
+ # Check if this strategy is enabled.
48
+ #
49
+ # @return [Boolean] True if enabled
50
+ def enabled?
51
+ @enabled
52
+ end
53
+
54
+ # Get the max results configuration.
55
+ #
56
+ # @param default [Integer] Default value if not set
57
+ # @return [Integer] Max results
58
+ def max_results(default = 10)
59
+ @max_results || default
60
+ end
61
+
62
+ # Get a configuration value.
63
+ #
64
+ # @param key [Symbol] The config key
65
+ # @param default [Object] Default value if not set
66
+ # @return [Object] The config value
67
+ def get_config(key, default = nil)
68
+ @config.fetch(key, default)
69
+ end
70
+
71
+ # Check if a config value is present.
72
+ #
73
+ # @param key [Symbol] The config key
74
+ # @return [Boolean] True if config has the key
75
+ def has_config?(key)
76
+ @config.key?(key)
77
+ end
78
+
79
+ # Get the priority for this strategy.
80
+ #
81
+ # @return [Integer] Priority (lower = higher priority)
82
+ def priority
83
+ @config.fetch(:priority, 100)
84
+ end
85
+
86
+ # Check if this strategy should handle the context.
87
+ #
88
+ # Default implementation checks if the word is not in the dictionary.
89
+ # Subclasses can override for more specific logic.
90
+ #
91
+ # @param context [Context] The suggestion context
92
+ # @return [Boolean] True if the strategy should handle this context
93
+ def handles?(context)
94
+ return false unless enabled?
95
+
96
+ !dictionary_lookup(context, context.word)
97
+ end
98
+
99
+ # Whether the confidence cascade may skip this strategy when
100
+ # the traditional strategies are already confident about the
101
+ # word (see {Suggestions::SemanticCascade}). Only expensive,
102
+ # optional rerankers (e.g. {SemanticStrategy}) opt in.
103
+ #
104
+ # @return [Boolean]
105
+ def skip_when_confident?
106
+ false
107
+ end
108
+
109
+ # Create a suggestion from a word.
110
+ #
111
+ # @param word [String] The suggested word
112
+ # @param distance [Integer] Edit distance
113
+ # @param confidence [Float] Confidence score
114
+ # @param metadata [Hash] Additional metadata for ranking
115
+ # @return [Suggestion] New suggestion
116
+ def create_suggestion(word, distance: 0, confidence: 1.0, **metadata)
117
+ Suggestion.new(
118
+ word: word,
119
+ distance: distance,
120
+ confidence: confidence,
121
+ source: @name,
122
+ **metadata
123
+ )
124
+ end
125
+
126
+ # Create a suggestion set from words.
127
+ #
128
+ # @param words [Array<String>] Array of words
129
+ # @param distances [Hash] Optional word => distance mapping
130
+ # @param original_word [String] The original misspelled word (for ranking)
131
+ # @return [SuggestionSet] New suggestion set
132
+ def create_suggestion_set(words, distances: {}, original_word: nil)
133
+ suggestions = words.map do |word|
134
+ # Try case-sensitive first, then case-insensitive for distance lookup
135
+ distance = if distances.key?(word)
136
+ distances[word]
137
+ else
138
+ distances.fetch(word.downcase, 1)
139
+ end
140
+ confidence = calculate_confidence(distance)
141
+
142
+ # Calculate n-gram similarity (like Hunspell) for better ranking
143
+ ngram_score = if original_word
144
+ calculate_ngram_similarity(original_word, word)
145
+ else
146
+ 0
147
+ end
148
+
149
+ metadata = {
150
+ original_length: original_word&.length || word.length,
151
+ ngram_score: ngram_score
152
+ }
153
+
154
+ create_suggestion(word, distance: distance, confidence: confidence, **metadata)
155
+ end
156
+ SuggestionSet.new(suggestions, max_size: max_results)
157
+ end
158
+
159
+ # Calculate typo correction similarity between two words.
160
+ #
161
+ # This is a custom similarity metric designed specifically for spelling
162
+ # correction, combining:
163
+ # - Character overlap (how many characters are shared)
164
+ # - Prefix weight (common prefix is very important for typos)
165
+ # - Suffix weight (common ending is also important)
166
+ # - Length penalty (very different lengths are less similar)
167
+ #
168
+ # Returns a value from 0.0 (no similarity) to 1.0 (identical).
169
+ #
170
+ # @param word1 [String] First word
171
+ # @param word2 [String] Second word
172
+ # @return [Float] Typo correction similarity (0.0 to 1.0)
173
+ def calculate_ngram_similarity(word1, word2)
174
+ return 0 if word1.nil? || word2.nil? || word1.empty? || word2.empty?
175
+
176
+ w1 = word1.downcase
177
+ w2 = word2.downcase
178
+
179
+ # Identical strings have maximum similarity
180
+ return 1.0 if w1 == w2
181
+
182
+ len1 = w1.length
183
+ len2 = w2.length
184
+ max_len = [len1, len2].max
185
+
186
+ # Calculate common prefix length (up to 4 characters)
187
+ prefix_len = 0
188
+ (0...[len1, len2, 4].min).each do |i|
189
+ break if w1[i] != w2[i]
190
+
191
+ prefix_len += 1
192
+ end
193
+
194
+ # Calculate common suffix length
195
+ suffix_len = 0
196
+ (1..[len1, len2, 4].min).each do |i|
197
+ break if w1[-i] != w2[-i]
198
+
199
+ suffix_len += 1
200
+ end
201
+
202
+ # Calculate character overlap (how many characters from w1 are in w2)
203
+ w2_chars = w2.chars
204
+ overlap = w1.chars.count { |c| w2_chars.include?(c) }
205
+
206
+ # Calculate similarity score
207
+ # 1. Base score from character overlap
208
+ similarity = overlap.to_f / max_len
209
+
210
+ # 2. Prefix bonus (common start is very important for typos)
211
+ prefix_bonus = prefix_len * 0.15
212
+
213
+ # 3. Suffix bonus (common ending is also important)
214
+ suffix_bonus = suffix_len * 0.05
215
+
216
+ # 4. Length penalty (very different lengths are less similar)
217
+ length_diff = (len1 - len2).abs
218
+ length_penalty = length_diff * 0.1
219
+
220
+ # Combine all factors
221
+ similarity = similarity + prefix_bonus + suffix_bonus - length_penalty
222
+
223
+ # Cap at 1.0, floor at 0.0
224
+ [[similarity, 1.0].min, 0.0].max
225
+ end
226
+
227
+ # Generate n-grams for a word.
228
+ #
229
+ # @param word [String] The word
230
+ # @param n [Integer] N-gram size
231
+ # @return [Set<String>] Set of n-grams
232
+ def generate_ngrams(word, n)
233
+ ngrams = Set.new
234
+ (word.length - n + 1).times do |i|
235
+ ngrams.add(word[i, n])
236
+ end
237
+ ngrams
238
+ end
239
+
240
+ # Convert strategy to string.
241
+ #
242
+ # @return [String] String representation
243
+ def to_s
244
+ "#{self.class.name}(name: #{@name}, enabled: #{enabled?})"
245
+ end
246
+ alias inspect to_s
247
+
248
+ private
249
+
250
+ # Look up a word in the dictionary.
251
+ #
252
+ # Type-driven dispatch (no `respond_to?`): IndexedDictionary is
253
+ # checked first because it is the most common case in the
254
+ # hot path; Hash and Array are ad-hoc dictionaries for tests
255
+ # and one-off scripts; anything else must implement the
256
+ # documented `lookup(word)` interface. Passing an object that
257
+ # doesn't quack like a dictionary surfaces as a NoMethodError.
258
+ #
259
+ # @param context [Context] The suggestion context
260
+ # @param word [String] The word to look up
261
+ # @return [Boolean] True if word exists
262
+ def dictionary_lookup(context, word)
263
+ dictionary = context.dictionary
264
+
265
+ case dictionary
266
+ when Kotoshu::Core::IndexedDictionary
267
+ dictionary.has_word?(word)
268
+ when Hash
269
+ dictionary.key?(word)
270
+ when Array
271
+ dictionary.include?(word)
272
+ else
273
+ dictionary.lookup(word)
274
+ end
275
+ end
276
+
277
+ # Get all words from the dictionary.
278
+ #
279
+ # Type-driven dispatch (no `respond_to?`). Mirrors
280
+ # {#dictionary_lookup}'s ordering.
281
+ #
282
+ # @param context [Context] The suggestion context
283
+ # @return [Array<String>] All words
284
+ def dictionary_words(context)
285
+ dictionary = context.dictionary
286
+
287
+ case dictionary
288
+ when Kotoshu::Core::IndexedDictionary
289
+ dictionary.words
290
+ when Hash
291
+ dictionary.keys
292
+ when Array
293
+ dictionary
294
+ else
295
+ dictionary.words
296
+ end
297
+ end
298
+
299
+ # Calculate confidence from distance.
300
+ #
301
+ # Higher distance = lower confidence.
302
+ #
303
+ # @param distance [Integer] Edit distance
304
+ # @return [Float] Confidence score (0.0 to 1.0)
305
+ def calculate_confidence(distance)
306
+ return 1.0 if distance.zero?
307
+
308
+ # Simple decay: confidence = 1 / (1 + distance)
309
+ # Can be overridden by subclasses for more sophisticated calculations
310
+ 1.0 / (1.0 + distance)
311
+ end
312
+ end
313
+ end
314
+ end
315
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # Composite strategy that chains multiple suggestion strategies.
7
+ # Implements the Composite Pattern for extensible suggestion generation.
8
+ #
9
+ # This is MORE OOP than Spylls which has a procedural suggestion pipeline.
10
+ # Here, strategies are proper objects that can be added/removed/reordered.
11
+ #
12
+ # @example Using composite strategy
13
+ # pipeline = CompositeStrategy.new(name: :pipeline)
14
+ # pipeline.add(EditDistanceStrategy.new)
15
+ # pipeline.add(PhoneticStrategy.new)
16
+ # pipeline.add(NgramStrategy.new)
17
+ # suggestions = pipeline.generate(context)
18
+ class CompositeStrategy < BaseStrategy
19
+ attr_reader :strategies
20
+
21
+ # @param name [String, Symbol] Name of the composite
22
+ # @param strategies [Array<BaseStrategy>] Initial strategies
23
+ # @param config [Hash] Configuration options
24
+ def initialize(name:, strategies: [], **config)
25
+ @strategies = strategies
26
+ super(name: name, **config)
27
+ end
28
+
29
+ # Add a strategy to the pipeline.
30
+ #
31
+ # @param strategy [BaseStrategy] The strategy to add
32
+ # @return [CompositeStrategy] Self for chaining
33
+ def add(strategy)
34
+ @strategies << strategy
35
+ self
36
+ end
37
+ alias << add
38
+
39
+ # Remove a strategy from the pipeline.
40
+ #
41
+ # @param strategy [BaseStrategy] The strategy to remove
42
+ # @return [CompositeStrategy] Self for chaining
43
+ def remove(strategy)
44
+ @strategies.delete(strategy)
45
+ self
46
+ end
47
+
48
+ # Clear all strategies.
49
+ #
50
+ # @return [CompositeStrategy] Self for chaining
51
+ def clear
52
+ @strategies.clear
53
+ self
54
+ end
55
+
56
+ # Get strategies that can handle the given context.
57
+ #
58
+ # @param context [Context] The suggestion context
59
+ # @return [Array<BaseStrategy>] Applicable strategies
60
+ def applicable_strategies(context)
61
+ @strategies.select { |s| s.handles?(context) }
62
+ end
63
+
64
+ # Generate suggestions by delegating to all applicable strategies.
65
+ #
66
+ # Candidates are collected from every strategy first, then handed
67
+ # to a single SuggestionSet so dedup and ranking run once over the
68
+ # full pool (not as a side effect of each merge). TODO 56 T5.1.
69
+ #
70
+ # Strategies that declare themselves skippable by the confidence
71
+ # cascade ({BaseStrategy#skip_when_confident?} — the semantic
72
+ # rerank) run after the traditional ones and are skipped
73
+ # entirely when the traditional pool already meets the
74
+ # configured threshold ({SemanticCascade}). At the default
75
+ # threshold (1.0, always rerank) the cascade is disarmed and
76
+ # every strategy runs exactly as before.
77
+ #
78
+ # @param context [Context] The suggestion context
79
+ # @return [SuggestionSet] Combined suggestions from all strategies
80
+ def generate(context)
81
+ cascade = semantic_cascade
82
+ strategies = applicable_strategies(context)
83
+ deferred = strategies.select(&:skip_when_confident?)
84
+
85
+ if deferred.empty? || !cascade.armed?
86
+ return generate_all(strategies, context)
87
+ end
88
+
89
+ traditional = strategies.reject(&:skip_when_confident?)
90
+ pool = SuggestionSet.new(
91
+ traditional.flat_map { |strategy| strategy.generate(context).suggestions },
92
+ max_size: context.max_results
93
+ )
94
+
95
+ if cascade.skip?(pool.suggestions)
96
+ record_cascade_skip(context, pool.first, cascade)
97
+ return pool
98
+ end
99
+
100
+ pool.concat(deferred.flat_map { |strategy| strategy.generate(context).suggestions })
101
+ end
102
+
103
+ # Check if any strategy can handle the context.
104
+ #
105
+ # @param context [Context] The suggestion context
106
+ # @return [Boolean] True if any strategy handles the context
107
+ def handles?(context)
108
+ applicable_strategies(context).any?
109
+ end
110
+
111
+ # Get the number of strategies.
112
+ #
113
+ # @return [Integer] Number of strategies
114
+ def size
115
+ @strategies.size
116
+ end
117
+ alias count size
118
+
119
+ # Check if the composite has any strategies.
120
+ #
121
+ # @return [Boolean] True if there are strategies
122
+ def any?
123
+ @strategies.any?
124
+ end
125
+
126
+ # Iterate over strategies.
127
+ #
128
+ # @yield [strategy] Each strategy
129
+ # @return [Enumerator] Enumerator if no block given
130
+ def each_strategy(&)
131
+ return enum_for(:each_strategy) unless block_given?
132
+
133
+ @strategies.each(&)
134
+ end
135
+
136
+ # Sort strategies by priority.
137
+ #
138
+ # @return [CompositeStrategy] Self for chaining
139
+ def sort_by_priority!
140
+ @strategies.sort_by!(&:priority)
141
+ self
142
+ end
143
+
144
+ # Convert to string.
145
+ #
146
+ # @return [String] String representation
147
+ def to_s
148
+ "#{self.class.name}(name: #{@name}, strategies: #{@strategies.map(&:name).join(', ')})"
149
+ end
150
+ alias inspect to_s
151
+
152
+ # Create a composite strategy with default algorithms.
153
+ #
154
+ # @param config [Hash] Configuration
155
+ # @return [CompositeStrategy] New composite with default strategies
156
+ def self.with_defaults(**config)
157
+ new(name: :default, **config)
158
+ end
159
+
160
+ private
161
+
162
+ # Run every strategy and merge into a single set — the
163
+ # pre-cascade shape, byte-identical to the original pipeline.
164
+ def generate_all(strategies, context)
165
+ candidates = strategies.flat_map do |strategy|
166
+ strategy.generate(context).suggestions
167
+ end
168
+ SuggestionSet.new(candidates, max_size: context.max_results)
169
+ end
170
+
171
+ # The cascade decision object. An explicit
172
+ # +semantic_cascade_threshold+ in this composite's config wins;
173
+ # otherwise the process Configuration applies (SCHEMA default
174
+ # 1.0, KOTOSHU_SEMANTIC_CASCADE_THRESHOLD ENV).
175
+ def semantic_cascade
176
+ threshold = get_config(:semantic_cascade_threshold)
177
+ if threshold.nil?
178
+ SemanticCascade.from_configuration(Kotoshu.configuration)
179
+ else
180
+ SemanticCascade.new(threshold: threshold)
181
+ end
182
+ end
183
+
184
+ # Count and (under debug/verbose) log a cascade skip.
185
+ def record_cascade_skip(context, top, cascade)
186
+ Kotoshu::Metrics.record_semantic_cascade_skip
187
+ return unless Kotoshu::Debug.enabled?
188
+
189
+ Kotoshu::Debug.logger&.verbose(
190
+ "semantic cascade: skipped rerank for \"#{context.word}\" " \
191
+ "(top confidence #{top.confidence.round(3)} >= #{cascade.threshold})"
192
+ )
193
+ end
194
+ end
195
+ end
196
+ end
197
+ end