kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # String similarity metrics for spell checking.
5
+ #
6
+ # Ported from Spylls (Python) string_metrics.py
7
+ #
8
+ # These metrics are used for:
9
+ # - Computing word similarity
10
+ # - Ranking suggestions
11
+ # - N-gram based scoring
12
+ module StringMetrics
13
+ # Number of occurrences of the exactly same characters in exactly same position.
14
+ #
15
+ # @param s1 [String] First string
16
+ # @param s2 [String] Second string
17
+ # @return [Integer] Count of matching characters at same positions
18
+ #
19
+ # @example
20
+ # Kotoshu::StringMetrics.commoncharacters("hello", "hallo") # => 4 ('h', 'l', 'l', 'o' match)
21
+ def self.commoncharacters(s1, s2)
22
+ return 0 if s1.nil? || s2.nil?
23
+
24
+ # Zip strings and count matching character pairs
25
+ [s1.length, s2.length].min.times.count do |i|
26
+ s1[i] == s2[i]
27
+ end
28
+ end
29
+
30
+ # Size of the common start of two strings.
31
+ #
32
+ # @param s1 [String] First string
33
+ # @param s2 [String] Second string
34
+ # @return [Integer] Length of common prefix
35
+ #
36
+ # @example
37
+ # Kotoshu::StringMetrics.leftcommonsubstring("foo", "bar") # => 0
38
+ # Kotoshu::StringMetrics.leftcommonsubstring("built", "build") # => 4
39
+ # Kotoshu::StringMetrics.leftcommonsubstring("cat", "cats") # => 3
40
+ def self.leftcommonsubstring(s1, s2)
41
+ return 0 if s1.nil? || s2.nil?
42
+
43
+ # Find first position where characters differ
44
+ s1.chars.zip(s2.chars).each_with_index do |(c1, c2), i|
45
+ return i if c1 != c2
46
+ end
47
+
48
+ # All characters matched up to shorter string length
49
+ [s1.length, s2.length].min
50
+ end
51
+
52
+ # Calculate n-gram similarity between two strings.
53
+ #
54
+ # Calculates how many n-grams of s1 are contained in s2 (the more the number,
55
+ # the more words are similar).
56
+ #
57
+ # @param max_ngram_size [Integer] Maximum n-gram size to check
58
+ # @param s1 [String] String to compare
59
+ # @param s2 [String] String to compare
60
+ # @param weighted [Boolean] Subtract from result for ngrams NOT contained
61
+ # @param any_mismatch [Boolean] Add penalty for any string length difference
62
+ # @param longer_worse [Boolean] Add penalty when second string is longer
63
+ # @return [Integer] N-gram similarity score (higher is more similar)
64
+ #
65
+ # @example
66
+ # Kotoshu::StringMetrics.ngram(4, "hello", "help") # => 6
67
+ # Kotoshu::StringMetrics.ngram(4, "teachings", "teaching") # => higher score
68
+ def self.ngram(max_ngram_size, s1, s2, weighted: false, any_mismatch: false, longer_worse: false)
69
+ l2 = s2.length
70
+ return 0 if l2.zero?
71
+
72
+ l1 = s1.length
73
+ nscore = 0
74
+
75
+ # For all sizes of ngram up to desired...
76
+ (1..max_ngram_size).each do |ngram_size|
77
+ ns = 0
78
+
79
+ # Check every position in the first string
80
+ (0..(l1 - ngram_size)).each do |pos|
81
+ ngram = s1[pos, ngram_size]
82
+
83
+ # If the ngram is present in ANY place in second string, increase score
84
+ if s2.include?(ngram)
85
+ ns += 1
86
+ elsif weighted
87
+ # For "weighted" ngrams, decrease score if ngram is not found
88
+ ns -= 1
89
+ # Decrease once more if it was the beginning or end of first string
90
+ ns -= 1 if pos.zero? || pos + ngram_size == l1
91
+ end
92
+ end
93
+
94
+ nscore += ns
95
+
96
+ # There is no need to check for 4-gram if there were only one 3-gram
97
+ break if ns < 2 && !weighted
98
+ end
99
+
100
+ # Calculate penalty based on settings
101
+ penalty = if longer_worse
102
+ # Add penalty when second string is longer
103
+ (l2 - l1) - 2
104
+ elsif any_mismatch
105
+ # Add penalty for any string length difference
106
+ (l2 - l1).abs - 2
107
+ else
108
+ 0
109
+ end
110
+
111
+ # Apply penalty if positive
112
+ penalty > 0 ? nscore - penalty : nscore
113
+ end
114
+
115
+ # Calculate LCS (Longest Common Subsequence) length.
116
+ #
117
+ # Classic dynamic programming algorithm. This is different from
118
+ # longest common substring - subsequence doesn't require contiguity.
119
+ #
120
+ # @param s1 [String] First string
121
+ # @param s2 [String] Second string
122
+ # @return [Integer] Length of longest common subsequence
123
+ #
124
+ # @example
125
+ # Kotoshu::StringMetrics.lcslen("AGGTAB", "GXTXAYB") # => 4 ("GTAB")
126
+ def self.lcslen(s1, s2)
127
+ return 0 if s1.nil? || s2.nil? || s1.empty? || s2.empty?
128
+
129
+ m = s1.length
130
+ n = s2.length
131
+
132
+ # Create DP table
133
+ # Using a 2D array for clarity, though we could optimize space
134
+ c = Array.new(m + 1) { Array.new(n + 1, 0) }
135
+
136
+ (0...m).each do |i|
137
+ (0...n).each do |j|
138
+ c[i + 1][j + 1] = if s1[i] == s2[j]
139
+ # Characters match - extend diagonal
140
+ c[i][j] + 1
141
+ elsif c[i][j + 1] >= c[i + 1][j]
142
+ # Take max from top or left
143
+ c[i][j + 1]
144
+ else
145
+ c[i + 1][j]
146
+ end
147
+ end
148
+ end
149
+
150
+ c[m][n]
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Context object passed to suggestion strategies.
6
+ # Encapsulates the state and parameters for suggestion generation.
7
+ class Context
8
+ attr_reader :word, :dictionary, :max_results, :options
9
+
10
+ def initialize(word:, dictionary:, max_results: 10, **options)
11
+ @word = word
12
+ @dictionary = dictionary
13
+ @max_results = max_results
14
+ @options = options
15
+ end
16
+
17
+ # Get an option value.
18
+ #
19
+ # @param key [Symbol] The option key
20
+ # @param default [Object] Default value if not found
21
+ # @return [Object] The option value
22
+ def option(key, default = nil)
23
+ @options.fetch(key, default)
24
+ end
25
+
26
+ # Check if an option is present.
27
+ #
28
+ # @param key [Symbol] The option key
29
+ # @return [Boolean] True if option exists
30
+ def has_option?(key)
31
+ @options.key?(key)
32
+ end
33
+
34
+ # Inspect the context.
35
+ #
36
+ # @return [String] Inspection string
37
+ def inspect
38
+ "Context(word: '#{@word}', max_results: #{@max_results})"
39
+ end
40
+ alias to_s inspect
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Loads and caches Kelly Project word-frequency tiers per language.
6
+ #
7
+ # Extracted from EditDistanceStrategy (TODO 56 T5.1 step 3, Phase A)
8
+ # so that strategy construction no longer performs disk IO and
9
+ # network access. The strategy holds a reference to a provider
10
+ # instance; the provider encapsulates the tiered fallback
11
+ # (FrequencyCache → frozen embedded tiers → local YAML → empty).
12
+ # The frozen tier layer (plan 119) carries the same en tables the
13
+ # Rust engine embeds, so a cache-cold machine ranks exactly like
14
+ # the frozen conformance vectors instead of the differently
15
+ # curated YAML.
16
+ #
17
+ # The provider memoizes per-language tiers, so repeated lookups
18
+ # for the same language are free after the first call.
19
+ class FrequencyProvider
20
+ EMPTY_TIERS = {
21
+ top_50: Set.new,
22
+ top_200: Set.new,
23
+ top_1000: Set.new
24
+ }.freeze
25
+
26
+ # @param frequency_cache [Cache::FrequencyCache, nil] Injectable
27
+ # cache instance (used by tests); defaults to a fresh
28
+ # FrequencyCache per load.
29
+ def initialize(frequency_cache: nil)
30
+ @frequency_cache = frequency_cache
31
+ @tiers_by_language = {}
32
+ end
33
+
34
+ # Return the frequency tiers for +language_code+.
35
+ #
36
+ # @param language_code [String] ISO 639-1 language code
37
+ # @return [Hash{Symbol => Set}] Hash with :top_50, :top_200, :top_1000
38
+ def tiers_for(language_code)
39
+ @tiers_by_language[language_code] ||= load(language_code)
40
+ end
41
+
42
+ private
43
+
44
+ def load(language_code)
45
+ cache_result = try_load_from_frequency_cache(language_code)
46
+ return cache_result[:tiers] if cache_result && cache_result[:tiers] && cache_result[:tiers][:top_1000].any?
47
+
48
+ frozen = FrozenTiers.tiers_for(language_code)
49
+ return frozen if frozen
50
+
51
+ yaml_data = Data::CommonWordsLoader.load(language_code)
52
+ return yaml_data[:tiers] if yaml_data[:tiers][:top_1000].any?
53
+
54
+ EMPTY_TIERS
55
+ end
56
+
57
+ def try_load_from_frequency_cache(language_code)
58
+ cache = @frequency_cache || Cache::FrequencyCache.new
59
+ # Read cache-only: the suggestion hot path must never trigger a
60
+ # download (downloads happen only through explicit setup /
61
+ # `kotoshu cache download`). TTL expiry is a refresh signal,
62
+ # NOT a reason to switch datasets: expired-but-present bytes
63
+ # are still the checksummed Kelly tiers the conformance
64
+ # vectors freeze, while the YAML fallback below is a different,
65
+ # more coarsely curated set whose bonus differences re-scale
66
+ # every suggestion confidence (plan 117). So: any readable
67
+ # cached data wins; the YAML only serves machines with no
68
+ # frequency data at all.
69
+ return nil unless cache.cached_data?(language_code)
70
+
71
+ begin
72
+ cache.load_cached(language_code)
73
+ rescue StandardError => e
74
+ warn "Warning: Failed to load frequency cache for #{language_code}: #{e.message}" if $VERBOSE
75
+ nil
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,169 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Generator for spelling suggestions.
6
+ #
7
+ # This class orchestrates multiple suggestion algorithms to generate
8
+ # comprehensive spelling suggestions.
9
+ #
10
+ # @example Using default algorithms
11
+ # generator = Generator.new(dictionary)
12
+ # suggestions = generator.generate("helo")
13
+ #
14
+ # @example Using custom algorithms
15
+ # custom_strategy = MyStrategy.new
16
+ # generator = Generator.new(dictionary, algorithms: [custom_strategy])
17
+ class Generator
18
+ # Default suggestion algorithms.
19
+ DEFAULT_ALGORITHMS = [
20
+ Strategies::EditDistanceStrategy,
21
+ Strategies::PhoneticStrategy,
22
+ Strategies::KeyboardProximityStrategy,
23
+ Strategies::NgramStrategy
24
+ ].freeze
25
+
26
+ # @return [Object] The dictionary (any dictionary backend)
27
+ attr_reader :dictionary
28
+
29
+ # @return [Strategies::CompositeStrategy] The composite strategy
30
+ attr_reader :strategy
31
+
32
+ # Create a new suggestion generator.
33
+ #
34
+ # @param dictionary [Object] The dictionary instance
35
+ # @param algorithms [Array<Class, Strategies::BaseStrategy>, nil] Algorithm classes or instances
36
+ # @param max_suggestions [Integer] Maximum suggestions to return
37
+ # @param config [Hash] Configuration options
38
+ def initialize(dictionary, algorithms: nil, max_suggestions: 10, **config)
39
+ @dictionary = dictionary
40
+ @max_suggestions = max_suggestions
41
+ # Use default algorithms if none provided
42
+ algorithms_to_use = algorithms || DEFAULT_ALGORITHMS
43
+ @strategy = build_strategy(algorithms_to_use, config)
44
+ end
45
+
46
+ # Generate suggestions for a word.
47
+ #
48
+ # @param word [String] The misspelled word
49
+ # @param max_suggestions [Integer] Maximum suggestions (optional)
50
+ # @return [SuggestionSet] Generated suggestions
51
+ #
52
+ # @example
53
+ # generator.generate("helo")
54
+ # # => #<Kotoshu::Suggestions::SuggestionSet ...>
55
+ def generate(word, max_suggestions: nil)
56
+ return SuggestionSet.empty if word.nil? || word.empty?
57
+
58
+ context = Context.new(
59
+ word: word,
60
+ dictionary: @dictionary,
61
+ max_results: max_suggestions || @max_suggestions
62
+ )
63
+
64
+ @strategy.generate(context)
65
+ end
66
+
67
+ # Alias for {#generate}, for API consistency (same parameters and
68
+ # return value).
69
+ #
70
+ # @example
71
+ # generator.suggest("helo")
72
+ # # => #<Kotoshu::Suggestions::SuggestionSet ...>
73
+ #
74
+ # @see #generate
75
+ alias suggest generate
76
+
77
+ # Check if a word is correct.
78
+ #
79
+ # @param word [String] The word to check
80
+ # @return [Boolean] True if the word is in the dictionary
81
+ #
82
+ # @example
83
+ # generator.correct?("hello") # => true
84
+ # generator.correct?("helo") # => false
85
+ def correct?(word)
86
+ return false if word.nil? || word.empty?
87
+
88
+ dictionary_lookup(word)
89
+ end
90
+
91
+ # Check if a word is incorrect.
92
+ #
93
+ # @param word [String] The word to check
94
+ # @return [Boolean] True if the word is not in the dictionary
95
+ def incorrect?(word)
96
+ !correct?(word)
97
+ end
98
+ alias misspelled? incorrect?
99
+
100
+ # Get the default algorithms.
101
+ #
102
+ # @return [Array<Class>] Default algorithm classes
103
+ #
104
+ # @example
105
+ # Generator.default_algorithms
106
+ def self.default_algorithms
107
+ DEFAULT_ALGORITHMS.dup
108
+ end
109
+
110
+ # Set the default algorithms.
111
+ #
112
+ # @param algorithms [Array<Class>] Algorithm classes
113
+ #
114
+ # @example
115
+ # Generator.default_algorithms = [MyCustomStrategy]
116
+ class << self
117
+ attr_writer :default_algorithms
118
+ end
119
+
120
+ private
121
+
122
+ # Build the composite strategy from algorithm classes.
123
+ #
124
+ # @param algorithms [Array<Class, Strategies::BaseStrategy>] Algorithm classes or instances
125
+ # @param config [Hash] Configuration options
126
+ # @return [Strategies::CompositeStrategy] The composite strategy
127
+ def build_strategy(algorithms, config)
128
+ composite = Strategies::CompositeStrategy.new(name: :default, **config)
129
+
130
+ algorithms.each do |alg|
131
+ strategy = if alg.is_a?(Strategies::BaseStrategy)
132
+ alg
133
+ elsif alg.is_a?(Class) && alg < Strategies::BaseStrategy
134
+ alg.new(**config)
135
+ else
136
+ raise ArgumentError, "Invalid algorithm: #{alg.inspect}"
137
+ end
138
+
139
+ composite.add(strategy)
140
+ end
141
+
142
+ composite
143
+ end
144
+
145
+ # Look up a word in the dictionary.
146
+ #
147
+ # Dispatch is type-driven (no `respond_to?`): Hash and Array are
148
+ # supported as ad-hoc dictionaries for tests and one-off scripts;
149
+ # everything else is treated as a real dictionary backend that
150
+ # implements the documented `lookup(word)` interface. Passing an
151
+ # object that doesn't quack like a dictionary surfaces as a
152
+ # NoMethodError on `lookup` — that's intentional; the caller
153
+ # misconfigured the generator.
154
+ #
155
+ # @param word [String] The word
156
+ # @return [Boolean] True if found
157
+ def dictionary_lookup(word)
158
+ case @dictionary
159
+ when Hash
160
+ @dictionary.key?(word)
161
+ when Array
162
+ @dictionary.include?(word)
163
+ else
164
+ @dictionary.lookup(word)
165
+ end
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Pipeline for composable suggestion strategies.
6
+ #
7
+ # Allows chaining multiple suggestion strategies that execute in sequence,
8
+ # with optional early termination when a stage produces no results.
9
+ #
10
+ # @example Creating a pipeline
11
+ # pipeline = Pipeline.new do |p|
12
+ # p.add :sym_spell
13
+ # p.add :phonetic
14
+ # p.add :ngram
15
+ # end
16
+ #
17
+ # @example Executing a pipeline
18
+ # result = pipeline.execute(context, strategies)
19
+ class Pipeline
20
+ # @return [Array<Symbol>] Ordered stage names
21
+ attr_reader :stages
22
+
23
+ # Create a new pipeline.
24
+ #
25
+ # @yield [pipeline] Optional block to add stages
26
+ # @return [Pipeline] New pipeline
27
+ #
28
+ # @example With block
29
+ # pipeline = Pipeline.new do |p|
30
+ # p.add :sym_spell
31
+ # p.add :phonetic
32
+ # end
33
+ def initialize
34
+ @stages = []
35
+ yield self if block_given?
36
+ end
37
+
38
+ # Add a stage to the pipeline.
39
+ #
40
+ # @param stage_name [Symbol] Name of the stage
41
+ # @return [Pipeline] Self for chaining
42
+ #
43
+ # @example
44
+ # pipeline.add(:sym_spell)
45
+ def add(stage_name)
46
+ @stages << stage_name
47
+ self
48
+ end
49
+
50
+ # Remove a stage from the pipeline.
51
+ #
52
+ # @param stage_name [Symbol] Name of the stage to remove
53
+ # @return [Pipeline] Self for chaining
54
+ #
55
+ # @example
56
+ # pipeline.remove(:phonetic)
57
+ def remove(stage_name)
58
+ @stages.delete(stage_name)
59
+ self
60
+ end
61
+
62
+ # Execute strategies through the pipeline.
63
+ #
64
+ # Strategies are executed in sequence. If a strategy returns
65
+ # an empty SuggestionSet, subsequent strategies are still executed
66
+ # unless early_termination is enabled.
67
+ #
68
+ # @param context [Context] The suggestion context
69
+ # @param strategies [Hash] Hash of stage_name => strategy_instance
70
+ # @param early_termination [Boolean] Whether to stop on empty result
71
+ # @return [SuggestionSet] Combined results from all stages
72
+ #
73
+ # @example
74
+ # strategies = { sym_spell: sym_spell_strategy, phonetic: phonetic_strategy }
75
+ # result = pipeline.execute(context, strategies)
76
+ def execute(context, strategies = nil, early_termination: false)
77
+ combined = SuggestionSet.empty
78
+
79
+ @stages.each do |stage_name|
80
+ strategy = if strategies.is_a?(Hash)
81
+ strategies[stage_name]
82
+ else
83
+ strategies
84
+ end
85
+
86
+ next unless strategy
87
+
88
+ result = strategy.generate(context)
89
+
90
+ # Combine results
91
+ combined = combine_results(combined, result)
92
+
93
+ # Early termination on empty result
94
+ break if early_termination && result.empty?
95
+ end
96
+
97
+ combined
98
+ end
99
+
100
+ # Check if pipeline has a stage.
101
+ #
102
+ # @param stage_name [Symbol] Stage name
103
+ # @return [Boolean] True if stage exists
104
+ def has_stage?(stage_name)
105
+ @stages.include?(stage_name)
106
+ end
107
+
108
+ # Clear all stages.
109
+ #
110
+ # @return [Pipeline] Self for chaining
111
+ def clear
112
+ @stages.clear
113
+ self
114
+ end
115
+
116
+ # Clone the pipeline.
117
+ #
118
+ # @return [Pipeline] New pipeline with same stages
119
+ def clone
120
+ self.class.new.tap { |p| @stages.each { |s| p.add(s) } }
121
+ end
122
+
123
+ private
124
+
125
+ # Combine two suggestion sets.
126
+ #
127
+ # @param combined [SuggestionSet] Current combined results
128
+ # @param new_result [SuggestionSet] New results to add
129
+ # @return [SuggestionSet] Combined suggestion set
130
+ def combine_results(combined, new_result)
131
+ combined.concat(new_result.suggestions).unique
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Confidence cascade for the semantic rerank (TODO.impl/70).
6
+ #
7
+ # Formalizes the hybrid dictionary-first architecture: the
8
+ # expensive, ONNX-backed semantic rerank only runs when the
9
+ # traditional composite strategies are not already confident
10
+ # about the word (dictionary verdict first, neural rerank only
11
+ # for uncertain candidates).
12
+ #
13
+ # @example From the process configuration
14
+ # cascade = SemanticCascade.from_configuration(Kotoshu.configuration)
15
+ # cascade.skip?(suggestion_set.suggestions) # => true / false
16
+ #
17
+ # @example With an explicit threshold
18
+ # SemanticCascade.new(threshold: 0.9)
19
+ class SemanticCascade
20
+ # Sentinel threshold meaning "always rerank" (the default).
21
+ # NOT a plain >= comparison: composite confidence legitimately
22
+ # reaches exactly 1.0 (distance-0 matches, the default
23
+ # confidence of a fresh Suggestion), so `>= 1.0` would change
24
+ # the default behavior — threshold 1.0 must mean never-skip.
25
+ ALWAYS_RERANK = 1.0
26
+
27
+ # @return [Float] The configured skip threshold
28
+ attr_reader :threshold
29
+
30
+ # Create a cascade decision object.
31
+ #
32
+ # @param threshold [Float, nil] Skip threshold in [0.0, 1.0];
33
+ # nil falls back to the always-rerank default
34
+ def initialize(threshold: ALWAYS_RERANK)
35
+ @threshold = threshold.nil? ? ALWAYS_RERANK : threshold.to_f
36
+ end
37
+
38
+ # Build from a Configuration, so the SCHEMA default and the
39
+ # KOTOSHU_SEMANTIC_CASCADE_THRESHOLD ENV var apply automatically.
40
+ #
41
+ # @param configuration [Configuration] The configuration to read
42
+ # @return [SemanticCascade]
43
+ def self.from_configuration(configuration)
44
+ new(threshold: configuration.semantic_cascade_threshold)
45
+ end
46
+
47
+ # Whether the cascade can ever skip. Only a threshold strictly
48
+ # below 1.0 arms it; 0.0 means never rerank.
49
+ #
50
+ # @return [Boolean]
51
+ def armed?
52
+ threshold < ALWAYS_RERANK
53
+ end
54
+
55
+ # Whether the semantic rerank should be skipped for this pool of
56
+ # traditional candidates. Skips only when the pool already has a
57
+ # top candidate at or above the threshold; an empty pool never
58
+ # skips (the dictionary path found nothing — the rerank may
59
+ # still help).
60
+ #
61
+ # @param suggestions [Array<Suggestion>] Sorted traditional candidates
62
+ # @return [Boolean]
63
+ def skip?(suggestions)
64
+ return false unless armed?
65
+
66
+ top = suggestions.first
67
+ return false unless top
68
+
69
+ top.confidence >= threshold
70
+ end
71
+ end
72
+ end
73
+ end