kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 1: Basic Word Checking
5
+ #
6
+ # This example demonstrates the simplest way to use Kotoshu
7
+ # to check if words are spelled correctly.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 1: Basic Word Checking ==="
12
+ puts
13
+
14
+ # Check if words are correct
15
+ puts "Is 'hello' correct? #{Kotoshu.correct?("hello")}"
16
+ puts "Is 'world' correct? #{Kotoshu.correct?("world")}"
17
+ puts "Is 'helo' correct? #{Kotoshu.correct?("helo")}"
18
+ puts "Is 'Kotoshu' correct? #{Kotoshu.correct?("Kotoshu")}"
19
+ puts
20
+
21
+ # Get suggestions for misspelled words
22
+ puts "Suggestions for 'helo':"
23
+ suggestions = Kotoshu.suggest("helo")
24
+ puts suggestions.to_words.join(", ")
25
+ puts
26
+
27
+ puts "Suggestions for 'wrold':"
28
+ suggestions = Kotoshu.suggest("wrold")
29
+ puts suggestions.to_words.join(", ")
30
+ puts
31
+
32
+ # Check multiple words
33
+ words = %w[hello world test helo wrold]
34
+ puts "Checking multiple words:"
35
+ words.each do |word|
36
+ status = Kotoshu.correct?(word) ? "✓" : "✗"
37
+ puts " #{status} #{word}"
38
+ end
@@ -0,0 +1,77 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 2: Text and Document Checking
5
+ #
6
+ # This example demonstrates how to check paragraphs and documents
7
+ # for spelling errors and get detailed results.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 2: Text and Document Checking ==="
12
+ puts
13
+
14
+ # Check a paragraph of text
15
+ text = <<~TEXT
16
+ Hello wrold!
17
+
18
+ This is a test document with some misspelled words.
19
+ We want to see if the spellchcker can find them al.
20
+
21
+ Teh quick brown fox jumps over the lazy dog.
22
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
23
+ TEXT
24
+
25
+ puts "Checking text:"
26
+ puts "-" * 40
27
+ result = Kotoshu.check(text)
28
+
29
+ puts result
30
+ unless result.success?
31
+ puts
32
+ puts "Errors found:"
33
+ result.each_error do |error|
34
+ suggestions_str = if error.has_suggestions?
35
+ " (did you mean #{error.top_suggestions(3).join(", ")}?)"
36
+ else
37
+ ""
38
+ end
39
+ puts " • #{error.word}#{suggestions_str}"
40
+ end
41
+ end
42
+
43
+ puts
44
+ puts "=" * 40
45
+ puts
46
+
47
+ # Check a file
48
+ file_path = "spec/fixtures/documents/with_errors.txt"
49
+ if File.exist?(file_path)
50
+ puts "Checking file: #{file_path}"
51
+ puts "-" * 40
52
+
53
+ file_result = Kotoshu.check_file(file_path)
54
+
55
+ if file_result.success?
56
+ puts "✓ No errors found (#{file_result.word_count} words checked)"
57
+ else
58
+ puts "✗ #{file_result.error_count} error(s) found:"
59
+ puts
60
+ file_result.each_unique_error do |word, errors|
61
+ puts " • #{word} (appears #{errors.size}x)"
62
+ first_error = errors.first
63
+ puts " Suggestions: #{first_error.top_suggestions(3).join(", ")}" if first_error.has_suggestions?
64
+ end
65
+ end
66
+ end
67
+
68
+ puts
69
+ puts "=" * 40
70
+ puts
71
+
72
+ # Document result statistics
73
+ puts "Document Statistics:"
74
+ puts " Word count: #{result.word_count}"
75
+ puts " Error count: #{result.error_count}"
76
+ puts " Unique errors: #{result.unique_error_count}"
77
+ puts " Error summary: #{result.error_summary.inspect}"
@@ -0,0 +1,137 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 3: Using Different Dictionary Backends
5
+ #
6
+ # This example demonstrates how to use different dictionary backends
7
+ # including UnixWords, PlainText, Custom, and Hunspell.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 3: Dictionary Backends ==="
12
+ puts
13
+
14
+ # Example 1: UnixWords dictionary
15
+ puts "1. UnixWords Dictionary (System Dictionary)"
16
+ puts "-" * 40
17
+
18
+ unix_dict = Kotoshu::Dictionary::UnixWords.detect(language_code: "en-US")
19
+ if unix_dict
20
+ puts "Loaded: #{unix_dict.path}"
21
+ puts "Words: #{unix_dict.size}"
22
+ puts "Has 'hello': #{unix_dict.lookup?("hello")}"
23
+ puts "Has 'Kotoshu': #{unix_dict.lookup?("Kotoshu")}"
24
+ suggestions = unix_dict.suggest("helo", max_suggestions: 5)
25
+ puts "Suggestions for 'helo': #{suggestions.join(", ")}"
26
+ else
27
+ puts "No system dictionary found"
28
+ end
29
+
30
+ puts
31
+ puts "=" * 40
32
+ puts
33
+
34
+ # Example 2: PlainText dictionary
35
+ puts "2. PlainText Dictionary"
36
+ puts "-" * 40
37
+
38
+ plain_dict = Kotoshu::Dictionary::PlainText.from_words(
39
+ %w[hello world kotoshu ruby spellchecker],
40
+ language_code: "en"
41
+ )
42
+
43
+ puts "Created dictionary with #{plain_dict.size} words"
44
+ puts "Has 'hello': #{plain_dict.lookup?("hello")}"
45
+ puts "Has 'ruby': #{plain_dict.lookup?("ruby")}"
46
+ puts "Has 'python': #{plain_dict.lookup?("python")}"
47
+
48
+ # Add a word dynamically
49
+ plain_dict.add_word("python")
50
+ puts "After adding 'python': #{plain_dict.lookup?("python")}"
51
+ plain_dict.add_word("Kotoshu")
52
+ puts "After adding 'Kotoshu': #{plain_dict.lookup?("Kotoshu")}"
53
+
54
+ puts
55
+ puts "=" * 40
56
+ puts
57
+
58
+ # Example 3: Custom dictionary
59
+ puts "3. Custom Dictionary (In-Memory)"
60
+ puts "-" * 40
61
+
62
+ custom_dict = Kotoshu::Dictionary::Custom.new(
63
+ words: %w[Kotoshu spellchecker ruby],
64
+ language_code: "en"
65
+ )
66
+
67
+ puts "Created custom dictionary"
68
+ puts "Words: #{custom_dict.words.inspect}"
69
+ puts "Size: #{custom_dict.size}"
70
+ puts "Has 'Kotoshu': #{custom_dict.lookup?("Kotoshu")}"
71
+
72
+ # Merge with another array
73
+ custom_dict.merge(%w[gem library code])
74
+ puts "After merging: #{custom_dict.words.inspect}"
75
+
76
+ puts
77
+ puts "=" * 40
78
+ puts
79
+
80
+ # Example 4: Hunspell dictionary (if available)
81
+ puts "4. Hunspell Dictionary"
82
+ puts "-" * 40
83
+
84
+ hunspell_dic = "dictionaries/hunspell/test/en_US_test.dic"
85
+ hunspell_aff = "dictionaries/hunspell/test/en_US_test.aff"
86
+
87
+ if File.exist?(hunspell_dic) && File.exist?(hunspell_aff)
88
+ hunspell_dict = Kotoshu::Dictionary::Hunspell.new(
89
+ dic_path: hunspell_dic,
90
+ aff_path: hunspell_aff,
91
+ language_code: "en-US"
92
+ )
93
+
94
+ puts "Loaded Hunspell dictionary"
95
+ puts "Words: #{hunspell_dict.size}"
96
+ puts "Has 'hello': #{hunspell_dict.lookup?("hello")}"
97
+ puts "Has 'hello' (case-insensitive): #{hunspell_dict.lookup?("HELLO")}"
98
+ puts "Has 'runs': #{hunspell_dict.lookup?("runs")}"
99
+ puts "Has 'running': #{hunspell_dict.lookup?("running")}"
100
+
101
+ # Show word variants using affix rules
102
+ puts "\nWord variants for 'run':"
103
+ variants = hunspell_dict.word_variants("run")
104
+ puts " #{variants.inspect}"
105
+ else
106
+ puts "Hunspell test dictionary not found at:"
107
+ puts " #{hunspell_dic}"
108
+ puts " #{hunspell_aff}"
109
+ end
110
+
111
+ puts
112
+ puts "=" * 40
113
+ puts
114
+
115
+ # Example 5: CSpell dictionary
116
+ puts "5. CSpell Dictionary (Trie-based)"
117
+ puts "-" * 40
118
+
119
+ cspell_dict = Kotoshu::Dictionary::CSpell.from_words(
120
+ %w[hello world kotoshu ruby gem],
121
+ language_code: "en"
122
+ )
123
+
124
+ puts "Created CSpell dictionary with trie"
125
+ puts "Words: #{cspell_dict.words.inspect}"
126
+ puts "Size: #{cspell_dict.size}"
127
+ puts "Has 'hello': #{cspell_dict.lookup?("hello")}"
128
+ puts "Has prefix 'hel': #{cspell_dict.has_prefix?("hel")}"
129
+ puts "Words with prefix 'hel': #{cspell_dict.words_with_prefix("hel").inspect}"
130
+
131
+ # Convert to trie
132
+ trie = cspell_dict.trie
133
+ puts "\nTrie structure:"
134
+ puts " Has 'hello': #{trie.has_word?("hello")}"
135
+ puts " Has prefix 'wo': #{trie.has_prefix?("wo")}"
136
+ puts " Words with prefix 'wo': #{trie.words_with_prefix("wo").inspect}"
137
+ puts " Suggestions for 'he': #{trie.suggestions("he").inspect}"
@@ -0,0 +1,146 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 4: Trie Data Structure
5
+ #
6
+ # This example demonstrates how to use the Trie data structure
7
+ # for efficient word lookup and prefix-based operations.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 4: Trie Data Structure ==="
12
+ puts
13
+
14
+ # Build a trie from an array of words
15
+ words = %w[
16
+ hello help held heap
17
+ world work word
18
+ test text toast
19
+ run running runner
20
+ ]
21
+
22
+ trie = Kotoshu.trie(words)
23
+
24
+ puts "Built trie with #{words.size} words"
25
+ puts "All words: #{trie.all_words.inspect}"
26
+ puts
27
+
28
+ # Lookup operations
29
+ puts "Lookup Operations:"
30
+ puts "-" * 20
31
+ puts "Has 'hello': #{trie.has_word?("hello")}"
32
+ puts "Has 'hell': #{trie.has_word?("hell")}"
33
+ puts "Has 'HELLO' (case-sensitive): #{trie.has_word?("HELLO")}"
34
+ puts
35
+
36
+ # Prefix operations
37
+ puts "Prefix Operations:"
38
+ puts "-" * 20
39
+ puts "Has prefix 'hel': #{trie.has_prefix?("hel")}"
40
+ puts "Has prefix 'wor': #{trie.has_prefix?("wor")}"
41
+ puts "Has prefix 'xyz': #{trie.has_prefix?("xyz")}"
42
+ puts
43
+
44
+ # Words with prefix
45
+ puts "Words with prefix 'hel': #{trie.words_with_prefix("hel").inspect}"
46
+ puts "Words with prefix 'te': #{trie.words_with_prefix("te").inspect}"
47
+ puts
48
+
49
+ # Suggestions based on prefix
50
+ puts "Suggestions for 'hel':"
51
+ puts " #{trie.suggestions("hel", max_results: 10).inspect}"
52
+ puts
53
+
54
+ puts "Suggestions for 'te':"
55
+ puts " #{trie.suggestions("te", max_results: 10).inspect}"
56
+ puts
57
+
58
+ # Traverse the trie
59
+ puts "Traversing trie:"
60
+ puts "-" * 20
61
+ trie.each_word do |word, payload|
62
+ puts " #{word} (payload: #{payload.inspect})"
63
+ end
64
+ puts
65
+
66
+ # Trie builder methods
67
+ puts "Building tries from different sources:"
68
+ puts "-" * 20
69
+
70
+ # From string
71
+ string_trie = Kotoshu.trie("hello world test")
72
+ puts "From string: #{string_trie.all_words.inspect}"
73
+
74
+ # From file (if exists)
75
+ test_file = "dictionaries/plain_text/en_US/words.txt"
76
+ if File.exist?(test_file)
77
+ file_trie = Kotoshu.trie(test_file)
78
+ puts "From file: loaded #{file_trie.size} words"
79
+ puts "First 5 words: #{file_trie.all_words.first(5).inspect}"
80
+ end
81
+
82
+ # Trie set operations
83
+ puts
84
+ puts "Trie Set Operations:"
85
+ puts "-" * 20
86
+
87
+ trie1 = Kotoshu.trie(%w[hello world test])
88
+ trie2 = Kotoshu.trie(%w[hello world ruby])
89
+
90
+ puts "Trie 1: #{trie1.all_words.inspect}"
91
+ puts "Trie 2: #{trie2.all_words.inspect}"
92
+ puts
93
+
94
+ # Union (|)
95
+ union = trie1 | trie2
96
+ puts "Union: #{union.all_words.inspect}"
97
+
98
+ # Intersection (&)
99
+ intersection = trie1 & trie2
100
+ puts "Intersection: #{intersection.all_words.inspect}"
101
+
102
+ # Merge (mutating)
103
+ merged = trie1.dup
104
+ merged.merge!(trie2)
105
+ puts "Merged: #{merged.all_words.inspect}"
106
+
107
+ # Difference
108
+ # Note: Trie doesn't have difference (-) operator, but we can simulate it
109
+ all_words = trie1.all_words | trie2.all_words
110
+ common = trie1.all_words & trie2.all_words
111
+ difference = all_words - common
112
+ puts "Words in only one trie: #{difference.inspect}"
113
+
114
+ puts
115
+ puts "Trie Statistics:"
116
+ puts "-" * 20
117
+ puts "Total words: #{trie.size}"
118
+ puts "Unique prefixes: #{trie.size}"
119
+ # puts "Max depth: #{trie.max_depth}" # Method not implemented yet
120
+
121
+ # Advanced: Payload storage
122
+ puts
123
+ puts "Advanced: Payload Storage:"
124
+ puts "-" * 20
125
+
126
+ payload_trie = Kotoshu::Core::Trie::Builder.new
127
+ payload_trie.add_word("hello", { definition: "a greeting", count: 5 })
128
+ payload_trie.add_word("help", { definition: "assistance", count: 3 })
129
+ payload_trie.add_word("world", { definition: "earth", count: 1 })
130
+
131
+ payload_trie_obj = payload_trie.build
132
+
133
+ puts "Word 'hello' payload: #{payload_trie_obj.find_node("hello")&.payload.inspect}"
134
+ puts "Word 'help' payload: #{payload_trie_obj.find_node("help")&.payload.inspect}"
135
+
136
+ # Convert IndexedDictionary to trie
137
+ puts
138
+ puts "IndexedDictionary to Trie:"
139
+ puts "-" * 20
140
+
141
+ dict = Kotoshu.dictionary(%w[hello world test])
142
+ trie_from_dict = dict.to_trie
143
+
144
+ puts "Dictionary words: #{dict.words.inspect}"
145
+ puts "Trie words: #{trie_from_dict.all_words.inspect}"
146
+ puts "Trie has 'hello': #{trie_from_dict.has_word?("hello")}"
@@ -0,0 +1,239 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 5: Suggestion Algorithms
5
+ #
6
+ # This example demonstrates how to use different suggestion algorithms
7
+ # and build custom suggestion pipelines.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 5: Suggestion Algorithms ==="
12
+ puts
13
+
14
+ # Create a test dictionary
15
+ test_words = %w[
16
+ hello help held heap world
17
+ test text toast tost
18
+ run running runner
19
+ code coding coded
20
+ write writing writer
21
+ speak speaking speaker
22
+ read reading reader
23
+ walk walking walker
24
+ talk talking talker
25
+ ]
26
+
27
+ test_dict = Kotoshu::Dictionary::Custom.new(
28
+ words: test_words,
29
+ language_code: "en"
30
+ )
31
+
32
+ puts "Test dictionary: #{test_dict.size} words"
33
+ puts
34
+
35
+ # Example 1: Edit Distance Strategy
36
+ puts "1. Edit Distance Strategy"
37
+ puts "-" * 40
38
+
39
+ edit_strategy = Kotoshu::Suggestions::Strategies::EditDistanceStrategy.new
40
+ context = Kotoshu::Suggestions::Context.new(
41
+ word: "helo",
42
+ dictionary: test_dict,
43
+ max_results: 5
44
+ )
45
+
46
+ result = edit_strategy.generate(context)
47
+ puts "Suggestions for 'helo':"
48
+ puts " #{result.to_words.join(", ")}"
49
+ puts " Details:"
50
+ result.each do |sugg|
51
+ puts " • #{sugg.word} (distance: #{sugg.distance}, confidence: #{sugg.confidence.round(2)})"
52
+ end
53
+
54
+ puts
55
+ puts "=" * 40
56
+ puts
57
+
58
+ # Example 2: Phonetic Strategy (Soundex)
59
+ puts "2. Phonetic Strategy (Soundex)"
60
+ puts "-" * 40
61
+
62
+ phonetic_strategy = Kotoshu::Suggestions::Strategies::PhoneticStrategy.new(
63
+ algorithm: :soundex
64
+ )
65
+
66
+ context2 = Kotoshu::Suggestions::Context.new(
67
+ word: "hel",
68
+ dictionary: test_dict,
69
+ max_results: 5
70
+ )
71
+
72
+ result2 = phonetic_strategy.generate(context2)
73
+ puts "Suggestions for 'hel' (Soundex):"
74
+ puts " #{result2.to_words.join(", ")}"
75
+
76
+ # Show Soundex codes
77
+ puts "\nSoundex codes:"
78
+ puts " 'hel' -> #{phonetic_strategy.send(:soundex_code, "hel")}"
79
+ test_words.each do |word|
80
+ code = phonetic_strategy.send(:soundex_code, word)
81
+ puts " '#{word}' -> #{code}"
82
+ end
83
+
84
+ puts
85
+ puts "=" * 40
86
+ puts
87
+
88
+ # Example 3: Phonetic Strategy (Metaphone)
89
+ puts "3. Phonetic Strategy (Metaphone)"
90
+ puts "-" * 40
91
+
92
+ metaphone_strategy = Kotoshu::Suggestions::Strategies::PhoneticStrategy.new(
93
+ algorithm: :metaphone
94
+ )
95
+
96
+ context3 = Kotoshu::Suggestions::Context.new(
97
+ word: "fnix", # Should suggest "Phoenix"
98
+ dictionary: test_dict,
99
+ max_results: 5
100
+ )
101
+
102
+ # Add "phoenix" to dictionary for testing
103
+ test_dict.add_word("phoenix")
104
+
105
+ result3 = metaphone_strategy.generate(context3)
106
+ puts "Suggestions for 'fnix' (Metaphone):"
107
+ puts " #{result3.to_words.join(", ")}"
108
+
109
+ puts "\nMetaphone codes:"
110
+ puts " 'fnix' -> #{metaphone_strategy.send(:metaphone_code, "fnix")}"
111
+ puts " 'phoenix' -> #{metaphone_strategy.send(:metaphone_code, "phoenix")}"
112
+ puts " 'finish' -> #{metaphone_strategy.send(:metaphone_code, "finish")}"
113
+
114
+ puts
115
+ puts "=" * 40
116
+ puts
117
+
118
+ # Example 4: N-Gram Strategy
119
+ puts "4. N-Gram Strategy"
120
+ puts "-" * 40
121
+
122
+ ngram_strategy = Kotoshu::Suggestions::Strategies::NgramStrategy.new(
123
+ n: 2,
124
+ min_similarity: 0.2
125
+ )
126
+
127
+ context4 = Kotoshu::Suggestions::Context.new(
128
+ word: "tsting", # Should suggest "testing"
129
+ dictionary: test_dict,
130
+ max_results: 5
131
+ )
132
+
133
+ # Add "testing" to dictionary
134
+ test_dict.add_word("testing")
135
+
136
+ result4 = ngram_strategy.generate(context4)
137
+ puts "Suggestions for 'tsting' (N-Gram, n=2):"
138
+ puts " #{result4.to_words.join(", ")}"
139
+
140
+ puts
141
+ puts "=" * 40
142
+ puts
143
+
144
+ # Example 5: Composite Strategy (Pipeline)
145
+ puts "5. Composite Strategy (Pipeline)"
146
+ puts "-" * 40
147
+
148
+ # Build a pipeline with multiple strategies
149
+ pipeline = Kotoshu.suggestion_pipeline(
150
+ Kotoshu::Suggestions::Strategies::EditDistanceStrategy.new,
151
+ Kotoshu::Suggestions::Strategies::PhoneticStrategy.new,
152
+ Kotoshu::Suggestions::Strategies::NgramStrategy.new(n: 2)
153
+ )
154
+
155
+ context5 = Kotoshu::Suggestions::Context.new(
156
+ word: "wrld",
157
+ dictionary: test_dict,
158
+ max_results: 10
159
+ )
160
+
161
+ result5 = pipeline.generate(context5)
162
+ puts "Suggestions for 'wrld' (Composite Pipeline):"
163
+ puts " #{result5.to_words.join(", ")}"
164
+
165
+ puts
166
+ puts "Breakdown by source:"
167
+ result5.from_source(:edit_distance).each do |sugg|
168
+ puts " EditDistance: #{sugg.word} (distance: #{sugg.distance})"
169
+ end
170
+ result5.from_source(:phonetic).each do |sugg|
171
+ puts " Phonetic: #{sugg.word}"
172
+ end
173
+ result5.from_source(:ngram).each do |sugg|
174
+ puts " N-Gram: #{sugg.word}"
175
+ end
176
+
177
+ puts
178
+ puts "=" * 40
179
+ puts
180
+
181
+ # Example 6: Custom Strategy
182
+ puts "6. Custom Strategy"
183
+ puts "-" * 40
184
+
185
+ class PrefixStrategy < Kotoshu::Suggestions::Strategies::BaseStrategy
186
+ def generate(context)
187
+ word = context.word
188
+ dict_words = dictionary_words(context)
189
+
190
+ # Find words with same prefix
191
+ prefix_len = [word.length - 1, 3].max
192
+ prefix = word[0...prefix_len]
193
+
194
+ candidates = dict_words.select { |w| w.start_with?(prefix) && w != word }
195
+ create_suggestion_set(candidates)
196
+ end
197
+ end
198
+
199
+ prefix_strategy = PrefixStrategy.new(name: :prefix)
200
+
201
+ context6 = Kotoshu::Suggestions::Context.new(
202
+ word: "hel", # Incomplete word
203
+ dictionary: test_dict,
204
+ max_results: 10
205
+ )
206
+
207
+ result6 = prefix_strategy.generate(context6)
208
+ puts "Suggestions for 'hel' (Prefix-based):"
209
+ puts " #{result6.to_words.join(", ")}"
210
+
211
+ puts
212
+ puts "=" * 40
213
+ puts
214
+
215
+ # Example 7: Suggestion Generator
216
+ puts "7. Suggestion Generator (High-level API)"
217
+ puts "-" * 40
218
+
219
+ generator = Kotoshu::Suggestions::Generator.new(
220
+ test_dict,
221
+ max_suggestions: 10,
222
+ algorithms: [
223
+ Kotoshu::Suggestions::Strategies::EditDistanceStrategy,
224
+ Kotoshu::Suggestions::Strategies::PhoneticStrategy
225
+ ]
226
+ )
227
+
228
+ puts "Generator configured with:"
229
+ puts " Dictionary: #{test_dict.size} words"
230
+ puts " Max suggestions: 10"
231
+ puts " Algorithms: EditDistanceStrategy, PhoneticStrategy"
232
+ puts
233
+
234
+ test_words = %w[helo wrld tsting fnix]
235
+ test_words.each do |word|
236
+ suggestions = generator.suggest(word)
237
+ puts "Suggestions for '#{word}':"
238
+ puts " #{suggestions.to_words.join(", ")}"
239
+ end