kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,154 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "lutaml/model"
4
+
5
+ module Kotoshu
6
+ module Suggestions
7
+ # A single suggestion with associated metadata and behavior.
8
+ #
9
+ # Serialized via lutaml-model. Use +to_hash+ / +Suggestion.as_json(instance)+ /
10
+ # +Suggestion.from_hash(hash)+ / +Suggestion.from_json(string)+ for the
11
+ # wire forms — no hand-rolled +to_h+ / +as_json+ on the model.
12
+ class Suggestion < Lutaml::Model::Serializable
13
+ attribute :word, :string
14
+ attribute :distance, :integer, default: 0
15
+ attribute :confidence, :float, default: 1.0
16
+ attribute :source, :string, default: "unknown"
17
+ attribute :metadata, :hash, default: {}
18
+
19
+ # Support the legacy +**metadata+ kwarg catch-all so existing callers
20
+ # (e.g., +Suggestion.new(word:, distance:, source:, original_length: 5)+)
21
+ # continue to work; extra kwargs land in the +metadata+ attribute.
22
+ # Source is stored as a string for clean serialization; +from_source?+
23
+ # normalizes Symbol/String comparison so callers can pass either.
24
+ #
25
+ # +word+ defaults to nil purely to accommodate lutaml-model's
26
+ # deserialization pathway, which allocates a shell via
27
+ # +new(lutaml_register:)+ before applying attribute values through
28
+ # the deserialize pipeline. Direct callers must still pass +word:+
29
+ # — a Suggestion without a word is degenerate and not user-facing.
30
+ def initialize(word: nil, distance: 0, confidence: 1.0, source: "unknown", **metadata)
31
+ lutaml_register = metadata.delete(:lutaml_register)
32
+ kwargs = {
33
+ word: word,
34
+ distance: distance,
35
+ confidence: confidence,
36
+ source: source.to_s,
37
+ metadata: metadata
38
+ }
39
+ kwargs[:lutaml_register] = lutaml_register if lutaml_register
40
+ super(**kwargs)
41
+ end
42
+
43
+ # Check if this is a high-confidence suggestion.
44
+ #
45
+ # @return [Boolean] True if confidence >= 0.8
46
+ def high_confidence?
47
+ confidence >= 0.8
48
+ end
49
+
50
+ # Check if this is a low-confidence suggestion.
51
+ #
52
+ # @return [Boolean] True if confidence < 0.5
53
+ def low_confidence?
54
+ confidence < 0.5
55
+ end
56
+
57
+ # Calculate combined score considering distance and confidence.
58
+ #
59
+ # @param distance_weight [Float] Weight for distance (default: 0.3)
60
+ # @param confidence_weight [Float] Weight for confidence (default: 0.7)
61
+ # @return [Float] Combined score (0.0 to 1.0, higher is better)
62
+ def combined_score(distance_weight: 0.3, confidence_weight: 0.7)
63
+ normalized_distance = [distance, 5].min / 5.0
64
+ distance_score = 1.0 - normalized_distance
65
+
66
+ (distance_score * distance_weight) + (confidence * confidence_weight)
67
+ end
68
+
69
+ # Check if this suggestion is the same word as another.
70
+ #
71
+ # @param other [Suggestion, String] The other suggestion or word string
72
+ # @return [Boolean] True if words match (case-insensitive)
73
+ def same_word?(other)
74
+ other_word = other.is_a?(Suggestion) ? other.word : other.to_s
75
+ word.downcase == other_word.downcase
76
+ end
77
+
78
+ # Check if this suggestion comes from a specific source.
79
+ #
80
+ # Source is stored as a string; comparison normalizes Symbol/String.
81
+ #
82
+ # @param source [String, Symbol] The source to check
83
+ # @return [Boolean] True if this suggestion came from the source
84
+ def from_source?(source)
85
+ self.source == source.to_s
86
+ end
87
+
88
+ # Compare suggestions for sorting (higher combined score first).
89
+ #
90
+ # Ranking priority (following CSpell/Hunspell approach):
91
+ # 1. Combined score (higher is better)
92
+ # 2. Edit distance (lower is better)
93
+ # 3. Length similarity (prefer similar length to original word)
94
+ # 4. N-gram similarity (more shared n-grams is better)
95
+ # 5. Alphabetical (ONLY as final tiebreaker)
96
+ #
97
+ # @param other [Suggestion] The other suggestion
98
+ # @return [Integer, nil] -1, 0, or 1; nil if +other+ is not a Suggestion
99
+ def <=>(other)
100
+ return nil unless other.is_a?(Suggestion)
101
+
102
+ score_cmp = other.combined_score <=> combined_score
103
+ return score_cmp unless score_cmp.zero?
104
+
105
+ distance_cmp = distance <=> other.distance
106
+ return distance_cmp unless distance_cmp.zero?
107
+
108
+ orig_len = metadata[:original_length] || word.length
109
+ other_orig_len = other.metadata[:original_length] || other.word.length
110
+
111
+ my_len_diff = (word.length - orig_len).abs
112
+ other_len_diff = (other.word.length - other_orig_len).abs
113
+
114
+ len_cmp = my_len_diff <=> other_len_diff
115
+ return len_cmp unless len_cmp.zero?
116
+
117
+ my_ngram = metadata[:ngram_score] || 0
118
+ other_ngram = other.metadata[:ngram_score] || 0
119
+
120
+ ngram_cmp = other_ngram <=> my_ngram
121
+ return ngram_cmp unless ngram_cmp.zero?
122
+
123
+ word.downcase <=> other.word.downcase
124
+ end
125
+
126
+ def ==(other)
127
+ return false unless other.is_a?(Suggestion)
128
+
129
+ word.downcase == other.word.downcase
130
+ end
131
+ alias eql? ==
132
+
133
+ def hash
134
+ word.downcase.hash
135
+ end
136
+
137
+ def to_s
138
+ format("Suggestion(word: '%<word>s', distance: %<distance>d, confidence: %<confidence>.2f, source: %<source>s)",
139
+ word: word, distance: distance, confidence: confidence, source: source)
140
+ end
141
+
142
+ alias inspect to_s
143
+
144
+ # Create a suggestion from a simple word (convenience method).
145
+ #
146
+ # @param word [String] The word
147
+ # @param source [String, Symbol] The source
148
+ # @return [Suggestion] New suggestion
149
+ def self.from_word(word, source: :unknown)
150
+ new(word: word, distance: 0, confidence: 1.0, source: source)
151
+ end
152
+ end
153
+ end
154
+ end
@@ -0,0 +1,270 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # A collection of suggestions with rich query methods.
6
+ # This is MORE OOP than Spylls which returns plain iterators of strings.
7
+ class SuggestionSet
8
+ include Enumerable
9
+
10
+ # @return [Integer] Number of duplicate suggestions removed by the
11
+ # last call to {#deduplicate!}. Zero until deduplicate! runs.
12
+ attr_reader :suggestions, :max_size, :duplicates_removed
13
+
14
+ # @param suggestions [Array<Suggestion>] Initial suggestions
15
+ # @param max_size [Integer] Maximum number of suggestions to keep
16
+ # @param ranked [Boolean] When true, the producer already ranked,
17
+ # deduplicated, and limited these suggestions and the set adopts
18
+ # their order verbatim (only truncation to +max_size+ applies).
19
+ # Used by producers whose ranking is conformance-frozen — the
20
+ # native engine (NativeBackend), whose tie order can depend on
21
+ # ranking signals (e.g. ngram scores) not carried on the Suggestion
22
+ # objects themselves; re-sorting here would permute those ties.
23
+ def initialize(suggestions = [], max_size: 10, ranked: false)
24
+ @suggestions = suggestions
25
+ @max_size = max_size
26
+ @duplicates_removed = 0
27
+ ranked ? limit! : sort_and_limit!
28
+ end
29
+
30
+ # Add a suggestion to the set.
31
+ #
32
+ # @param suggestion [Suggestion] The suggestion to add
33
+ # @return [SuggestionSet] Self for chaining
34
+ def add(suggestion)
35
+ @suggestions << suggestion
36
+ sort_and_limit!
37
+ self
38
+ end
39
+ alias << add
40
+
41
+ # Add multiple suggestions.
42
+ #
43
+ # @param new_suggestions [Array<Suggestion>] Suggestions to add
44
+ # @return [SuggestionSet] Self for chaining
45
+ def concat(new_suggestions)
46
+ @suggestions.concat(new_suggestions)
47
+ sort_and_limit!
48
+ self
49
+ end
50
+
51
+ # Merge another suggestion set into this one.
52
+ #
53
+ # @param other [SuggestionSet] The other set
54
+ # @return [SuggestionSet] Self for chaining
55
+ def merge!(other)
56
+ concat(other.suggestions)
57
+ self
58
+ end
59
+
60
+ # Get suggestions by source.
61
+ #
62
+ # @param source [String, Symbol] The source to filter by
63
+ # @return [SuggestionSet] New set with filtered suggestions
64
+ def from_source(source)
65
+ SuggestionSet.new(@suggestions.select { |s| s.from_source?(source) }, max_size: @max_size)
66
+ end
67
+
68
+ # Get high-confidence suggestions.
69
+ #
70
+ # @return [SuggestionSet] New set with high-confidence suggestions
71
+ def high_confidence
72
+ SuggestionSet.new(@suggestions.select(&:high_confidence?), max_size: @max_size)
73
+ end
74
+
75
+ # Get low-confidence suggestions.
76
+ #
77
+ # @return [SuggestionSet] New set with low-confidence suggestions
78
+ def low_confidence
79
+ SuggestionSet.new(@suggestions.select(&:low_confidence?), max_size: @max_size)
80
+ end
81
+
82
+ # Get suggestions within a distance range.
83
+ #
84
+ # @param min_distance [Integer] Minimum distance
85
+ # @param max_distance [Integer] Maximum distance
86
+ # @return [SuggestionSet] New set with filtered suggestions
87
+ def within_distance(min_distance: 0, max_distance: 2)
88
+ filtered = @suggestions.select do |s|
89
+ s.distance >= min_distance && s.distance <= max_distance
90
+ end
91
+ SuggestionSet.new(filtered, max_size: @max_size)
92
+ end
93
+
94
+ # Check if set contains a specific word.
95
+ #
96
+ # @param word [String] The word to check
97
+ # @return [Boolean] True if word is in suggestions
98
+ def include?(word)
99
+ @suggestions.any? { |s| s.same_word?(word) }
100
+ end
101
+ alias has_word? include?
102
+
103
+ # Find a suggestion by word.
104
+ #
105
+ # @param word [String] The word to find
106
+ # @return [Suggestion, nil] The suggestion or nil
107
+ def find_word(word)
108
+ @suggestions.find { |s| s.same_word?(word) }
109
+ end
110
+
111
+ # Get the top N suggestions.
112
+ #
113
+ # @param n [Integer] Number of suggestions to get
114
+ # @return [Array<Suggestion>] Top N suggestions
115
+ def top(n)
116
+ @suggestions.first(n)
117
+ end
118
+
119
+ # Get the first (best) suggestion.
120
+ #
121
+ # @return [Suggestion, nil] The best suggestion or nil
122
+ def first
123
+ @suggestions.first
124
+ end
125
+
126
+ # Get the last suggestion.
127
+ #
128
+ # @return [Suggestion, nil] The last suggestion or nil
129
+ def last
130
+ @suggestions.last
131
+ end
132
+
133
+ # Check if the set is empty.
134
+ #
135
+ # @return [Boolean] True if no suggestions
136
+ def empty?
137
+ @suggestions.empty?
138
+ end
139
+
140
+ # Get the number of suggestions.
141
+ #
142
+ # @return [Integer] Number of suggestions
143
+ def size
144
+ @suggestions.size
145
+ end
146
+ alias count size
147
+ alias length size
148
+
149
+ # Iterate over suggestions.
150
+ #
151
+ # @yield [suggestion] Each suggestion
152
+ # @return [Enumerator] Enumerator if no block given
153
+ def each(&)
154
+ return enum_for(:each) unless block_given?
155
+
156
+ @suggestions.each(&)
157
+ end
158
+
159
+ # Get unique suggestions (by word, case-insensitive).
160
+ #
161
+ # @return [SuggestionSet] New set with unique suggestions
162
+ def unique
163
+ seen = {}
164
+ unique_suggestions = @suggestions.select do |s|
165
+ word = s.word.downcase
166
+ if seen[word]
167
+ false
168
+ else
169
+ seen[word] = true
170
+ true
171
+ end
172
+ end
173
+ SuggestionSet.new(unique_suggestions, max_size: @max_size)
174
+ end
175
+
176
+ # Convert to array of words.
177
+ #
178
+ # @return [Array<String>] Array of suggestion words
179
+ def to_words
180
+ @suggestions.map(&:word)
181
+ end
182
+ alias words to_words
183
+
184
+ # Convert to an array of hashes for serialization.
185
+ #
186
+ # Use this when you need the Hash shape (e.g. JSON output). For the
187
+ # enumerable contract, {#to_a} / {#each} yield {Suggestion} objects.
188
+ #
189
+ # @return [Array<Hash>] Array of suggestion hashes
190
+ def to_hashes
191
+ @suggestions.map(&:to_hash)
192
+ end
193
+
194
+ # Convert to JSON-compatible array.
195
+ #
196
+ # @return [Array<Hash>] JSON-compatible array
197
+ def as_json(*)
198
+ to_hashes
199
+ end
200
+
201
+ # String representation.
202
+ #
203
+ # @return [String] String representation
204
+ def to_s
205
+ "SuggestionSet(size: #{size}, max_size: #{@max_size})"
206
+ end
207
+
208
+ # Inspect the suggestion set.
209
+ #
210
+ # @return [String] Inspection string
211
+ def inspect
212
+ if @suggestions.empty?
213
+ to_s
214
+ else
215
+ "#{self} [#{@suggestions.map(&:word).join(', ')}]"
216
+ end
217
+ end
218
+
219
+ # Create an empty suggestion set.
220
+ #
221
+ # @param max_size [Integer] Maximum size
222
+ # @return [SuggestionSet] Empty set
223
+ def self.empty(max_size: 10)
224
+ new([], max_size: max_size)
225
+ end
226
+
227
+ # Create a suggestion set from an array of words.
228
+ #
229
+ # @param words [Array<String>] Array of words
230
+ # @param source [String, Symbol] The source
231
+ # @param max_size [Integer] Maximum size
232
+ # @return [SuggestionSet] New set
233
+ def self.from_words(words, source: :unknown, max_size: 10)
234
+ suggestions = words.map { |w| Suggestion.from_word(w, source: source) }
235
+ new(suggestions, max_size: max_size)
236
+ end
237
+
238
+ private
239
+
240
+ # Sort suggestions by combined score.
241
+ def sort!
242
+ @suggestions.sort!
243
+ end
244
+
245
+ # Remove duplicate suggestions (same word, case-insensitive).
246
+ # Called explicitly so the dedup is visible as a named step,
247
+ # not buried inside sort_and_limit as a side effect (TODO 56 T5.1).
248
+ # Tracks the count of removed duplicates for diagnostics.
249
+ def deduplicate!
250
+ before = @suggestions.length
251
+ @suggestions.uniq! { |s| s.word.downcase }
252
+ @duplicates_removed = before - @suggestions.length
253
+ end
254
+
255
+ # Truncate to max_size.
256
+ def limit!
257
+ @suggestions = @suggestions.first(@max_size)
258
+ end
259
+
260
+ # Sort, deduplicate, then limit. The canonical pipeline called
261
+ # after every mutation (add, concat, merge!). Each step is a
262
+ # named method so callers can reason about the order.
263
+ def sort_and_limit!
264
+ sort!
265
+ deduplicate!
266
+ limit!
267
+ end
268
+ end
269
+ end
270
+ end
@@ -0,0 +1,137 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # Per-dictionary sweep invariants, built once at the first
6
+ # suggestion sweep and kept for the dictionary's lifetime.
7
+ #
8
+ # Every strategy used to walk the whole word list per sweep and
9
+ # compute per-word values before its gate — the length window, the
10
+ # Soundex code, the n-gram length bound — so a full dictionary paid
11
+ # for work on words a sweep was about to skip. Those per-word
12
+ # values never change with the query: they are computed here one
13
+ # time (char lengths, packed Soundex codes, length buckets), and
14
+ # the sweeps touch only the entries their gates admit.
15
+ #
16
+ # Soundex codes are the 4-character {Algorithms::Soundex} strings
17
+ # (frozen); two letter-less words both code to +""+ and compare
18
+ # equal, exactly as the live computation does. Lengths are char
19
+ # counts. +by_length+ groups word indices by length in original
20
+ # order so the edit-distance sweep can iterate only its ±2 window;
21
+ # {#indices_in_length_range} restores word-list order because ties
22
+ # in the strategies' ranking sorts are decided by input order.
23
+ #
24
+ # The index derives only from the word list it was built over.
25
+ # Dictionary backends memoize it (see
26
+ # {Dictionary::Base#sweep_index}) and invalidate the memo wherever
27
+ # they mutate the word list ({Dictionary::Base#add_word} /
28
+ # {Dictionary::Base#remove_word} implementations).
29
+ #
30
+ # This mirrors SweepIndex in kotoshu-rs
31
+ # (kotoshu/src/suggest/sweep_index.rs).
32
+ class SweepIndex
33
+ # @return [Array<String>] The words in list order (as captured at
34
+ # build time)
35
+ attr_reader :words
36
+
37
+ # Build an index over a word list.
38
+ #
39
+ # @param words [Array<String>] Dictionary words in list order
40
+ # @return [SweepIndex] The built index
41
+ def self.build(words)
42
+ lengths = Array.new(words.length)
43
+ soundex_codes = Array.new(words.length)
44
+ by_length = {}
45
+ words.each_with_index do |word, idx|
46
+ length = word.length
47
+ lengths[idx] = length
48
+ soundex_codes[idx] = Algorithms::Soundex.code(word).freeze
49
+ (by_length[length] ||= []) << idx
50
+ end
51
+ new(words, lengths, soundex_codes, by_length)
52
+ end
53
+
54
+ # Create an index from prebuilt structures.
55
+ #
56
+ # @param words [Array<String>] The indexed words in list order
57
+ # @param lengths [Array<Integer>] Char length per word index
58
+ # @param soundex_codes [Array<String>] Soundex code per word index
59
+ # @param by_length [Hash<Integer, Array<Integer>>] Length => word
60
+ # indices in list order
61
+ def initialize(words, lengths, soundex_codes, by_length)
62
+ @words = words
63
+ @lengths = lengths
64
+ @soundex_codes = soundex_codes
65
+ @by_length = by_length
66
+ end
67
+
68
+ # Char length of the word at +idx+.
69
+ #
70
+ # @param idx [Integer] Word index
71
+ # @return [Integer] Char count
72
+ def length(idx)
73
+ @lengths[idx]
74
+ end
75
+
76
+ # Soundex code of the word at +idx+ — equal codes compare equal,
77
+ # including two letter-less empty codes.
78
+ #
79
+ # @param idx [Integer] Word index
80
+ # @return [String] The frozen 4-char (or empty) Soundex code
81
+ def soundex(idx)
82
+ @soundex_codes[idx]
83
+ end
84
+
85
+ # Word indices whose length is within +min..max+, every word in
86
+ # exactly one bucket, each bucket in original order.
87
+ #
88
+ # @param min [Integer] Minimum length (inclusive)
89
+ # @param max [Integer] Maximum length (inclusive)
90
+ # @return [Array<Integer>] Matching indices in word-list order
91
+ def indices_in_length_range(min, max)
92
+ hits = []
93
+ (min..max).each do |length|
94
+ bucket = @by_length[length]
95
+ hits.concat(bucket) if bucket
96
+ end
97
+ # Restore the word-list order the strategies' candidate arrays
98
+ # were built in before this index existed — ties in the ranking
99
+ # sort are decided by input order, so this must not drift.
100
+ hits.sort!
101
+ end
102
+
103
+ # Words whose length is within +min..max+, in word-list order —
104
+ # exactly the set and order {Dictionary::Base#find_by_length_range}
105
+ # historically returned.
106
+ #
107
+ # @param min [Integer] Minimum length (inclusive)
108
+ # @param max [Integer] Maximum length (inclusive)
109
+ # @return [Array<String>] Matching words in word-list order
110
+ def words_in_length_range(min, max)
111
+ indices_in_length_range(min, max).map { |idx| @words[idx] }
112
+ end
113
+
114
+ # Yield every word with its memoized char length.
115
+ #
116
+ # @yield [String, Integer] Each word and its char count
117
+ # @return [void]
118
+ def each_with_length
119
+ @words.each_with_index do |word, idx|
120
+ yield word, @lengths[idx]
121
+ end
122
+ nil
123
+ end
124
+
125
+ # Yield every word with its memoized Soundex code.
126
+ #
127
+ # @yield [String, String] Each word and its Soundex code
128
+ # @return [void]
129
+ def each_with_soundex
130
+ @words.each_with_index do |word, idx|
131
+ yield word, @soundex_codes[idx]
132
+ end
133
+ nil
134
+ end
135
+ end
136
+ end
137
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Suggestion generation system: strategies, generator, and result types.
5
+ module Suggestions
6
+ autoload :Context, "kotoshu/suggestions/context"
7
+ autoload :FrequencyProvider, "kotoshu/suggestions/frequency_provider"
8
+ autoload :FrozenTiers, "kotoshu/data/frozen_kelly"
9
+ autoload :Generator, "kotoshu/suggestions/generator"
10
+ autoload :Pipeline, "kotoshu/suggestions/pipeline"
11
+ autoload :SemanticCascade, "kotoshu/suggestions/semantic_cascade"
12
+ autoload :Suggestion, "kotoshu/suggestions/suggestion"
13
+ autoload :SuggestionSet, "kotoshu/suggestions/suggestion_set"
14
+ autoload :SweepIndex, "kotoshu/suggestions/sweep_index"
15
+
16
+ # Strategies sub-namespace.
17
+ module Strategies
18
+ autoload :BaseStrategy, "kotoshu/suggestions/strategies/base_strategy"
19
+ autoload :CompositeStrategy, "kotoshu/suggestions/strategies/composite_strategy"
20
+ autoload :EditDistanceStrategy, "kotoshu/suggestions/strategies/edit_distance_strategy"
21
+ autoload :KeyboardProximityStrategy, "kotoshu/suggestions/strategies/keyboard_proximity_strategy"
22
+ autoload :NgramStrategy, "kotoshu/suggestions/strategies/ngram_strategy"
23
+ autoload :PhoneticStrategy, "kotoshu/suggestions/strategies/phonetic_strategy"
24
+ autoload :SemanticStrategy, "kotoshu/suggestions/strategies/semantic_strategy"
25
+ autoload :SymspellStrategy, "kotoshu/suggestions/strategies/symspell_strategy"
26
+ autoload :WordIndex, "kotoshu/suggestions/strategies/word_index"
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Rake integration namespace (plan 89). Requiring
5
+ # `kotoshu/tasks` installs the default {CheckTask}; build custom
6
+ # tasks by constructing {CheckTask} yourself.
7
+ module Tasks
8
+ # Rake task checking a repository text files with the plan-88
9
+ # file selection (plan 89, item 3).
10
+ #
11
+ # @example Default task via require
12
+ # # Rakefile
13
+ # require "kotoshu/tasks"
14
+ # # provides rake kotoshu:check over the repo text files
15
+ #
16
+ # @example Configured
17
+ # Kotoshu::Tasks::CheckTask.new do |t|
18
+ # t.files = FileList["_posts/**/*.md"]
19
+ # t.language = "en"
20
+ # t.fail_on_error = true
21
+ # end
22
+ class CheckTask
23
+ include Rake::DSL
24
+
25
+ # @param name [Symbol] task name under the kotoshu namespace
26
+ # @yieldparam task [CheckTask] self for configuration
27
+ def initialize(name = :check, &block)
28
+ @name = name
29
+ @files = nil
30
+ @root = Dir.pwd
31
+ @language = nil
32
+ @fail_on_error = true
33
+ @baseline_path = nil
34
+ yield self if block
35
+
36
+ define
37
+ end
38
+
39
+ attr_accessor :name, :files, :root, :language, :fail_on_error, :baseline_path
40
+
41
+ # The files the task will check: the configured list, or the
42
+ # plan-88 selection over the root.
43
+ #
44
+ # @return [Array<String>]
45
+ def target_files
46
+ return Array(files) if files
47
+
48
+ Cli::DirectoryWalker.new.files(root)
49
+ end
50
+
51
+ private
52
+
53
+ # Register the kotoshu:<name> task.
54
+ def define
55
+ desc "Check spelling in the repository text files"
56
+ namespace :kotoshu do
57
+ task(name) { run }
58
+ end
59
+ end
60
+
61
+ # Run the check and abort on errors when fail_on_error.
62
+ def run
63
+ runner = Cli::DirectoryCheck.new(
64
+ target_files,
65
+ check: ->(text, suggestions_filter: nil) { check_text(text, suggestions_filter: suggestions_filter) },
66
+ baseline_path: baseline_path
67
+ )
68
+ runner.run
69
+ report(runner)
70
+ if fail_on_error && runner.failed?
71
+ abort("kotoshu: #{runner.error_count} spelling error(s) in " \
72
+ "#{runner.file_count} file(s)")
73
+ end
74
+ end
75
+
76
+ # Check one text through the configured language.
77
+ #
78
+ # @param text [String] file contents
79
+ # @param suggestions_filter [#call, nil] baseline budget filter
80
+ # (plan 116) — see Baseline::Store#suggestions_filter_for
81
+ # @return [Models::Result::DocumentResult]
82
+ def check_text(text, suggestions_filter: nil)
83
+ if language
84
+ Kotoshu.spellchecker_for(language).check(text, suggestions_filter: suggestions_filter)
85
+ else
86
+ Kotoshu.check(text, suggestions_filter: suggestions_filter)
87
+ end
88
+ end
89
+
90
+ # Print one line per file with errors plus a summary.
91
+ def report(runner)
92
+ runner.outcomes.each do |outcome|
93
+ next if outcome.result.success?
94
+
95
+ outcome.result.each_error do |error|
96
+ suggestions = error.has_suggestions? ? " -> #{error.top_suggestions(3).join(', ')}" : ""
97
+ puts "#{outcome.path}: #{error.word}#{suggestions}"
98
+ end
99
+ end
100
+ puts "#{runner.file_count} file(s) checked, #{runner.error_count} error(s)"
101
+ end
102
+ end
103
+ end
104
+ end