kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Dictionary catalog: registry of known dictionary sources per language.
5
+ module Dictionaries
6
+ autoload :Catalog, "kotoshu/dictionaries/catalog"
7
+ end
8
+ end
@@ -0,0 +1,324 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Dictionary
5
+ # Base class for all dictionary backends.
6
+ #
7
+ # This abstract class defines the interface that all dictionary
8
+ # implementations must follow.
9
+ #
10
+ # @note Subclasses must implement the abstract methods: {#lookup},
11
+ # {#suggest}, {#add_word}, and {#remove_word}.
12
+ #
13
+ # @example Implementing a custom dictionary
14
+ # class MyDictionary < Base
15
+ # def initialize(path, language_code:, locale: nil)
16
+ # super(language_code, locale: locale)
17
+ # @words = load_words(path)
18
+ # end
19
+ #
20
+ # def lookup(word)
21
+ # @words.include?(word.downcase)
22
+ # end
23
+ #
24
+ # # ... implement other abstract methods
25
+ # end
26
+ class Base
27
+ # @return [String] The language code (e.g., "en-US", "en-GB")
28
+ attr_reader :language_code
29
+
30
+ # @return [String, nil] The locale (e.g., "en", "en_US")
31
+ attr_reader :locale
32
+
33
+ # @return [String, nil] On-disk path for file-backed dictionaries.
34
+ # Nil for in-memory backends (e.g. Custom). Subclasses that load
35
+ # from a file override this with their own +attr_reader :path+.
36
+ def path
37
+ nil
38
+ end
39
+
40
+ # @return [Hash] Additional metadata
41
+ attr_reader :metadata
42
+
43
+ # Create a new dictionary.
44
+ #
45
+ # @param language_code [String] The language code (e.g., "en-US")
46
+ # @param locale [String, nil] The locale (optional)
47
+ # @param metadata [Hash] Additional metadata (optional)
48
+ def initialize(language_code, locale: nil, metadata: {})
49
+ raise ArgumentError, "Language code cannot be empty" if language_code.nil? || language_code.empty?
50
+
51
+ @language_code = language_code.dup.freeze
52
+ @locale = locale&.dup&.freeze
53
+ @metadata = metadata.dup.freeze
54
+ end
55
+
56
+ # Check if a word exists in the dictionary.
57
+ #
58
+ # @abstract Subclasses must implement this method.
59
+ # @param word [String] The word to look up
60
+ # @return [Boolean] True if the word exists
61
+ # @raise [NotImplementedError] Subclass must implement
62
+ def lookup(word)
63
+ raise NotImplementedError, "#{self.class} must implement #lookup"
64
+ end
65
+
66
+ # Check if a word exists in the dictionary (alias for lookup).
67
+ #
68
+ # Dispatches dynamically to whatever +lookup+ the concrete subclass
69
+ # defines. (Ruby's +alias+ keyword would bind to Base#lookup at
70
+ # definition time and bypass subclass overrides — these one-line
71
+ # method definitions are what makes the alias actually work.)
72
+ #
73
+ # @param word [String] The word to look up
74
+ # @return [Boolean] True if the word exists
75
+ def lookup?(word)
76
+ lookup(word)
77
+ end
78
+
79
+ def has_word?(word)
80
+ lookup(word)
81
+ end
82
+
83
+ def include?(word)
84
+ lookup(word)
85
+ end
86
+
87
+ def contains?(word)
88
+ lookup(word)
89
+ end
90
+
91
+ # Generate spelling suggestions for a word.
92
+ #
93
+ # @abstract Subclasses must implement this method.
94
+ # @param word [String] The misspelled word
95
+ # @param max_suggestions [Integer] Maximum number of suggestions
96
+ # @return [Array<String>] List of suggested words
97
+ # @raise [NotImplementedError] Subclass must implement
98
+ def suggest(word, max_suggestions: 10)
99
+ raise NotImplementedError, "#{self.class} must implement #suggest"
100
+ end
101
+
102
+ # Add a word to the dictionary.
103
+ #
104
+ # @abstract Subclasses must implement this method.
105
+ # @param word [String] The word to add
106
+ # @param flags [Array<String>] Morphological flags (optional)
107
+ # @return [Boolean] True if the word was added
108
+ # @raise [NotImplementedError] Subclass must implement
109
+ def add_word(word, flags: [])
110
+ raise NotImplementedError, "#{self.class} must implement #add_word"
111
+ end
112
+
113
+ def <<(word, flags: [])
114
+ add_word(word, flags: flags)
115
+ end
116
+
117
+ # Remove a word from the dictionary.
118
+ #
119
+ # @abstract Subclasses must implement this method.
120
+ # @param word [String] The word to remove
121
+ # @return [Boolean] True if the word was removed
122
+ # @raise [NotImplementedError] Subclass must implement
123
+ def remove_word(word)
124
+ raise NotImplementedError, "#{self.class} must implement #remove_word"
125
+ end
126
+
127
+ # Get all words in the dictionary.
128
+ #
129
+ # @abstract Subclasses must implement this method.
130
+ # @return [Array<String>] All words
131
+ # @raise [NotImplementedError] Subclass must implement
132
+ def words
133
+ raise NotImplementedError, "#{self.class} must implement #words"
134
+ end
135
+
136
+ def all_words
137
+ words
138
+ end
139
+
140
+ # The dictionary's TRY string: characters to try when generating
141
+ # single-edit candidates (substitutions, insertions), from the
142
+ # Hunspell TRY directive. Backends without a TRY concept answer
143
+ # nil — callers must treat nil as "no substitution/insertion
144
+ # alphabet" and fall back to the edit operations that need no
145
+ # alphabet (transposition, deletion).
146
+ #
147
+ # @return [String, nil] TRY characters, or nil when unavailable
148
+ def try_string
149
+ nil
150
+ end
151
+
152
+ # The sweep invariants for this dictionary's word list — char
153
+ # lengths, Soundex codes and length buckets — built once at the
154
+ # first suggestion sweep and shared by every later sweep (see
155
+ # {Suggestions::SweepIndex}).
156
+ #
157
+ # The index derives only from {#words}. Backends that mutate the
158
+ # word list (their {#add_word} / {#remove_word} implementations)
159
+ # call {#reset_sweep_index} so the memo cannot go stale; backends
160
+ # that cannot mutate after load (e.g. CSpell's frozen trie) never
161
+ # need to.
162
+ #
163
+ # @return [Suggestions::SweepIndex] The memoized sweep index
164
+ def sweep_index
165
+ @sweep_index ||= Suggestions::SweepIndex.build(words)
166
+ end
167
+
168
+ # Drop the memoized {#sweep_index} so the next sweep rebuilds it
169
+ # from the current word list. Mutating backends call this after
170
+ # every successful word-list change.
171
+ #
172
+ # @return [self] Self for chaining
173
+ def reset_sweep_index
174
+ @sweep_index = nil
175
+ self
176
+ end
177
+
178
+ # Return words whose length is in [min_length, max_length].
179
+ #
180
+ # Uses the memoized {#sweep_index} length buckets — O(buckets in
181
+ # range) instead of an O(n) scan — and restores word-list order,
182
+ # exactly the set and order the historical whole-list filter
183
+ # produced. Subclasses that maintain their own length index
184
+ # (e.g. PlainText) keep their override.
185
+ #
186
+ # @param min_length [Integer] Minimum length (inclusive)
187
+ # @param max_length [Integer] Maximum length (inclusive)
188
+ # @return [Array<String>] Words in the length range
189
+ def find_by_length_range(min_length:, max_length:)
190
+ sweep_index.words_in_length_range(min_length, max_length)
191
+ end
192
+
193
+ # Get the number of words in the dictionary.
194
+ #
195
+ # @return [Integer] Word count
196
+ def size
197
+ words.length
198
+ end
199
+ alias count size
200
+ alias length size
201
+
202
+ # Check if the dictionary is empty.
203
+ #
204
+ # @return [Boolean] True if empty
205
+ def empty?
206
+ size.zero?
207
+ end
208
+
209
+ # Iterate over all words.
210
+ #
211
+ # @yield [word] Each word
212
+ # @return [Enumerator] Enumerator if no block given
213
+ def each_word(&)
214
+ return enum_for(:each_word) unless block_given?
215
+
216
+ words.each(&)
217
+ end
218
+
219
+ # Get words starting with a prefix.
220
+ #
221
+ # @param prefix [String] The prefix
222
+ # @return [Array<String>] Words with the prefix
223
+ def words_with_prefix(prefix)
224
+ words.select { |w| w.start_with?(prefix) }
225
+ end
226
+
227
+ # Get words matching a pattern.
228
+ #
229
+ # @param pattern [Regexp] The pattern
230
+ # @return [Array<String>] Matching words
231
+ def words_matching(pattern)
232
+ words.select { |w| w.match?(pattern) }
233
+ end
234
+
235
+ # Convert to string.
236
+ #
237
+ # @return [String] String representation
238
+ def to_s
239
+ "#{self.class.name}(language: #{@language_code}, size: #{size})"
240
+ end
241
+ alias inspect to_s
242
+
243
+ # Dictionary type identifier.
244
+ #
245
+ # @return [Symbol] The dictionary type
246
+ def type
247
+ self.class.name.split("::").last.gsub(/(.)([A-Z])/, '\1_\2').downcase.to_sym
248
+ end
249
+
250
+ # Register this dictionary type.
251
+ #
252
+ # @param type_key [Symbol] The type key to register as
253
+ #
254
+ # @example Registering a custom dictionary type
255
+ # MyDictionary.register_type(:my_custom)
256
+ def self.register_type(type_key)
257
+ Kotoshu::Dictionary.register_type(type_key, self)
258
+ end
259
+
260
+ # Class-level registry for dictionary types.
261
+ #
262
+ # @return [Hash] Registry of type keys to classes
263
+ def self.registry
264
+ @registry ||= {}
265
+ end
266
+
267
+ # Load a dictionary by type.
268
+ #
269
+ # @param type [Symbol] The dictionary type
270
+ # @param args [Array] Arguments to pass to constructor
271
+ # @return [Base] The loaded dictionary
272
+ # @raise [ConfigurationError] If type is not registered
273
+ def self.load(type, *args)
274
+ klass = registry[type]
275
+ raise ConfigurationError, "Unknown dictionary type: #{type}" unless klass
276
+
277
+ klass.new(*args)
278
+ end
279
+ end
280
+
281
+ # Module-level registry for dictionary types.
282
+ #
283
+ # @return [Hash] Registry of type keys to classes
284
+ def self.registry
285
+ @registry ||= {}
286
+ end
287
+
288
+ # Reset the type registry. Used by specs that register throw-away
289
+ # dictionary classes; not part of the runtime contract.
290
+ #
291
+ # @return [void]
292
+ def self.reset_registry!
293
+ @registry = {}
294
+ end
295
+
296
+ # Register a dictionary type.
297
+ #
298
+ # @param type [Symbol] The type key
299
+ # @param klass [Class] The dictionary class
300
+ #
301
+ # @example Registering a custom dictionary type
302
+ # Dictionary.register_type(:my_custom, MyDictionary)
303
+ def self.register_type(type, klass)
304
+ @registry ||= {}
305
+ @registry[type] = klass
306
+ end
307
+
308
+ # Load a dictionary by type.
309
+ #
310
+ # @param type [Symbol] The dictionary type
311
+ # @param args [Array] Arguments to pass to constructor
312
+ # @return [Base] The loaded dictionary
313
+ #
314
+ # @example Loading a dictionary
315
+ # dict = Dictionary.load(:unix_words, "/usr/share/dict/words",
316
+ # language_code: "en-US")
317
+ def self.load(type, *args)
318
+ klass = registry[type]
319
+ raise ConfigurationError, "Unknown dictionary type: #{type}" unless klass
320
+
321
+ klass.new(*args)
322
+ end
323
+ end
324
+ end
@@ -0,0 +1,243 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Dictionary
5
+ # CSpell dictionary backend.
6
+ #
7
+ # This dictionary reads CSpell-formatted dictionary files (plain text .txt
8
+ # or compressed .trie files). CSpell is the spell checker used by VS Code.
9
+ #
10
+ # File format:
11
+ # - .txt: Plain text with one word per line, # comments supported
12
+ # - .trie: Compressed trie format (DAFSA - Deterministic Acyclic Finite State Automaton)
13
+ #
14
+ # @example Creating from a text file
15
+ # dict = CSpell.new("words.txt", language_code: "en-US")
16
+ # dict.lookup?("hello") # => true
17
+ #
18
+ # @example Creating from a trie file
19
+ # dict = CSpell.new("words.trie", language_code: "en")
20
+ class CSpell < Base
21
+ # @return [String] The path to the dictionary file
22
+ attr_reader :path
23
+
24
+ # @return [Boolean] Whether lookups are case-sensitive
25
+ attr_reader :case_sensitive
26
+
27
+ # @return [Core::Trie::Trie] The trie data structure
28
+ attr_reader :trie
29
+
30
+ # Create a new CSpell dictionary.
31
+ #
32
+ # .trie). Nil when constructing from in-memory +words:+.
33
+ # @param language_code [String] The language code
34
+ # @param locale [String, nil] The locale (optional)
35
+ # @param case_sensitive [Boolean] Whether lookups are case-sensitive
36
+ # @param metadata [Hash] Additional metadata (optional)
37
+ # @param words [Array<String>, nil] Initial word list (bypasses
38
+ # file loading; used by .from_words).
39
+ def initialize(path = nil, language_code:, locale: nil, case_sensitive: false,
40
+ metadata: {}, words: nil)
41
+ super(language_code, locale: locale, metadata: metadata)
42
+
43
+ @path = path ? File.expand_path(path) : nil
44
+ @case_sensitive = case_sensitive
45
+
46
+ if words
47
+ normalized = words.map { |w| case_sensitive ? w : w.downcase }.uniq
48
+ @trie = Core::Trie::Builder.from_array(normalized)
49
+ else
50
+ raise DictionaryNotFoundError, @path unless File.exist?(@path)
51
+
52
+ @trie = if @path.end_with?(".trie")
53
+ load_trie_file(@path)
54
+ else
55
+ load_text_file(@path)
56
+ end
57
+ end
58
+
59
+ # Register this dictionary type
60
+ self.class.register_type(:cspell) unless Dictionary.registry.key?(:cspell)
61
+ end
62
+
63
+ # Check if a word exists in the dictionary.
64
+ #
65
+ # @param word [String] The word to look up
66
+ # @return [Boolean] True if the word exists
67
+ def lookup(word)
68
+ return false if word.nil? || word.empty?
69
+
70
+ lookup_word = @case_sensitive ? word : word.downcase
71
+ @trie.lookup(lookup_word)
72
+ end
73
+
74
+ # Check if the dictionary has words with a prefix.
75
+ #
76
+ # @param prefix [String] The prefix
77
+ # @return [Boolean] True if words exist with the prefix
78
+ def has_prefix?(prefix)
79
+ return false if prefix.nil? || prefix.empty?
80
+
81
+ lookup_prefix = @case_sensitive ? prefix : prefix.downcase
82
+ @trie.has_prefix?(lookup_prefix)
83
+ end
84
+
85
+ # Generate spelling suggestions.
86
+ #
87
+ # Uses trie walk to find similar words.
88
+ #
89
+ # @param word [String] The misspelled word
90
+ # @param max_suggestions [Integer] Maximum suggestions
91
+ # @return [Array<String>] List of suggested words
92
+ def suggest(word, max_suggestions: 10)
93
+ return [] if word.nil? || word.empty?
94
+
95
+ lookup_word = @case_sensitive ? word : word.downcase
96
+
97
+ # First try prefix-based suggestions
98
+ prefix_suggestions = @trie.suggestions(lookup_word, max_results: max_suggestions)
99
+
100
+ # If we have enough prefix suggestions, return them
101
+ return prefix_suggestions if prefix_suggestions.length >= max_suggestions
102
+
103
+ # Otherwise, use edit distance for more suggestions
104
+ all_words = @trie.all_words
105
+ candidates = all_words.select do |w|
106
+ w.length >= lookup_word.length - 2 &&
107
+ w.length <= lookup_word.length + 2
108
+ end
109
+
110
+ # Calculate edit distances
111
+ results = candidates.map do |dict_word|
112
+ dist = edit_distance(lookup_word, dict_word)
113
+ [dict_word, dist]
114
+ end.select { |_, dist| dist.positive? && dist <= 2 }
115
+ .sort_by { |_, dist| dist }
116
+ .first(max_suggestions - prefix_suggestions.length)
117
+ .map(&:first)
118
+
119
+ # Combine both sets
120
+ (prefix_suggestions + results).uniq.first(max_suggestions)
121
+ end
122
+
123
+ # Add a word to the dictionary.
124
+ #
125
+ # @param word [String] The word to add
126
+ # @param flags [Array<String>] Flags (ignored for CSpell)
127
+ # @return [Boolean] Always false — CSpell dictionaries are loaded
128
+ # into a frozen Trie (see Core::Trie::Builder#build) and cannot
129
+ # be mutated after load. The Trie itself rejects inserts with
130
+ # +FrozenError+; this method short-circuits so callers see the
131
+ # same "no" answer they would from {#remove_word}.
132
+ def add_word(word, flags: [])
133
+ false
134
+ end
135
+
136
+ # Remove a word from the dictionary.
137
+ #
138
+ # @param _word [String] The word to remove (unused — the trie
139
+ # does not support removal)
140
+ # @return [Boolean] Always false; trie dictionaries are immutable
141
+ # @note CSpell dictionaries are typically immutable after loading
142
+ def remove_word(_word)
143
+ # Trie doesn't support removal easily
144
+ # Would need to rebuild the trie
145
+ false
146
+ end
147
+
148
+ # Get all words in the dictionary.
149
+ #
150
+ # @return [Array<String>] All words
151
+ def words
152
+ @trie.all_words
153
+ end
154
+
155
+ # Get words with a prefix.
156
+ #
157
+ # @param prefix [String] The prefix
158
+ # @return [Array<String>] Words with the prefix
159
+ def words_with_prefix(prefix)
160
+ return [] if prefix.nil? || prefix.empty?
161
+
162
+ lookup_prefix = @case_sensitive ? prefix : prefix.downcase
163
+ @trie.words_with_prefix(lookup_prefix)
164
+ end
165
+
166
+ # Create a dictionary from an array of words.
167
+ #
168
+ # @param words [Array<String>] The words
169
+ # @param language_code [String] The language code
170
+ # @param locale [String, nil] The locale (optional)
171
+ # @param case_sensitive [Boolean] Whether lookups are case-sensitive
172
+ # @return [CSpell] New dictionary
173
+ #
174
+ # @example
175
+ # dict = CSpell.from_words(%w[hello world test], language_code: "en")
176
+ def self.from_words(words, language_code:, locale: nil, case_sensitive: false)
177
+ new(words: words, language_code: language_code,
178
+ locale: locale, case_sensitive: case_sensitive)
179
+ end
180
+
181
+ private
182
+
183
+ # Load a text dictionary file.
184
+ #
185
+ # @param path [String] The file path
186
+ # @return [Core::Trie::Trie] The loaded trie
187
+ def load_text_file(path)
188
+ words = File.foreach(path, chomp: true)
189
+ .reject { |line| line.empty? || line.strip.empty? || line.strip.start_with?("#") }
190
+ .map(&:strip)
191
+ .map { |word| @case_sensitive ? word : word.downcase }
192
+ .uniq
193
+
194
+ Core::Trie::Builder.from_array(words)
195
+ end
196
+
197
+ # Load a compressed trie file.
198
+ #
199
+ # @param path [String] The file path
200
+ # @return [Core::Trie::Trie] The loaded trie
201
+ #
202
+ # @note For now, this falls back to treating the file as text.
203
+ # Full .trie format support would require implementing DAFSA decompression.
204
+ def load_trie_file(path)
205
+ # For now, treat as text file
206
+ # Full implementation would parse the CSpell .trie format
207
+ # which uses DAFSA (Deterministic Acyclic Finite State Automaton) compression
208
+ load_text_file(path)
209
+ end
210
+
211
+ # Calculate Levenshtein edit distance.
212
+ #
213
+ # @param str1 [String] First string
214
+ # @param str2 [String] Second string
215
+ # @return [Integer] Edit distance
216
+ def edit_distance(str1, str2)
217
+ return str2.length if str1.empty?
218
+ return str1.length if str2.empty?
219
+
220
+ # Use smaller string for inner loop
221
+ str1, str2 = str2, str1 if str1.length > str2.length
222
+
223
+ previous = (0..str1.length).to_a
224
+
225
+ str2.each_char.with_index do |char2, j|
226
+ current = [j + 1]
227
+
228
+ str1.each_char.with_index do |char1, i|
229
+ insert_cost = current[i] + 1
230
+ delete_cost = previous[i + 1] + 1
231
+ substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
232
+
233
+ current << [insert_cost, delete_cost, substitute_cost].min
234
+ end
235
+
236
+ previous = current
237
+ end
238
+
239
+ previous.last
240
+ end
241
+ end
242
+ end
243
+ end