kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "set"
5
+
6
+ module Kotoshu
7
+ module Suggestions
8
+ # The frozen Kelly frequency tiers for English, read once per
9
+ # process from the generated JSON beside this file (provenance and
10
+ # the upstream sha256 live inside it). Both engines ship identical
11
+ # membership - the Rust side embeds kotoshu-rs
12
+ # kotoshu/src/suggest/frequency_data.rs, this file is generated
13
+ # from those tables - so a cache-cold Ruby machine ranks
14
+ # suggestions exactly like the frozen conformance vectors
15
+ # (plan 119). Regeneration goes through
16
+ # scripts/generate_frozen_tiers.rb, never by hand.
17
+ module FrozenTiers
18
+ DATA_PATH = File.expand_path("frozen_kelly/en.json", __dir__)
19
+
20
+ # The tiers for +language_code+ in the Set shape the suggestion
21
+ # strategies consume, or nil when no frozen table exists for the
22
+ # language (the caller falls back further).
23
+ #
24
+ # @param language_code [String] ISO 639-1 code
25
+ # @return [Hash{Symbol => Set}, nil]
26
+ def self.tiers_for(language_code)
27
+ return nil unless language_code.to_s == "en"
28
+
29
+ @tiers ||= begin
30
+ raw = JSON.parse(File.read(DATA_PATH), symbolize_names: true)
31
+ raw.fetch(:tiers).to_h do |name, words|
32
+ [name.to_sym, Set.new(words)]
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Bundled data resources (common-word lists, frequency tables).
5
+ module Data
6
+ autoload :CommonWordsLoader, "kotoshu/data/common_words_loader"
7
+ end
8
+ end
@@ -0,0 +1,188 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module DataStructures
5
+ # Bloom filter - probabilistic data structure for fast membership testing.
6
+ #
7
+ # A Bloom filter is a space-efficient probabilistic data structure that
8
+ # is used to test whether an element is a member of a set. False positive
9
+ # matches are possible, but false negatives are not.
10
+ #
11
+ # @example Basic usage
12
+ # filter = BloomFilter.new
13
+ # filter.add("hello")
14
+ # filter.include?("hello") # => true (definitely in set)
15
+ # filter.include?("world") # => false (probably not in set)
16
+ #
17
+ # @see https://en.wikipedia.org/wiki/Bloom_filter Bloom filter Wikipedia
18
+ class BloomFilter
19
+ # Default false positive rate (1%)
20
+ DEFAULT_FALSE_POSITIVE_RATE = 0.01
21
+
22
+ # Default expected number of elements
23
+ DEFAULT_EXPECTED_SIZE = 10_000
24
+
25
+ # @return [Integer] Size of the bit array
26
+ attr_reader :size
27
+
28
+ # @return [Integer] Number of hash functions
29
+ attr_reader :hash_count
30
+
31
+ # @return [Integer] Number of items added
32
+ attr_reader :item_count
33
+
34
+ # Read-only view of the internal bit array. Exposed so {#merge}
35
+ # can OR bits across two filters without poking at another
36
+ # instance's ivars.
37
+ # @return [Array<Boolean>] bit array, length == +size+
38
+ attr_reader :bits
39
+
40
+ # Create a new Bloom filter.
41
+ #
42
+ # @param expected_size [Integer] Expected number of elements (default: 10_000)
43
+ # @param false_positive_rate [Float] Desired false positive rate (default: 0.01)
44
+ # @param case_sensitive [Boolean] Whether lookups are case-sensitive (default: false)
45
+ def initialize(expected_size: DEFAULT_EXPECTED_SIZE,
46
+ false_positive_rate: DEFAULT_FALSE_POSITIVE_RATE,
47
+ case_sensitive: false)
48
+ @case_sensitive = case_sensitive
49
+ @item_count = 0
50
+
51
+ # Calculate optimal size and hash count
52
+ # m = -n * ln(p) / (ln(2)^2)
53
+ # k = (m/n) * ln(2)
54
+ @size = calculate_size(expected_size, false_positive_rate)
55
+ @hash_count = calculate_hash_count(@size, expected_size)
56
+
57
+ # Initialize bit array
58
+ @bits = Array.new(@size, false)
59
+ end
60
+
61
+ # Add an element to the filter.
62
+ #
63
+ # @param item [String] The item to add
64
+ # @return [self] Self for chaining
65
+ def add(item)
66
+ normalized_item = normalize_item(item)
67
+
68
+ @hash_count.times do |i|
69
+ index = hash_index(normalized_item, i)
70
+ @bits[index] = true
71
+ end
72
+
73
+ @item_count += 1
74
+ self
75
+ end
76
+
77
+ # Check if an element might be in the filter.
78
+ #
79
+ # Note: Returns false if the element is definitely NOT in the filter.
80
+ # Returns true if the element is PROBABLY in the filter (may be false positive).
81
+ #
82
+ # @param item [String] The item to check
83
+ # @return [Boolean] True if possibly in filter, false if definitely not
84
+ def include?(item)
85
+ normalized_item = normalize_item(item)
86
+
87
+ @hash_count.times do |i|
88
+ index = hash_index(normalized_item, i)
89
+ return false unless @bits[index]
90
+ end
91
+
92
+ true
93
+ end
94
+ alias include? include?
95
+ alias might_include? include?
96
+
97
+ # Merge another bloom filter into this one.
98
+ #
99
+ # @param other [BloomFilter] Another bloom filter with same parameters
100
+ # @return [self] Self for chaining
101
+ def merge(other)
102
+ raise ArgumentError, "Cannot merge filters with different sizes" unless other.size == @size
103
+ raise ArgumentError, "Cannot merge filters with different hash counts" unless other.hash_count == @hash_count
104
+
105
+ @size.times do |i|
106
+ @bits[i] = @bits[i] || other.bits[i]
107
+ end
108
+
109
+ @item_count += other.item_count
110
+ self
111
+ end
112
+
113
+ # Clear all elements from the filter.
114
+ #
115
+ # @return [self] Self for chaining
116
+ def clear
117
+ @bits = Array.new(@size, false)
118
+ @item_count = 0
119
+ self
120
+ end
121
+
122
+ # Get filter statistics.
123
+ #
124
+ # @return [Hash] Statistics including :size, :hash_count, :item_count
125
+ def stats
126
+ {
127
+ size: @size,
128
+ hash_count: @hash_count,
129
+ item_count: @item_count
130
+ }
131
+ end
132
+
133
+ private
134
+
135
+ # Normalize item for consistent hashing.
136
+ #
137
+ # @param item [String] The item to normalize
138
+ # @return [String] Normalized item
139
+ def normalize_item(item)
140
+ @case_sensitive ? item.to_s : item.to_s.downcase
141
+ end
142
+
143
+ # Calculate optimal bit array size.
144
+ #
145
+ # @param n [Integer] Expected number of elements
146
+ # @param p [Float] False positive rate
147
+ # @return [Integer] Optimal size in bits
148
+ def calculate_size(n, p)
149
+ # m = -n * ln(p) / (ln(2)^2)
150
+ m = (-n * Math.log(p)) / (Math.log(2)**2)
151
+ m.ceil.to_i
152
+ end
153
+
154
+ # Calculate optimal number of hash functions.
155
+ #
156
+ # @param m [Integer] Size of bit array
157
+ # @param n [Integer] Expected number of elements
158
+ # @return [Integer] Optimal number of hash functions
159
+ def calculate_hash_count(m, n)
160
+ # k = (m/n) * ln(2)
161
+ k = (m.to_f / n) * Math.log(2)
162
+ [1, k.ceil.to_i].max # At least 1 hash function
163
+ end
164
+
165
+ # Calculate hash index for item with seed.
166
+ #
167
+ # Uses double hashing (Kirsch-Mitzenmacher):
168
+ # h_i(x) = (h1(x) + i * h2(x)) mod m
169
+ #
170
+ # The scheme requires h1 and h2 to be *independent* hash
171
+ # functions. Using `item.hash` for both (as an earlier version
172
+ # did) makes them linearly dependent and inflates the false-
173
+ # positive rate well above the configured target — observed
174
+ # ~6–16% versus the theoretical 1% on a filter sized for
175
+ # 1% FPR. Deriving h2 from a salted copy of the item restores
176
+ # independence and brings the observed rate back into line.
177
+ #
178
+ # @param item [String] The item to hash
179
+ # @param seed [Integer] Hash function index
180
+ # @return [Integer] Bit array index
181
+ def hash_index(item, seed)
182
+ hash1 = item.hash
183
+ hash2 = "#{item}#bloom#salt".hash
184
+ (hash1 + (seed * hash2.abs)) % @size
185
+ end
186
+ end
187
+ end
188
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Pure data structure primitives (bloom filter, etc.).
5
+ module DataStructures
6
+ autoload :BloomFilter, "kotoshu/data_structures/bloom_filter"
7
+ end
8
+ end
@@ -0,0 +1,146 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Debug
5
+ # Debug logger for detailed spellchecking information.
6
+ #
7
+ # Provides structured logging for lookup operations, suggestion generation,
8
+ # cache behavior, and decision trees.
9
+ class Logger
10
+ # Log levels
11
+ LEVELS = %i[info verbose trace].freeze
12
+
13
+ attr_reader :output, :level
14
+
15
+ # Create a new debug logger.
16
+ #
17
+ # @param output [IO] Output stream (default: $stderr)
18
+ # @param level [Symbol] Log level (:info, :verbose, :trace)
19
+ def initialize(output: $stderr, level: :info)
20
+ @output = output
21
+ @level = level
22
+ @indent = 0
23
+ end
24
+
25
+ # Log lookup operation.
26
+ #
27
+ # @param word [String] The word being looked up
28
+ # @param result [Boolean] The lookup result
29
+ # @param time [Float] Time taken in milliseconds
30
+ def debug_lookup(word, result:, time:)
31
+ return unless should_log?(:info)
32
+
33
+ status = result ? "✓" : "✗"
34
+ output.puts "DEBUG: lookup #{status} \"#{word}\" - #{time.round(3)}ms"
35
+ end
36
+
37
+ # Log suggestion generation.
38
+ #
39
+ # @param word [String] The input word
40
+ # @param suggestions [Array] Generated suggestions
41
+ # @param time [Float] Time taken in milliseconds
42
+ def debug_suggestions(word, suggestions:, time:)
43
+ return unless should_log?(:verbose)
44
+
45
+ output.puts "DEBUG: suggestions for \"#{word}\" (#{time.round(3)}ms)"
46
+
47
+ return unless should_log?(:trace)
48
+
49
+ @indent += 2
50
+ suggestions.each do |suggestion|
51
+ dist = suggestion.distance
52
+ conf = suggestion.confidence
53
+ source = suggestion.source
54
+ output.puts "#{' ' * @indent}#{suggestion.word} (dist: #{dist}, conf: #{conf.round(2)}, src: #{source})"
55
+ end
56
+ @indent -= 2
57
+ end
58
+
59
+ # Log cache operation.
60
+ #
61
+ # @param cache_type [String] Type of cache
62
+ # @param key [String] The cache key
63
+ # @param hit [Boolean] True if cache hit
64
+ def debug_cache(cache_type, key, hit:)
65
+ return unless should_log?(:trace)
66
+
67
+ status = hit ? "HIT" : "MISS"
68
+ output.puts "DEBUG: cache #{cache_type.upcase} #{status} \"#{key}\""
69
+ end
70
+
71
+ # Log decision tree.
72
+ #
73
+ # @param word [String] The input word
74
+ # @param decisions [Array] Array of decision nodes
75
+ def debug_decision_tree(word, decisions:)
76
+ return unless should_log?(:trace)
77
+
78
+ output.puts "DEBUG: decision tree for \"#{word}\""
79
+ @indent += 2
80
+ print_decisions(decisions)
81
+ @indent -= 2
82
+ end
83
+
84
+ # Log info message.
85
+ #
86
+ # @param message [String] The message
87
+ def info(message)
88
+ return unless should_log?(:info)
89
+
90
+ output.puts "DEBUG: #{message}"
91
+ end
92
+
93
+ # Log verbose message.
94
+ #
95
+ # @param message [String] The message
96
+ def verbose(message)
97
+ return unless should_log?(:verbose)
98
+
99
+ output.puts "DEBUG: #{message}"
100
+ end
101
+
102
+ # Log trace message.
103
+ #
104
+ # @param message [String] The message
105
+ def trace(message)
106
+ return unless should_log?(:trace)
107
+
108
+ output.puts "DEBUG: #{message}"
109
+ end
110
+
111
+ private
112
+
113
+ # Check if should log at current level.
114
+ #
115
+ # @param required_level [Symbol] Required level
116
+ # @return [Boolean] True if should log
117
+ def should_log?(required_level)
118
+ LEVELS.index(required_level) <= LEVELS.index(@level)
119
+ end
120
+
121
+ # Print decisions tree.
122
+ #
123
+ # @param decisions [Array] Decision nodes
124
+ def print_decisions(decisions, index = 0)
125
+ decisions.each do |decision|
126
+ prefix = "#{' ' * @indent}#{index}. "
127
+ output.puts "#{prefix}#{decision[:description]}"
128
+
129
+ if should_log?(:trace) && decision[:details]
130
+ @indent += 2
131
+ decision[:details].each do |key, value|
132
+ output.puts "#{' ' * @indent}#{key}: #{value}"
133
+ end
134
+ @indent -= 2
135
+ end
136
+
137
+ next unless decision[:children] && !decision[:children].empty?
138
+
139
+ @indent += 2
140
+ print_decisions(decision[:children], index + 1)
141
+ @indent -= 2
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Debug mode for detailed spellchecking insights.
5
+ #
6
+ # When enabled, debug mode provides:
7
+ # - Lookup timing information
8
+ # - Suggestion scoring details
9
+ # - Decision tree visualization
10
+ # - Cache hit/miss tracking
11
+ # - Performance metrics
12
+ #
13
+ # @example Enable debug mode
14
+ # Kotoshu::Debug.enable
15
+ # Kotoshu.correct?("hello")
16
+ # # Output: DEBUG: lookup "hello" - 0.001ms
17
+ #
18
+ # @example Disable debug mode
19
+ # Kotoshu::Debug.disable
20
+ module Debug
21
+ autoload :Logger, "kotoshu/debug_logger"
22
+
23
+ class << self
24
+ # Enable debug mode.
25
+ #
26
+ # @param output [IO] Output stream (default: $stderr)
27
+ # @param level [Symbol] Debug level (:info, :verbose, :trace)
28
+ def enable(output: $stderr, level: :info)
29
+ @enabled = true
30
+ @output = output
31
+ @level = level
32
+ @logger = Debug::Logger.new(output: output, level: level)
33
+ end
34
+
35
+ # Disable debug mode.
36
+ def disable
37
+ @enabled = false
38
+ @logger = nil
39
+ end
40
+
41
+ # Check if debug mode is enabled.
42
+ #
43
+ # @return [Boolean] True if enabled
44
+ def enabled?
45
+ @enabled ||= false
46
+ end
47
+
48
+ # Get the debug logger.
49
+ #
50
+ # @return [Debug::Logger, nil] The logger instance
51
+ attr_reader :logger
52
+
53
+ # Log a lookup operation.
54
+ #
55
+ # @param word [String] The word being looked up
56
+ # @param result [Boolean] The lookup result
57
+ # @param time [Float] Time taken in milliseconds
58
+ def log_lookup(word, result:, time:)
59
+ return unless enabled?
60
+
61
+ logger&.debug_lookup(word, result: result, time: time)
62
+ end
63
+
64
+ # Log a suggestion generation.
65
+ #
66
+ # @param word [String] The input word
67
+ # @param suggestions [Array] Generated suggestions
68
+ # @param time [Float] Time taken in milliseconds
69
+ def log_suggestions(word, suggestions:, time:)
70
+ return unless enabled?
71
+
72
+ logger&.debug_suggestions(word, suggestions: suggestions, time: time)
73
+ end
74
+
75
+ # Log a cache hit/miss.
76
+ #
77
+ # @param cache_type [String] Type of cache (lookup, suggestion)
78
+ # @param key [String] The cache key
79
+ # @param hit [Boolean] True if cache hit
80
+ def log_cache(cache_type, key, hit:)
81
+ return unless enabled?
82
+
83
+ logger&.debug_cache(cache_type, key, hit: hit)
84
+ end
85
+
86
+ # Log a decision tree.
87
+ #
88
+ # @param word [String] The input word
89
+ # @param decisions [Array] Array of decision nodes
90
+ def log_decision_tree(word, decisions:)
91
+ return unless enabled?
92
+
93
+ logger&.debug_decision_tree(word, decisions: decisions)
94
+ end
95
+
96
+ # Start a timing context.
97
+ #
98
+ # @yield Block to time
99
+ # @return [Object] Block result
100
+ def time(label)
101
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
102
+ result = yield
103
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
104
+
105
+ logger&.info("#{label}: #{elapsed.round(3)}ms")
106
+ result
107
+ end
108
+
109
+ # Measure and log a lookup.
110
+ #
111
+ # @yield Block that performs the lookup
112
+ # @return [Object] Block result
113
+ def measure_lookup(word)
114
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
115
+ result = yield
116
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
117
+
118
+ log_lookup(word, result: result, time: elapsed)
119
+ result
120
+ end
121
+
122
+ # Measure and log suggestions.
123
+ #
124
+ # @yield Block that generates suggestions
125
+ # @return [Object] Block result
126
+ def measure_suggestions(word)
127
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
128
+ result = yield
129
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
130
+
131
+ log_suggestions(word, suggestions: result, time: elapsed)
132
+ result
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Sensible defaults for Kotoshu configuration.
5
+ #
6
+ # Provides auto-detection of system dictionaries and fallback
7
+ # to bundled dictionaries, ensuring Kotoshu works out of the box.
8
+ module Defaults
9
+ # Standard system dictionary paths.
10
+ SYSTEM_DICTIONARY_PATHS = [
11
+ "/usr/share/dict/words",
12
+ "/usr/share/dict/web2",
13
+ "/usr/share/dict/web2a",
14
+ "/usr/dict/words"
15
+ ].freeze
16
+
17
+ # Bundled dictionary paths (relative to gem root).
18
+ BUNDLED_DICTIONARY_PATHS = [
19
+ "dictionaries/unix_words/words",
20
+ "dictionaries/unix_words/web2",
21
+ "dictionaries/unix_words/web2a"
22
+ ].freeze
23
+
24
+ class << self
25
+ # Detect system dictionary.
26
+ #
27
+ # @return [String, nil] Path to system dictionary or nil
28
+ def detect_system_dictionary
29
+ SYSTEM_DICTIONARY_PATHS.find do |path|
30
+ File.exist?(path)
31
+ end
32
+ end
33
+
34
+ # Get path to bundled dictionary.
35
+ #
36
+ # @return [String, nil] Path to bundled dictionary or nil
37
+ def bundled_dictionary_path
38
+ BUNDLED_DICTIONARY_PATHS.find do |path|
39
+ full_path = File.expand_path("../../#{path}", __dir__)
40
+ File.exist?(full_path)
41
+ end
42
+ end
43
+
44
+ # Get default dictionary.
45
+ #
46
+ # Tries system dictionary first, then bundled dictionary,
47
+ # then falls back to an empty custom dictionary.
48
+ #
49
+ # @return [Dictionary::Base] A working dictionary
50
+ def default_dictionary
51
+ # Try system dictionary
52
+ system_path = detect_system_dictionary
53
+ return Dictionary::PlainText.new(system_path, language_code: "en") if system_path
54
+
55
+ # Try bundled dictionary
56
+ bundled_path = bundled_dictionary_path
57
+ if bundled_path
58
+ full_path = File.expand_path("../../#{bundled_path}", __dir__)
59
+ return Dictionary::PlainText.new(full_path, language_code: "en")
60
+ end
61
+
62
+ # Fall back to minimal dictionary with common words
63
+ Dictionary::PlainText.from_words(
64
+ %w[the and for are but not you all any can had has him his how her its now our our was what],
65
+ language_code: "en"
66
+ )
67
+ end
68
+
69
+ # Configure Kotoshu with sensible defaults.
70
+ #
71
+ # @return [Configuration] The configured instance
72
+ def configure
73
+ default_dictionary
74
+
75
+ Configuration::Builder.build do |c|
76
+ c.dictionary_type = :plain_text
77
+ c.language = "en-US"
78
+ c.max_suggestions = 10
79
+ c.case_sensitive = false
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end