kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Per-language implementations (English, German, French, etc.).
5
+ #
6
+ # Sibling module to {Language} (the framework). Each per-language file
7
+ # calls Language::Registry.register at load time; {Language::Registry.ensure_languages_loaded}
8
+ # triggers these autoloads on first registry access.
9
+ module Languages
10
+ autoload :Arabic, "kotoshu/languages/ar/language"
11
+ autoload :English, "kotoshu/languages/en/language"
12
+ autoload :French, "kotoshu/languages/fr/language"
13
+ autoload :German, "kotoshu/languages/de/language"
14
+ autoload :Hebrew, "kotoshu/languages/he/language"
15
+ autoload :Japanese, "kotoshu/languages/ja/language"
16
+ autoload :Persian, "kotoshu/languages/fa/language"
17
+ autoload :Portuguese, "kotoshu/languages/pt/language"
18
+ autoload :Russian, "kotoshu/languages/ru/language"
19
+ autoload :Spanish, "kotoshu/languages/es/language"
20
+
21
+ # Shared base for the thin wave-1 modules (plan 84).
22
+ autoload :LatinBase, "kotoshu/languages/latin_base"
23
+ # Wave-1 modules (plan 84, Track B): ca cs da el hu it nl pl ro
24
+ # sv tr uk vi.
25
+ autoload :Catalan, "kotoshu/languages/ca/language"
26
+ autoload :Czech, "kotoshu/languages/cs/language"
27
+ autoload :Danish, "kotoshu/languages/da/language"
28
+ autoload :Greek, "kotoshu/languages/el/language"
29
+ autoload :Hungarian, "kotoshu/languages/hu/language"
30
+ autoload :Italian, "kotoshu/languages/it/language"
31
+ autoload :Dutch, "kotoshu/languages/nl/language"
32
+ autoload :Polish, "kotoshu/languages/pl/language"
33
+ autoload :Romanian, "kotoshu/languages/ro/language"
34
+ autoload :Swedish, "kotoshu/languages/sv/language"
35
+ autoload :NorwegianBokmal, "kotoshu/languages/nb/language"
36
+ autoload :Turkish, "kotoshu/languages/tr/language"
37
+ autoload :Ukrainian, "kotoshu/languages/uk/language"
38
+ autoload :Vietnamese, "kotoshu/languages/vi/language"
39
+ # Batch-3 modules (plan 100): id hr sl sk et lt lv (Latin) and
40
+ # bg sr (Cyrillic). The pre-existing ar fa he modules become
41
+ # full-feature in this batch (layout + downloadable dictionary).
42
+ autoload :Bulgarian, "kotoshu/languages/bg/language"
43
+ autoload :Croatian, "kotoshu/languages/hr/language"
44
+ autoload :Estonian, "kotoshu/languages/et/language"
45
+ autoload :Indonesian, "kotoshu/languages/id/language"
46
+ autoload :Latvian, "kotoshu/languages/lv/language"
47
+ autoload :Lithuanian, "kotoshu/languages/lt/language"
48
+ autoload :Serbian, "kotoshu/languages/sr/language"
49
+ autoload :Slovak, "kotoshu/languages/sk/language"
50
+ autoload :Slovenian, "kotoshu/languages/sl/language"
51
+ # Nynorsk module (plan 107): wired as full feature while basic
52
+ # support opened the staged manifest (dictionary + model staged).
53
+ autoload :NorwegianNynorsk, "kotoshu/languages/nn/language"
54
+ # ko and ne modules (plan 108): the last big-population languages
55
+ # — Hangul eojeol tokenizer + Dubeolsik grid, Devanagari grapheme
56
+ # tokenizer + InScript grid.
57
+ autoload :Korean, "kotoshu/languages/ko/language"
58
+ autoload :Nepali, "kotoshu/languages/ne/language"
59
+ # Serbian Latin module (plan 110): full-feature wiring over the
60
+ # distinct staged sr-Latn dictionary — Latin tokenizer, the
61
+ # South-Slavic QWERTZ grid shared with hr and sl.
62
+ autoload :SerbianLatin, "kotoshu/languages/sr-Latn/language"
63
+ end
64
+ end
@@ -0,0 +1,238 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Metrics
5
+ # Thread-safe metrics collector.
6
+ #
7
+ # Tracks performance metrics for spellchecking operations:
8
+ # - Lookup counts and timing
9
+ # - Cache hit/miss rates
10
+ # - Suggestion generation stats
11
+ #
12
+ # @example
13
+ # collector = Kotoshu::Metrics::Collector.new
14
+ # collector.record_lookup("hello", result: true, time: 0.5)
15
+ # collector.stats
16
+ # # => { lookups: 1, correct_lookups: 1, avg_lookup_time: 0.5, ... }
17
+ class Collector
18
+ # Initialize a new collector.
19
+ def initialize
20
+ @mutex = Mutex.new
21
+ reset
22
+ end
23
+
24
+ # Record a lookup operation.
25
+ #
26
+ # @param _word [String] The word being looked up (not retained)
27
+ # @param result [Boolean] The lookup result
28
+ # @param time [Float] Time taken in milliseconds
29
+ def record_lookup(_word, result:, time:)
30
+ @mutex.synchronize do
31
+ @metrics[:lookups] += 1
32
+ @metrics[:correct_lookups] += 1 if result
33
+ @metrics[:misspelled_lookups] += 1 unless result
34
+
35
+ @metrics[:lookup_times] << time
36
+ end
37
+ end
38
+
39
+ # Record a cache operation.
40
+ #
41
+ # @param cache_type [String] Type of cache (lookup, suggestion)
42
+ # @param hit [Boolean] True if cache hit
43
+ def record_cache(cache_type, hit:)
44
+ @mutex.synchronize do
45
+ key = :"#{cache_type}_cache_hits"
46
+ miss_key = :"#{cache_type}_cache_misses"
47
+
48
+ if hit
49
+ @metrics[key] += 1
50
+ else
51
+ @metrics[miss_key] += 1
52
+ end
53
+ end
54
+ end
55
+
56
+ # Record suggestion generation.
57
+ #
58
+ # @param _word [String] The input word (not retained)
59
+ # @param count [Integer] Number of suggestions generated
60
+ # @param time [Float] Time taken in milliseconds
61
+ def record_suggestions(_word, count:, time:)
62
+ @mutex.synchronize do
63
+ @metrics[:suggestion_requests] += 1
64
+ @metrics[:suggestions_generated] += count
65
+
66
+ @metrics[:suggestion_times] << time
67
+ end
68
+ end
69
+
70
+ # Record a skipped semantic rerank (confidence cascade skip).
71
+ def record_semantic_cascade_skip
72
+ @mutex.synchronize do
73
+ @metrics[:semantic_cascade_skips] += 1
74
+ end
75
+ end
76
+
77
+ # Get current metrics statistics.
78
+ #
79
+ # @return [Hash] Current statistics with computed averages
80
+ def stats
81
+ @mutex.synchronize do
82
+ calculate_stats
83
+ end
84
+ end
85
+
86
+ # Reset all metrics.
87
+ def reset
88
+ @mutex.synchronize do
89
+ @metrics = {
90
+ lookups: 0,
91
+ correct_lookups: 0,
92
+ misspelled_lookups: 0,
93
+ lookup_times: [],
94
+
95
+ lookup_cache_hits: 0,
96
+ lookup_cache_misses: 0,
97
+ suggestion_cache_hits: 0,
98
+ suggestion_cache_misses: 0,
99
+
100
+ suggestion_requests: 0,
101
+ suggestions_generated: 0,
102
+ suggestion_times: [],
103
+
104
+ semantic_cascade_skips: 0,
105
+
106
+ started_at: Time.now
107
+ }
108
+ end
109
+ end
110
+
111
+ # Export metrics in StatsD format.
112
+ #
113
+ # @return [String] StatsD protocol lines
114
+ def to_statsd
115
+ s = stats
116
+ prefix = "kotoshu"
117
+
118
+ lines = []
119
+ lines << "#{prefix}.lookups:#{s[:lookups]}|c"
120
+ lines << "#{prefix}.correct_lookups:#{s[:correct_lookups]}|c"
121
+ lines << "#{prefix}.misspelled_lookups:#{s[:misspelled_lookups]}|c"
122
+ lines << "#{prefix}.avg_lookup_time:#{s[:avg_lookup_time]}|ms"
123
+ lines << "#{prefix}.lookup_cache_hits:#{s[:lookup_cache_hits]}|c"
124
+ lines << "#{prefix}.lookup_cache_misses:#{s[:lookup_cache_misses]}|c"
125
+ lines << "#{prefix}.suggestion_requests:#{s[:suggestion_requests]}|c"
126
+ lines << "#{prefix}.suggestions_generated:#{s[:suggestions_generated]}|c"
127
+ lines << "#{prefix}.avg_suggestion_time:#{s[:avg_suggestion_time]}|ms"
128
+ lines << "#{prefix}.semantic_cascade_skips:#{s[:semantic_cascade_skips]}|c"
129
+
130
+ lines.join("\n")
131
+ end
132
+
133
+ # Export metrics in Prometheus exposition format.
134
+ #
135
+ # @return [String] Prometheus format
136
+ def to_prometheus
137
+ s = stats
138
+
139
+ lines = []
140
+ lines << "# HELP kotoshu_lookups Total number of word lookups"
141
+ lines << "# TYPE kotoshu_lookups counter"
142
+ lines << "kotoshu_lookups #{s[:lookups]}"
143
+
144
+ lines << "# HELP kotoshu_correct_lookups Number of correct word lookups"
145
+ lines << "# TYPE kotoshu_correct_lookups counter"
146
+ lines << "kotoshu_correct_lookups #{s[:correct_lookups]}"
147
+
148
+ lines << "# HELP kotoshu_misspelled_lookups Number of misspelled word lookups"
149
+ lines << "# TYPE kotoshu_misspelled_lookups counter"
150
+ lines << "kotoshu_misspelled_lookups #{s[:misspelled_lookups]}"
151
+
152
+ lines << "# HELP kotoshu_avg_lookup_time Average lookup time in milliseconds"
153
+ lines << "# TYPE kotoshu_avg_lookup_time gauge"
154
+ lines << "kotoshu_avg_lookup_time #{s[:avg_lookup_time]}"
155
+
156
+ lines << "# HELP kotoshu_lookup_cache_hits Number of lookup cache hits"
157
+ lines << "# TYPE kotoshu_lookup_cache_hits counter"
158
+ lines << "kotoshu_lookup_cache_hits #{s[:lookup_cache_hits]}"
159
+
160
+ lines << "# HELP kotoshu_lookup_cache_misses Number of lookup cache misses"
161
+ lines << "# TYPE kotoshu_lookup_cache_misses counter"
162
+ lines << "kotoshu_lookup_cache_misses #{s[:lookup_cache_misses]}"
163
+
164
+ lines << "# HELP kotoshu_suggestion_requests Number of suggestion requests"
165
+ lines << "# TYPE kotoshu_suggestion_requests counter"
166
+ lines << "kotoshu_suggestion_requests #{s[:suggestion_requests]}"
167
+
168
+ lines << "# HELP kotoshu_suggestions_generated Total number of suggestions generated"
169
+ lines << "# TYPE kotoshu_suggestions_generated counter"
170
+ lines << "kotoshu_suggestions_generated #{s[:suggestions_generated]}"
171
+
172
+ lines << "# HELP kotoshu_avg_suggestion_time Average suggestion generation time in milliseconds"
173
+ lines << "# TYPE kotoshu_avg_suggestion_time gauge"
174
+ lines << "kotoshu_avg_suggestion_time #{s[:avg_suggestion_time]}"
175
+
176
+ lines << "# HELP kotoshu_semantic_cascade_skips Number of semantic reranks skipped by the confidence cascade"
177
+ lines << "# TYPE kotoshu_semantic_cascade_skips counter"
178
+ lines << "kotoshu_semantic_cascade_skips #{s[:semantic_cascade_skips]}"
179
+
180
+ lines.join("\n")
181
+ end
182
+
183
+ private
184
+
185
+ # Calculate computed statistics.
186
+ #
187
+ # @return [Hash] Statistics with computed values
188
+ def calculate_stats
189
+ lookup_times = @metrics[:lookup_times]
190
+ suggestion_times = @metrics[:suggestion_times]
191
+
192
+ avg_lookup = lookup_times.empty? ? 0 : lookup_times.sum / lookup_times.size
193
+ avg_suggestion = suggestion_times.empty? ? 0 : suggestion_times.sum / suggestion_times.size
194
+
195
+ lookup_hit_rate = calculate_hit_rate(@metrics[:lookup_cache_hits], @metrics[:lookup_cache_misses])
196
+ suggestion_hit_rate = calculate_hit_rate(@metrics[:suggestion_cache_hits], @metrics[:suggestion_cache_misses])
197
+
198
+ {
199
+ lookups: @metrics[:lookups],
200
+ correct_lookups: @metrics[:correct_lookups],
201
+ misspelled_lookups: @metrics[:misspelled_lookups],
202
+ avg_lookup_time: avg_lookup.round(3),
203
+
204
+ lookup_cache_hits: @metrics[:lookup_cache_hits],
205
+ lookup_cache_misses: @metrics[:lookup_cache_misses],
206
+ lookup_cache_hit_rate: lookup_hit_rate,
207
+
208
+ suggestion_cache_hits: @metrics[:suggestion_cache_hits],
209
+ suggestion_cache_misses: @metrics[:suggestion_cache_misses],
210
+ suggestion_cache_hit_rate: suggestion_hit_rate,
211
+
212
+ suggestion_requests: @metrics[:suggestion_requests],
213
+ suggestions_generated: @metrics[:suggestions_generated],
214
+ avg_suggestions_per_request: if @metrics[:suggestion_requests].positive?
215
+ (@metrics[:suggestions_generated].to_f / @metrics[:suggestion_requests]).round(2)
216
+ else
217
+ 0
218
+ end,
219
+ avg_suggestion_time: avg_suggestion.round(3),
220
+
221
+ semantic_cascade_skips: @metrics[:semantic_cascade_skips],
222
+
223
+ uptime_seconds: (Time.now - @metrics[:started_at]).round(2)
224
+ }
225
+ end
226
+
227
+ # Calculate cache hit rate.
228
+ #
229
+ # @param hits [Integer] Number of hits
230
+ # @param misses [Integer] Number of misses
231
+ # @return [Float] Hit rate (0-1)
232
+ def calculate_hit_rate(hits, misses)
233
+ total = hits + misses
234
+ total.positive? ? (hits.to_f / total).round(4) : 0.0
235
+ end
236
+ end
237
+ end
238
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Metrics and instrumentation for Kotoshu.
5
+ #
6
+ # Provides thread-safe collection of performance metrics:
7
+ # - Lookup counts and timing
8
+ # - Cache hit/miss rates
9
+ # - Suggestion generation stats
10
+ # - Optional export to StatsD or Prometheus
11
+ #
12
+ # @example Enable metrics
13
+ # Kotoshu::Metrics.enable
14
+ # Kotoshu.correct?("hello")
15
+ # Kotoshu::Metrics.stats
16
+ # # => { lookups: 1, cache_hits: 0, cache_misses: 1, ... }
17
+ module Metrics
18
+ autoload :Collector, "kotoshu/metrics_collector"
19
+
20
+ class << self
21
+ # Enable metrics collection.
22
+ def enable
23
+ @enabled = true
24
+ @collector = Collector.new
25
+ end
26
+
27
+ # Disable metrics collection.
28
+ def disable
29
+ @enabled = false
30
+ @collector = nil
31
+ end
32
+
33
+ # Check if metrics are enabled.
34
+ #
35
+ # @return [Boolean] True if enabled
36
+ def enabled?
37
+ @enabled ||= false
38
+ end
39
+
40
+ # Get the metrics collector.
41
+ #
42
+ # @return [Collector, nil] The collector instance
43
+ attr_reader :collector
44
+
45
+ # Record a lookup operation.
46
+ #
47
+ # @param word [String] The word being looked up
48
+ # @param result [Boolean] The lookup result
49
+ # @param time [Float] Time taken in milliseconds
50
+ def record_lookup(word, result:, time:)
51
+ return unless enabled?
52
+
53
+ collector&.record_lookup(word, result: result, time: time)
54
+ end
55
+
56
+ # Record a cache operation.
57
+ #
58
+ # @param cache_type [String] Type of cache (lookup, suggestion)
59
+ # @param hit [Boolean] True if cache hit
60
+ def record_cache(cache_type, hit:)
61
+ return unless enabled?
62
+
63
+ collector&.record_cache(cache_type, hit: hit)
64
+ end
65
+
66
+ # Record suggestion generation.
67
+ #
68
+ # @param word [String] The input word
69
+ # @param count [Integer] Number of suggestions generated
70
+ # @param time [Float] Time taken in milliseconds
71
+ def record_suggestions(word, count:, time:)
72
+ return unless enabled?
73
+
74
+ collector&.record_suggestions(word, count: count, time: time)
75
+ end
76
+
77
+ # Record a skipped semantic rerank (confidence cascade): the
78
+ # traditional strategies were already confident, so the semantic
79
+ # rerank was not run (TODO.impl/70).
80
+ def record_semantic_cascade_skip
81
+ return unless enabled?
82
+
83
+ collector&.record_semantic_cascade_skip
84
+ end
85
+
86
+ # Get current metrics statistics.
87
+ #
88
+ # @return [Hash] Current statistics
89
+ def stats
90
+ return {} unless enabled?
91
+
92
+ collector&.stats || {}
93
+ end
94
+
95
+ # Reset all metrics.
96
+ def reset
97
+ return unless enabled?
98
+
99
+ collector&.reset
100
+ end
101
+
102
+ # Get metrics as StatsD format.
103
+ #
104
+ # @return [String] StatsD protocol lines
105
+ def to_statsd
106
+ return "" unless enabled?
107
+
108
+ collector&.to_statsd || ""
109
+ end
110
+
111
+ # Get metrics as Prometheus format.
112
+ #
113
+ # @return [String] Prometheus exposition format
114
+ def to_prometheus
115
+ return "" unless enabled?
116
+
117
+ collector&.to_prometheus || ""
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Value object for text context around an error.
6
+ #
7
+ # Provides the surrounding text before, current, and after
8
+ # an error location for context display and analysis.
9
+ #
10
+ # @example Creating context
11
+ # context = Context.new(
12
+ # before: "The quick brown",
13
+ # current: "fox",
14
+ # after: "jumps over",
15
+ # location: Location.new(line: 5, column: 16)
16
+ # )
17
+ # context.full_context # => "The quick brown fox jumps over"
18
+ class Context
19
+ attr_reader :before, :current, :after, :location, :window, :full_context
20
+
21
+ # Create a new context object.
22
+ #
23
+ # @param before [String] Text before the error
24
+ # @param current [String] The current line/text containing the error
25
+ # @param after [String] Text after the error
26
+ # @param location [Object] The error location (line/column holder)
27
+ # @param window [Integer] Window size used for context (default: 5)
28
+ def initialize(before:, current:, after:, location:, window: 5)
29
+ @before = before
30
+ @current = current
31
+ @after = after
32
+ @location = location
33
+ @window = window
34
+ @full_context = [before, current, after].compact.join("\n")
35
+ freeze
36
+ end
37
+
38
+ # Get surrounding words around the error location.
39
+ #
40
+ # @param n [Integer] Number of words on each side (default: 3)
41
+ # @return [Array<String>] Surrounding words
42
+ def surrounding_words(n = 3)
43
+ return [] if @current.nil? || @current.empty?
44
+
45
+ words = @current.split
46
+ return [] if words.empty?
47
+
48
+ # Try to find the word at the error location
49
+ target_word = word_at_location
50
+ return words unless target_word
51
+
52
+ idx = words.index(target_word)
53
+ return words unless idx
54
+
55
+ # Get n words before and after
56
+ start_idx = [0, idx - n].max
57
+ end_idx = [words.size - 1, idx + n].min
58
+
59
+ words[start_idx..end_idx].to_a
60
+ end
61
+
62
+ # Get the word at the error location.
63
+ #
64
+ # @return [String, nil] The word at the error location
65
+ def word_at_location
66
+ return nil unless @location
67
+
68
+ if @location.column && @current && @location.column < @current.length
69
+ # Get character at column
70
+ return @current[@location.column]
71
+ end
72
+
73
+ # For node-based locations, return the current text
74
+ @current
75
+ end
76
+
77
+ # Check if this context equals another.
78
+ #
79
+ # @param other [Object] Another object
80
+ # @return [Boolean] True if contexts match
81
+ def ==(other)
82
+ return false unless other.is_a?(Context)
83
+
84
+ @location == other.location && @full_context == other.full_context
85
+ end
86
+ alias_method :eql?, :==
87
+
88
+ # Hash code for hash table usage.
89
+ #
90
+ # @return [Integer] Hash code
91
+ def hash
92
+ [@location, @full_context].hash
93
+ end
94
+
95
+ # String representation.
96
+ #
97
+ # @return [String] Human-readable representation
98
+ def to_s
99
+ if @location.line
100
+ "Line #{@location.line}: #{@full_context}"
101
+ else
102
+ @full_context
103
+ end
104
+ end
105
+ alias_method :inspect, :to_s
106
+
107
+ # Get context as a formatted string with error highlighting.
108
+ #
109
+ # @param error_word [String] The error word to highlight
110
+ # @return [String] Formatted context with ANSI codes
111
+ def with_highlight(error_word)
112
+ return @full_context unless error_word
113
+
114
+ # Find and highlight the error word
115
+ @full_context.gsub(error_word) { |m| "\033[4m#{m}\033[0m" }
116
+ end
117
+ end
118
+ end
119
+ end