kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,511 @@
1
+ # frozen_string_literal: true
2
+
3
+ # base_cache autoloaded via Kotoshu::Cache
4
+
5
+ module Kotoshu
6
+ module Cache
7
+ # Language cache for dynamic dictionary and grammar rule downloads.
8
+ #
9
+ # Manages per-language dictionary and grammar rule downloads from a remote
10
+ # repository. Resources are cached locally in `$XDG_CACHE_HOME/kotoshu/languages/{code}/`
11
+ # with metadata for versioning and expiration.
12
+ #
13
+ # Extends BaseCache for common download, metadata, and validation logic.
14
+ #
15
+ # @example Getting a cached spelling dictionary
16
+ # cache = LanguageCache.new
17
+ # result = cache.get('en')
18
+ # # => { aff_path: "~/.cache/kotoshu/languages/en/spelling/index.aff",
19
+ # # dic_path: "~/.cache/kotoshu/languages/en/spelling/index.dic",
20
+ # # metadata: { ... } }
21
+ #
22
+ # @example Checking cache statistics
23
+ # stats = cache.stats
24
+ # # => { hits: 5, misses: 1, hit_rate: 0.83, ... }
25
+ class LanguageCache < BaseCache
26
+ # Supported resource types
27
+ RESOURCE_TYPES = %w[spelling grammar frequency].freeze
28
+
29
+ # Available languages (plan 107): every language staged in the
30
+ # dictionaries manifest, vendored offline-safe by
31
+ # Cache::StagedLanguages and refreshed by
32
+ # `rake kotoshu:staged_languages:sync`. Two tiers:
33
+ # - full feature — a language module under Kotoshu::Languages
34
+ # (registered keyboard + verified specimens); see
35
+ # {full_feature_languages};
36
+ # - basic — manifest-present but module-less: the script
37
+ # fallback tokenizer (Language::Script) and the generic
38
+ # keyboard default still serve setup/check/suggest.
39
+ # Local files come via setup --aff/--dic instead.
40
+ AVAILABLE_LANGUAGES = StagedLanguages::CODES
41
+
42
+ # Get or download spelling dictionary for a language.
43
+ #
44
+ # @param language [String] Language code (e.g., 'en', 'de')
45
+ # @param force_download [Boolean] Force re-download even if cached
46
+ # @return [Hash] Dictionary paths and metadata
47
+ def get_spelling(language, force_download: false)
48
+ resource_id = "#{language}:spelling"
49
+ result = get(resource_id, force_download: force_download)
50
+ result || download_spelling(language)
51
+ end
52
+
53
+ # Install a spelling dictionary from local files (no download).
54
+ # Used by ResourceManager.setup_from_local when the user already
55
+ # has .aff/.dic files on disk. Symlinks the source files into the
56
+ # cache directory so subsequent cache lookups find them. Existing
57
+ # symlinks are replaced when force: true; existing real files
58
+ # raise ArgumentError unless force: true.
59
+ #
60
+ # @param language [String] Language code
61
+ # @param aff [String] Path to .aff file
62
+ # @param dic [String] Path to .dic file
63
+ # @param force [Boolean] Overwrite existing install
64
+ # @return [Hash] Installed paths
65
+ def install_local(language, aff:, dic:, force: false)
66
+ require "fileutils"
67
+
68
+ resource_id = "#{language}:spelling"
69
+ lang_path = resource_dir_for(resource_id)
70
+ FileUtils.mkdir_p(lang_path)
71
+
72
+ target_aff = File.join(lang_path, "index.aff")
73
+ target_dic = File.join(lang_path, "index.dic")
74
+
75
+ if File.exist?(target_aff) || File.symlink?(target_aff)
76
+ raise ArgumentError, "#{target_aff} already exists (use force: true to overwrite)" unless force
77
+
78
+ File.unlink(target_aff)
79
+ end
80
+ if File.exist?(target_dic) || File.symlink?(target_dic)
81
+ raise ArgumentError, "#{target_dic} already exists (use force: true to overwrite)" unless force
82
+
83
+ File.unlink(target_dic)
84
+ end
85
+
86
+ File.symlink(File.expand_path(aff), target_aff)
87
+ File.symlink(File.expand_path(dic), target_dic)
88
+
89
+ write_metadata(metadata_path_for(resource_id),
90
+ build_metadata(language, "spelling", "local-source"))
91
+
92
+ { aff_path: target_aff, dic_path: target_dic, source: :local }
93
+ end
94
+
95
+ # Alias for get_spelling for backward compatibility.
96
+ #
97
+ # @param language [String] Language code
98
+ # @param force_download [Boolean] Force re-download
99
+ # @return [Hash] Dictionary paths and metadata
100
+ def get_dictionary(language, force_download: false)
101
+ get_spelling(language, force_download)
102
+ end
103
+
104
+ # Get or download grammar rules for a language.
105
+ #
106
+ # @param language [String] Language code
107
+ # @param force_download [Boolean] Force re-download
108
+ # @return [Hash] Rules path and metadata
109
+ def get_grammar(language, force_download: false)
110
+ resource_id = "#{language}:grammar"
111
+ result = get(resource_id, force_download: force_download)
112
+ result || download_grammar(language)
113
+ end
114
+
115
+ # Check if frequency data is available for a language.
116
+ #
117
+ # @param language_code [String] ISO 639-1 language code
118
+ # @return [Boolean] True if frequency data exists
119
+ def frequency_available?(language_code)
120
+ resource_id = "#{language_code}:frequency"
121
+ available?(resource_id)
122
+ end
123
+
124
+ # Get list of available languages.
125
+ #
126
+ # @return [Array<String>] List of supported language codes
127
+ def available_languages
128
+ AVAILABLE_LANGUAGES.dup
129
+ end
130
+
131
+ # Languages on the full-feature tier: staged in the manifest
132
+ # AND wired as a gem module (Kotoshu::Languages) — registered
133
+ # keyboard layout plus engine-verified specimens. Everything
134
+ # else in {available_languages} is the basic tier (plan 107).
135
+ #
136
+ # @return [Array<String>] Sorted full-feature language codes
137
+ def self.full_feature_languages
138
+ Language::Registry.supported_codes
139
+ .map { |code| code.split("-").first }
140
+ .uniq
141
+ .select { |base| AVAILABLE_LANGUAGES.include?(base) }
142
+ .sort
143
+ end
144
+
145
+ # Absolute on-disk path for a (language, resource_type) pair.
146
+ #
147
+ # Composes `cache_path/languages/{lang}/{type}`. Use this rather
148
+ # than reaching into `cache_path` from outside the cache — it
149
+ # keeps the on-disk layout encapsulated and lets the layout
150
+ # evolve without breaking callers.
151
+ #
152
+ # @param language [String] Language code (e.g., 'en', 'de')
153
+ # @param type [String] Resource type ('spelling', 'grammar', 'frequency')
154
+ # @return [String] Absolute directory path for the resource
155
+ def language_path(language, type)
156
+ File.join(@cache_path, "languages", language, type)
157
+ end
158
+
159
+ # Get language metadata (word count, license, source).
160
+ #
161
+ # @param language_code [String] The language code
162
+ # @return [Hash] Language info
163
+ def language_info(language_code)
164
+ {
165
+ "de" => { name: "German", word_count: 75_873, license: "GPL", source: "igerman98" },
166
+ "en" => { name: "English", word_count: 49_568, license: "LGPL/MPL/GPL", source: "SCOWL" },
167
+ "es" => { name: "Spanish", word_count: 57_344, license: "GPL", source: "LibreOffice" },
168
+ "fr" => { name: "French", word_count: 84_310, license: "MPL 2.0", source: "Grammalecte" },
169
+ "pt" => { name: "Portuguese", word_count: 312_368, license: "LGPLv3 + MPL", source: "VERO" },
170
+ "ru" => { name: "Russian", word_count: 146_269, license: "BSD-style", source: "Alexander Lebedev" }
171
+ }[language_code] || { name: language_code.upcase, word_count: 0, license: "Unknown", source: "Unknown" }
172
+ end
173
+
174
+ # Get cache size in bytes (override for language-specific tracking).
175
+ #
176
+ # @return [Integer] Total size in bytes
177
+ def cache_size
178
+ total = 0
179
+ Dir.glob(File.join(@cache_path, "languages", "**", "*.dic")).each do |path|
180
+ total += File.size(path) if File.file?(path)
181
+ end
182
+ total
183
+ end
184
+
185
+ # List all cached resources.
186
+ #
187
+ # @return [Array<String>] List of cached resource identifiers
188
+ def cached_resources
189
+ Dir.glob(File.join(@cache_path, "languages", "**", "metadata.json")).map do |path|
190
+ relative = Pathname.new(path).relative_path_from(Pathname.new(@cache_path))
191
+ parts = relative.to_s.split("/")
192
+ "#{parts[1]}:#{parts[2]}"
193
+ end.uniq
194
+ end
195
+
196
+ # Check if a resource type is supported.
197
+ #
198
+ # @param resource_id [String] The resource identifier (e.g., "en:spelling")
199
+ # @return [Boolean] True if supported
200
+ def supports_resource?(resource_id)
201
+ parts = resource_id.split(":")
202
+ return false unless parts.size == 2
203
+
204
+ language, type = parts
205
+ AVAILABLE_LANGUAGES.include?(language) && RESOURCE_TYPES.include?(type)
206
+ end
207
+
208
+ protected
209
+
210
+ # Download a spelling dictionary.
211
+ #
212
+ # @param language [String] Language code
213
+ # @return [Hash] Dictionary paths and metadata
214
+ # Fetch one spelling file, preferring the spelling/ sublayout and
215
+ # falling back to the flat {lang}/index.{ext} layout. Returns the
216
+ # URL that actually served the bytes alongside the content.
217
+ def fetch_spelling_file(language, ext)
218
+ primary = @source_registry.url_for(:spelling, lang: language, ext: ext)
219
+ begin
220
+ content = download_url(primary)
221
+ return [primary, content] if content
222
+ rescue Kotoshu::DictionaryNotFoundError
223
+ nil # fall through to the flat layout
224
+ end
225
+
226
+ flat = @source_registry.url_for(:spelling_flat, lang: language, ext: ext)
227
+ [flat, download_url(flat)]
228
+ end
229
+
230
+ def download_spelling(language)
231
+ raise Kotoshu::ResourceNotCachedError.new(language, "spelling") if offline?
232
+
233
+ lang_path = resource_dir_for("#{language}:spelling")
234
+ resource_id = "#{language}:spelling"
235
+
236
+ # Download index.aff / index.dic. Only `en` ships under the
237
+ # spelling/ sublayout upstream; staged languages sit flat at
238
+ # {lang}/index.* — try the sublayout first, fall back to flat.
239
+ aff_url, aff_content = fetch_spelling_file(language, "aff")
240
+ verify_and_audit(url: aff_url,
241
+ relative_path: "#{language}/spelling/index.aff",
242
+ content: aff_content,
243
+ resource_id: resource_id)
244
+ File.binwrite(File.join(lang_path, "index.aff"), aff_content)
245
+
246
+ dic_url, dic_content = fetch_spelling_file(language, "dic")
247
+ verify_and_audit(url: dic_url,
248
+ relative_path: "#{language}/spelling/index.dic",
249
+ content: dic_content,
250
+ resource_id: resource_id)
251
+ File.binwrite(File.join(lang_path, "index.dic"), dic_content)
252
+
253
+ # Save metadata
254
+ metadata = build_metadata(language, "spelling", checksum(dic_content))
255
+ write_metadata(metadata_path_for(resource_id), metadata)
256
+
257
+ {
258
+ aff_path: File.join(lang_path, "index.aff"),
259
+ dic_path: File.join(lang_path, "index.dic"),
260
+ cached: false,
261
+ metadata: metadata
262
+ }
263
+ end
264
+
265
+ # Download grammar rules.
266
+ #
267
+ # @param language [String] Language code
268
+ # @return [Hash] Rules path and metadata
269
+ def download_grammar(language)
270
+ raise Kotoshu::ResourceNotCachedError.new(language, "grammar") if offline?
271
+
272
+ lang_path = resource_dir_for("#{language}:grammar")
273
+ resource_id = "#{language}:grammar"
274
+
275
+ # Download rules.yaml
276
+ rules_url = @source_registry.url_for(:grammar, lang: language)
277
+ rules_content = download_url(rules_url)
278
+ verify_and_audit(url: rules_url,
279
+ relative_path: "#{language}/grammar/rules.yaml",
280
+ content: rules_content,
281
+ resource_id: resource_id)
282
+ File.binwrite(File.join(lang_path, "rules.yaml"), rules_content)
283
+
284
+ # Save metadata
285
+ metadata = build_metadata(language, "grammar", checksum(rules_content))
286
+ write_metadata(metadata_path_for(resource_id), metadata)
287
+
288
+ {
289
+ rules_path: lang_path,
290
+ cached: false,
291
+ metadata: metadata
292
+ }
293
+ end
294
+
295
+ # Download frequency data.
296
+ #
297
+ # @param language [String] Language code
298
+ # @return [Hash] Frequency data path and metadata
299
+ def download_frequency(language)
300
+ lang_path = resource_dir_for("#{language}:frequency")
301
+ resource_id = "#{language}:frequency"
302
+
303
+ # Download frequency.json from Kelly repository
304
+ freq_url = @source_registry.url_for(:frequency, lang: language)
305
+ freq_content = download_url(freq_url)
306
+ verify_and_audit(url: freq_url,
307
+ relative_path: "data/#{language}.json",
308
+ content: freq_content,
309
+ resource_id: resource_id)
310
+
311
+ # Validate JSON
312
+ JSON.parse(freq_content)
313
+
314
+ # Save frequency file
315
+ freq_file = File.join(lang_path, "frequency.json")
316
+ File.write(freq_file, freq_content)
317
+
318
+ # Save metadata (with custom URL for Kelly)
319
+ metadata = build_metadata(language, "kelly_frequency", checksum(freq_content))
320
+ metadata[:url] = freq_url # Override with specific Kelly URL
321
+ write_metadata(metadata_path_for(resource_id), metadata)
322
+
323
+ {
324
+ frequency_path: freq_file,
325
+ cached: false,
326
+ metadata: metadata
327
+ }
328
+ end
329
+
330
+ private
331
+
332
+ # LanguageCache serves from the kotoshu/dictionaries repo for spelling
333
+ # and grammar; frequency lives in a separate repo (kelly). Pin the
334
+ # manifest URL at the dictionaries repo since that's the primary
335
+ # surface users see. Kelly's manifest can be added when that repo
336
+ # ships one.
337
+ def manifest_url
338
+ @manifest_url || @source_registry.url_for(:dict_manifest)
339
+ end
340
+
341
+ # Download a specific resource (implements abstract method).
342
+ #
343
+ # @param resource_id [String] The resource identifier
344
+ # @param _dest_path [String] Destination directory (unused —
345
+ # spelling resources pick their own layout)
346
+ # @return [Object] Downloaded resource
347
+ def download_resource(resource_id, _dest_path)
348
+ language = extract_language(resource_id)
349
+ type = extract_type(resource_id)
350
+ return nil unless language && type
351
+
352
+ case type
353
+ when "spelling" then download_spelling(language)
354
+ when "grammar" then download_grammar(language)
355
+ when "frequency" then download_frequency(language)
356
+ else raise "Unknown resource type: #{type}"
357
+ end
358
+ end
359
+
360
+ # Load metadata for a resource.
361
+ #
362
+ # @param resource_id [String] The resource identifier
363
+ # @return [Hash, nil] Metadata or nil if not found
364
+ def load_metadata_for(resource_id)
365
+ metadata_path = metadata_path_for(resource_id)
366
+ return nil unless File.exist?(metadata_path)
367
+
368
+ read_metadata(metadata_path)
369
+ end
370
+
371
+ # Load cached resource by type.
372
+ #
373
+ # @param resource_id [String] The resource identifier
374
+ # @param type [String] The resource type
375
+ # @param metadata [Hash] The resource metadata
376
+ # @return [Hash, nil] Loaded resource or nil
377
+ def load_cached_resource_by_type(resource_id, type, metadata)
378
+ case type
379
+ when "spelling" then load_cached_spelling(resource_id, metadata)
380
+ when "grammar" then load_cached_grammar(resource_id, metadata)
381
+ when "frequency" then load_cached_frequency(resource_id, metadata)
382
+ end
383
+ end
384
+
385
+ # Load cached spelling resource.
386
+ #
387
+ # @param resource_id [String] The resource identifier
388
+ # @param metadata [Hash] The resource metadata
389
+ # @return [Hash] Spelling resource data
390
+ def load_cached_spelling(resource_id, metadata)
391
+ lang_path = resource_dir_for(resource_id)
392
+ {
393
+ aff_path: File.join(lang_path, "index.aff"),
394
+ dic_path: File.join(lang_path, "index.dic"),
395
+ cached: true,
396
+ metadata: metadata
397
+ }
398
+ end
399
+
400
+ # Load cached grammar resource.
401
+ #
402
+ # @param resource_id [String] The resource identifier
403
+ # @param metadata [Hash] The resource metadata
404
+ # @return [Hash] Grammar resource data
405
+ def load_cached_grammar(resource_id, metadata)
406
+ lang_path = resource_dir_for(resource_id)
407
+ {
408
+ rules_path: lang_path,
409
+ cached: true,
410
+ metadata: metadata
411
+ }
412
+ end
413
+
414
+ # Load cached frequency resource.
415
+ #
416
+ # @param resource_id [String] The resource identifier
417
+ # @param metadata [Hash] The resource metadata
418
+ # @return [Hash, nil] Frequency resource data or nil
419
+ def load_cached_frequency(resource_id, metadata)
420
+ # CommonWordsLoader autoloaded via Kotoshu::Data
421
+ freq_file = File.join(resource_dir_for(resource_id), "frequency.json")
422
+ return nil unless File.exist?(freq_file)
423
+
424
+ data = Data::CommonWordsLoader.load_from_frequency_file(freq_file)
425
+ {
426
+ frequency_path: freq_file,
427
+ tiers: data[:tiers],
428
+ metadata: metadata
429
+ }
430
+ end
431
+
432
+ # Build metadata hash for a resource.
433
+ #
434
+ # @param language [String] Language code
435
+ # @param type [String] Resource type
436
+ # @param content_checksum [String] SHA256 checksum of content
437
+ # @return [Hash] Metadata hash
438
+ def build_metadata(language, type, content_checksum)
439
+ {
440
+ version: Time.now.utc.iso8601,
441
+ url: @url_base,
442
+ language: language,
443
+ type: type,
444
+ checksum: content_checksum,
445
+ cached_at: Time.now.utc.iso8601
446
+ }
447
+ end
448
+
449
+ public
450
+
451
+ # Load cached resource data (implements abstract method).
452
+ #
453
+ # Public — this is the cache-only reader BaseCache declares
454
+ # publicly; resolve paths use it to read without any download.
455
+ #
456
+ # @param resource_id [String] The resource identifier
457
+ # @return [Object, nil] Loaded resource or nil
458
+ def load_cached(resource_id)
459
+ parts = parse_resource_id(resource_id)
460
+ return nil unless parts
461
+
462
+ type = parts[1]
463
+ metadata = load_metadata_for(resource_id)
464
+ return nil unless metadata
465
+
466
+ load_cached_resource_by_type(resource_id, type, metadata)
467
+ end
468
+
469
+ # Get metadata file path for a resource.
470
+ #
471
+ # @param resource_id [String] The resource identifier
472
+ # @return [String] Metadata file path
473
+ def metadata_path_for(resource_id)
474
+ language = extract_language(resource_id)
475
+ type = extract_type(resource_id)
476
+ File.join(language_path(language, type), "metadata.json")
477
+ end
478
+
479
+ # Get resource directory path.
480
+ #
481
+ # @param resource_id [String] The resource identifier
482
+ # @return [String] Resource directory path
483
+ def resource_dir_for(resource_id)
484
+ language_path(extract_language(resource_id), extract_type(resource_id))
485
+ end
486
+
487
+ # Check if all resource files exist.
488
+ #
489
+ # @param resource_id [String] The resource identifier
490
+ # @return [Boolean] True if all files exist
491
+ def resource_files_exist?(resource_id)
492
+ type = extract_type(resource_id)
493
+ return false unless type
494
+
495
+ lang_path = resource_dir_for(resource_id)
496
+
497
+ case type
498
+ when "spelling"
499
+ File.exist?(File.join(lang_path, "index.aff")) &&
500
+ File.exist?(File.join(lang_path, "index.dic"))
501
+ when "grammar"
502
+ File.exist?(File.join(lang_path, "rules.yaml"))
503
+ when "frequency"
504
+ File.exist?(File.join(lang_path, "frequency.json"))
505
+ else
506
+ false
507
+ end
508
+ end
509
+ end
510
+ end
511
+ end
@@ -0,0 +1,166 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Cache autoloaded via Kotoshu::Cache
4
+
5
+ module Kotoshu
6
+ module Cache
7
+ # LRU (Least Recently Used) cache for fast lookups.
8
+ #
9
+ # This cache automatically evicts the least recently used entries
10
+ # when the maximum size is reached.
11
+ #
12
+ # @example Basic usage
13
+ # cache = LookupCache.new(max_size: 1000)
14
+ # cache.write("key", "value")
15
+ # cache.read("key") # => "value"
16
+ #
17
+ # @example Using fetch for lazy computation
18
+ # cache.fetch("expensive_key") { compute_expensive_value() }
19
+ class LookupCache
20
+ include Cache
21
+
22
+ # Default maximum cache size
23
+ DEFAULT_MAX_SIZE = 1000
24
+
25
+ # @return [Integer] Maximum number of entries
26
+ attr_reader :max_size
27
+
28
+ # Create a new LRU cache.
29
+ #
30
+ # @param max_size [Integer] Maximum number of entries (default: 1000)
31
+ def initialize(max_size: DEFAULT_MAX_SIZE)
32
+ @max_size = max_size
33
+ @data = {} # key => [value, access_order]
34
+ @access_order = 0
35
+ @stats = { hits: 0, misses: 0 }
36
+ end
37
+
38
+ # Retrieve a value from cache, or compute it.
39
+ #
40
+ # @param key [Object] The cache key
41
+ # @param default [Object] Optional default value (if no block given)
42
+ # @yield Block to compute value on cache miss
43
+ # @return [Object] The cached or computed value
44
+ def fetch(key, default = nil)
45
+ if key?(key)
46
+ record_hit
47
+ @data[key][0] # Return value
48
+ else
49
+ record_miss
50
+ value = block_given? ? yield : default
51
+ write(key, value)
52
+ value
53
+ end
54
+ end
55
+
56
+ # Write a value to cache.
57
+ #
58
+ # @param key [Object] The cache key
59
+ # @param value [Object] The value to store
60
+ # @return [Object] The stored value
61
+ def write(key, value)
62
+ evict_if_needed
63
+
64
+ @access_order += 1
65
+ @data[key] = [value, @access_order]
66
+
67
+ value
68
+ end
69
+
70
+ # Read a value from cache.
71
+ #
72
+ # @param key [Object] The cache key
73
+ # @return [Object, nil] The cached value or nil
74
+ def read(key)
75
+ entry = @data[key]
76
+
77
+ if entry
78
+ record_hit
79
+ @access_order += 1
80
+ entry[1] = @access_order # Update access order
81
+ entry[0] # Return value
82
+ else
83
+ record_miss
84
+ nil
85
+ end
86
+ end
87
+
88
+ # Delete a value from cache.
89
+ #
90
+ # @param key [Object] The cache key
91
+ # @return [Object, nil] The deleted value or nil
92
+ def delete(key)
93
+ entry = @data.delete(key)
94
+ entry&.first # Return value or nil
95
+ end
96
+
97
+ # Clear all entries from cache.
98
+ #
99
+ # @return [self] Self for chaining
100
+ def clear
101
+ @data.clear
102
+ @access_order = 0
103
+ self
104
+ end
105
+
106
+ # Check if key exists in cache.
107
+ #
108
+ # @param key [Object] The cache key
109
+ # @return [Boolean] True if key exists
110
+ def key?(key)
111
+ @data.key?(key)
112
+ end
113
+
114
+ # Get number of entries in cache.
115
+ #
116
+ # @return [Integer] Number of entries
117
+ def size
118
+ @data.size
119
+ end
120
+
121
+ # Get cache statistics.
122
+ #
123
+ # @return [Hash] Statistics including :hits, :misses, :size, :hit_rate
124
+ def stats
125
+ total = @stats[:hits] + @stats[:misses]
126
+ hit_rate = total.positive? ? @stats[:hits].to_f / total : 0.0
127
+
128
+ {
129
+ hits: @stats[:hits],
130
+ misses: @stats[:misses],
131
+ size: size,
132
+ hit_rate: hit_rate.round(4)
133
+ }
134
+ end
135
+
136
+ # Reset statistics counters.
137
+ #
138
+ # @return [self] Self for chaining
139
+ def reset_stats
140
+ @stats = { hits: 0, misses: 0 }
141
+ self
142
+ end
143
+
144
+ private
145
+
146
+ # Record a cache hit.
147
+ def record_hit
148
+ @stats[:hits] += 1
149
+ end
150
+
151
+ # Record a cache miss.
152
+ def record_miss
153
+ @stats[:misses] += 1
154
+ end
155
+
156
+ # Evict least recently used entry if cache is full.
157
+ def evict_if_needed
158
+ return if @data.size < @max_size
159
+
160
+ # Find entry with lowest access order
161
+ lru_key = @data.min_by { |_, v| v[1] }&.first
162
+ @data.delete(lru_key) if lru_key
163
+ end
164
+ end
165
+ end
166
+ end