kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,778 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "net/http"
5
+ require "json"
6
+ require "digest"
7
+ require "uri"
8
+ require "time"
9
+
10
+ module Kotoshu
11
+ module Cache
12
+ # Abstract base class for all cache implementations.
13
+ #
14
+ # Provides common functionality for:
15
+ # - HTTP downloads with metadata
16
+ # - Cache validation (exists, expired)
17
+ # - Statistics tracking (hits, misses, hit rate)
18
+ # - TTL management
19
+ #
20
+ # Subclasses implement specific download and loading logic.
21
+ #
22
+ # @abstract Subclass must implement {#download_resource}, {#load_cached}
23
+ class BaseCache
24
+ # @return [String] Path to the cache directory
25
+ attr_reader :cache_path
26
+
27
+ # @return [String] Base URL for downloading resources
28
+ attr_reader :url_base
29
+
30
+ # @return [Integer] Cache TTL in seconds
31
+ attr_reader :cache_ttl
32
+
33
+ # @return [Integer] Maximum total cache size in bytes. Enforced
34
+ # by {#evict}; the value is pulled from
35
+ # +Kotoshu::Configuration.instance.max_cache_size+ by default
36
+ # (env: +KOTOSHU_MAX_CACHE_SIZE+).
37
+ attr_reader :max_cache_size
38
+
39
+ # @return [String] GitHub repository URL
40
+ attr_reader :github_url
41
+
42
+ # @return [Kotoshu::SourceRegistry] Single source of truth for
43
+ # per-repo URLs and pins. Subclasses MUST build URLs through
44
+ # this registry rather than constructing URL strings inline.
45
+ attr_reader :source_registry
46
+
47
+ # Create a new cache.
48
+ #
49
+ # @param cache_path [String] Path to cache directory
50
+ # @param url_base [String] Base URL for downloads (deprecated; pass source_registry instead).
51
+ # When given, it still controls where downloads come from: the
52
+ # registry is derived from it so the deprecated parameter keeps
53
+ # its documented meaning.
54
+ # @param cache_ttl [Integer] Cache TTL in seconds
55
+ # @param github_url [String] GitHub repository URL
56
+ # @param resource_pin [String] Branch/tag/commit for URL templates (deprecated; use source_registry)
57
+ # @param manifest_url [String, nil] Override manifest.json URL
58
+ # @param audit_log [Integrity::AuditLog, nil] Override audit log
59
+ # @param source_registry [Kotoshu::SourceRegistry, nil] Single source of truth for URLs/pins
60
+ # @param max_cache_size [Integer, nil] Maximum cache size in bytes (default 1 GB)
61
+ def initialize(cache_path: nil, url_base: nil, cache_ttl: nil, github_url: nil,
62
+ resource_pin: nil, manifest_url: nil, audit_log: nil,
63
+ source_registry: nil, max_cache_size: nil)
64
+ @cache_path = cache_path || default_cache_path
65
+ @source_registry = source_registry || source_registry_for(url_base)
66
+ @url_base = url_base || @source_registry.base_url
67
+ @cache_ttl = cache_ttl || default_cache_ttl
68
+ @github_url = github_url || default_github_url
69
+ @resource_pin = resource_pin || @source_registry.pin_for_source(:spelling)
70
+ @manifest_url = manifest_url
71
+ @audit_log = audit_log || default_audit_log
72
+ @max_cache_size = max_cache_size || default_max_cache_size
73
+ @manifest = nil
74
+ @manifest_loaded = false
75
+ @hits = 0
76
+ @misses = 0
77
+
78
+ # Ensure cache directory exists
79
+ FileUtils.mkdir_p(@cache_path)
80
+ FileUtils.mkdir_p(File.join(@cache_path, "tmp"))
81
+ end
82
+
83
+ # Check if a resource is available in cache.
84
+ #
85
+ # @param resource_id [String] The resource identifier (e.g., language code)
86
+ # @return [Boolean] True if resource is cached and valid
87
+ def available?(resource_id)
88
+ return false unless supports_resource?(resource_id)
89
+
90
+ metadata_path = metadata_path_for(resource_id)
91
+ return false unless File.exist?(metadata_path)
92
+ return false if expired?(metadata_path)
93
+
94
+ resource_files_exist?(resource_id)
95
+ end
96
+
97
+ # Whether readable cached data exists for +resource_id+, TTL
98
+ # ignored (plan 117). Expiry is a refresh signal for the setup /
99
+ # download paths, never a reason for readers to behave as if the
100
+ # data were absent: the bytes are checksummed at write time, so
101
+ # present-but-expired data is exactly the dataset it was.
102
+ #
103
+ # @param resource_id [String] The resource identifier
104
+ # @return [Boolean] True if metadata and data files exist
105
+ def cached_data?(resource_id)
106
+ return false unless supports_resource?(resource_id)
107
+
108
+ File.exist?(metadata_path_for(resource_id)) && resource_files_exist?(resource_id)
109
+ end
110
+
111
+ # Get a resource from cache or download it.
112
+ #
113
+ # @param resource_id [String] The resource identifier
114
+ # @param force_download [Boolean] Force re-download even if cached
115
+ # @return [Object, nil] The cached resource or nil if not available
116
+ def get(resource_id, force_download: false)
117
+ return nil unless supports_resource?(resource_id)
118
+
119
+ metadata_path = metadata_path_for(resource_id)
120
+
121
+ if !force_download && cached?(metadata_path) && !expired?(metadata_path)
122
+ @hits += 1
123
+ return load_cached(resource_id)
124
+ end
125
+
126
+ @misses += 1
127
+ download(resource_id)
128
+ end
129
+
130
+ # Clear a specific resource from cache.
131
+ #
132
+ # @param resource_id [String] The resource identifier
133
+ # @return [Boolean] True if cache was cleared
134
+ def clear(resource_id)
135
+ return false unless supports_resource?(resource_id)
136
+
137
+ resource_dir = resource_dir_for(resource_id)
138
+ if File.exist?(resource_dir)
139
+ FileUtils.rm_rf(resource_dir)
140
+ return true
141
+ end
142
+
143
+ false
144
+ end
145
+
146
+ # Clear all cached resources.
147
+ #
148
+ # @return [void]
149
+ def clear_all
150
+ @hits = 0
151
+ @misses = 0
152
+ FileUtils.rm_rf(@cache_path)
153
+ FileUtils.mkdir_p(@cache_path)
154
+ FileUtils.mkdir_p(File.join(@cache_path, "tmp"))
155
+ end
156
+
157
+ # Get cache statistics.
158
+ #
159
+ # @return [Hash] Statistics including :hits, :misses, :hit_rate, :size
160
+ def stats
161
+ total = @hits + @misses
162
+ hit_rate = total.positive? ? (@hits.to_f / total) : 0.0
163
+
164
+ {
165
+ hits: @hits,
166
+ misses: @misses,
167
+ total: total,
168
+ hit_rate: hit_rate,
169
+ cached_resources: cached_resources,
170
+ size_bytes: cache_size,
171
+ oldest_entry: oldest_entry
172
+ }
173
+ end
174
+
175
+ # Reset statistics counters.
176
+ #
177
+ # @return [self] Self for chaining
178
+ def reset_stats
179
+ @hits = 0
180
+ @misses = 0
181
+ self
182
+ end
183
+
184
+ # Clean expired cache entries.
185
+ #
186
+ # @return [Hash] Cleanup statistics
187
+ def clean
188
+ expired_count = clean_expired
189
+ size_reclaimed = clean_by_size
190
+
191
+ {
192
+ expired_entries_removed: expired_count,
193
+ bytes_reclaimed: size_reclaimed
194
+ }
195
+ end
196
+
197
+ # Enforce the cache size cap by evicting the oldest entries
198
+ # (LRU by +cached_at+). When +dry_run:+ true, returns the plan
199
+ # without modifying the disk; otherwise removes the evicted
200
+ # entries via +FileUtils.rm_rf+ and returns the same plan.
201
+ #
202
+ # The plan is computed by {Cache::EvictionPolicy} — a pure value
203
+ # object that performs no IO. This method is the only place that
204
+ # collects on-disk entries and the only place that mutates the
205
+ # cache directory based on the plan, keeping the policy testable
206
+ # in isolation.
207
+ #
208
+ # @param dry_run [Boolean] return the plan without writing
209
+ # @return [Hash] { evict: Array<Hash>, keep: Array<Hash>,
210
+ # bytes_reclaimed: Integer } where each entry hash carries
211
+ # :path, :size, :cached_at
212
+ def evict(dry_run: false)
213
+ policy = EvictionPolicy.new(max_size: @max_cache_size)
214
+ plan = policy.plan(collect_eviction_entries)
215
+ return plan if dry_run
216
+
217
+ plan[:evict].each { |e| FileUtils.rm_rf(e[:path]) }
218
+ plan
219
+ end
220
+
221
+ # List all cached resources.
222
+ #
223
+ # @return [Array<String>] List of cached resource identifiers
224
+ def cached_resources
225
+ raise NotImplementedError, "Subclass must implement"
226
+ end
227
+
228
+ # Check if a resource type is supported.
229
+ #
230
+ # @param resource_id [String] The resource identifier
231
+ # @return [Boolean] True if supported
232
+ def supports_resource?(resource_id)
233
+ raise NotImplementedError, "Subclass must implement"
234
+ end
235
+
236
+ # Download a resource from GitHub.
237
+ #
238
+ # @param resource_id [String] The resource identifier
239
+ # @return [Object, nil] Downloaded resource or nil on error
240
+ # @raise [Kotoshu::DictionaryNotFoundError] when the fetch fails
241
+ # (unreachable source, HTTP error); the documented cache API
242
+ # boundary raises the public error class rather than leaking
243
+ # transport errors
244
+ # @raise [Kotoshu::ResourceNotCachedError] in offline mode —
245
+ # only explicit paths (Kotoshu.setup, kotoshu cache download)
246
+ # reach here, so fail loudly instead of silently skipping
247
+ def download(resource_id)
248
+ return nil unless supports_resource?(resource_id)
249
+
250
+ language, explicit_type = resource_id.to_s.split(":", 2)
251
+ raise Kotoshu::ResourceNotCachedError.new(language, explicit_type || cache_kind) if offline?
252
+
253
+ resource_dir = resource_dir_for(resource_id)
254
+ FileUtils.mkdir_p(resource_dir)
255
+
256
+ begin
257
+ download_resource(resource_id, resource_dir)
258
+ rescue Kotoshu::DictionaryNotFoundError
259
+ raise
260
+ rescue StandardError => e
261
+ warn "Error downloading #{resource_id}: #{e.message}" if $VERBOSE
262
+ nil
263
+ end
264
+ end
265
+
266
+ # Abstract: Download a specific resource.
267
+ #
268
+ # @param resource_id [String] The resource identifier
269
+ # @param dest_path [String] Destination directory
270
+ # @return [Object] Downloaded resource
271
+ # @abstract Subclass must implement
272
+ def download_resource(resource_id, dest_path)
273
+ raise NotImplementedError, "Subclass must implement"
274
+ end
275
+
276
+ # True when downloads are forbidden (config.offline /
277
+ # KOTOSHU_OFFLINE=1). Checked in #download so every cache
278
+ # subclass inherits the enforcement.
279
+ #
280
+ # @return [Boolean] Whether offline mode is on
281
+ def offline?
282
+ Kotoshu.configuration.offline
283
+ end
284
+
285
+ # Short label for error messages ("frequency", "language",
286
+ # "model") derived from the nearest named class — anonymous
287
+ # subclasses (test recorders) fall back to their parent's name.
288
+ #
289
+ # @return [String] The cache kind
290
+ def cache_kind
291
+ klass = self.class
292
+ klass = klass.superclass while klass && klass.name.nil?
293
+ klass&.name.to_s[/(\w+)Cache\z/, 1]&.downcase || "resource"
294
+ end
295
+
296
+ # Abstract: Load cached resource data.
297
+ #
298
+ # @param resource_id [String] The resource identifier
299
+ # @return [Object, nil] Loaded resource or nil
300
+ # @abstract Subclass must implement
301
+ def load_cached(resource_id)
302
+ raise NotImplementedError, "Subclass must implement"
303
+ end
304
+
305
+ # Abstract: Get metadata file path for a resource.
306
+ #
307
+ # @param resource_id [String] The resource identifier
308
+ # @return [String] Metadata file path
309
+ # @abstract Subclass must implement
310
+ def metadata_path_for(resource_id)
311
+ raise NotImplementedError, "Subclass must implement"
312
+ end
313
+
314
+ # Abstract: Get resource directory path.
315
+ #
316
+ # @param resource_id [String] The resource identifier
317
+ # @return [String] Resource directory path
318
+ # @abstract Subclass must implement
319
+ def resource_dir_for(resource_id)
320
+ raise NotImplementedError, "Subclass must implement"
321
+ end
322
+
323
+ # Abstract: Check if all resource files exist.
324
+ #
325
+ # @param resource_id [String] The resource identifier
326
+ # @return [Boolean] True if all files exist
327
+ # @abstract Subclass must implement
328
+ def resource_files_exist?(resource_id)
329
+ raise NotImplementedError, "Subclass must implement"
330
+ end
331
+
332
+ protected
333
+
334
+ # Download content from a URL.
335
+ #
336
+ # Fetch failures — unreachable host, transport errors, HTTP error
337
+ # statuses — are mapped to {Kotoshu::DictionaryNotFoundError} so
338
+ # callers see the documented public error at the cache boundary.
339
+ #
340
+ # @param url [String] URL to download
341
+ # @return [String] Downloaded content
342
+ # @raise [Kotoshu::DictionaryNotFoundError] when the fetch fails
343
+ def download_url(url)
344
+ # Transport-level backstop: no bytes move in offline mode even
345
+ # for callers that bypass #download (e.g. get_* fallbacks).
346
+ raise "Refusing to download #{url}: offline mode is enabled (KOTOSHU_OFFLINE=1)" if offline?
347
+
348
+ uri = URI.parse(url)
349
+
350
+ http = Net::HTTP.new(uri.host, uri.port)
351
+ http.use_ssl = (uri.scheme == "https")
352
+ http.open_timeout = 10
353
+ http.read_timeout = 30
354
+
355
+ request = Net::HTTP::Get.new(uri.request_uri)
356
+
357
+ response = http.request(request)
358
+
359
+ unless response.is_a?(Net::HTTPSuccess)
360
+ raise download_failure(url, "#{response.code} #{response.message}")
361
+ end
362
+
363
+ response.body
364
+ rescue SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
365
+ Errno::ENETUNREACH, Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout,
366
+ Net::ProtocolError, EOFError, OpenSSL::SSL::SSLError => e
367
+ raise download_failure(url, "#{e.class}: #{e.message}")
368
+ end
369
+
370
+ # Build the public error for a failed fetch. The URL is stored as
371
+ # the error's #path so callers can report which source failed.
372
+ #
373
+ # @param url [String] URL that could not be fetched
374
+ # @param reason [String] Human-readable failure reason
375
+ # @return [Kotoshu::DictionaryNotFoundError]
376
+ def download_failure(url, reason)
377
+ Kotoshu::DictionaryNotFoundError.new(url, "Failed to download #{url}: #{reason}")
378
+ end
379
+
380
+ # Download a file to disk, streaming in chunks.
381
+ #
382
+ # @param url [String] Source URL
383
+ # @param dest_path [String] Destination file path
384
+ # @param reporter [#start,#update,#maybe_report_periodic,#finish,nil]
385
+ # Optional progress reporter. Defaults to
386
+ # Kotoshu.configuration.download_reporter (typically nil for
387
+ # programmatic use, set by the CLI during setup).
388
+ def download_file(url, dest_path, reporter: nil)
389
+ # Transport-level backstop, same as download_url.
390
+ raise "Refusing to download #{url}: offline mode is enabled (KOTOSHU_OFFLINE=1)" if offline?
391
+
392
+ reporter ||= Kotoshu.configuration.download_reporter
393
+ uri = URI.parse(url)
394
+
395
+ http = Net::HTTP.new(uri.host, uri.port)
396
+ http.use_ssl = (uri.scheme == "https")
397
+ http.open_timeout = 30
398
+ http.read_timeout = 300
399
+
400
+ request = Net::HTTP::Get.new(uri.request_uri)
401
+
402
+ http.request(request) do |response|
403
+ case response
404
+ when Net::HTTPSuccess
405
+ content_length = content_length_from(response)
406
+ FileUtils.mkdir_p(File.dirname(dest_path))
407
+ received = 0
408
+ reporter&.start(content_length)
409
+ File.open(dest_path, "wb") do |file|
410
+ response.read_body do |chunk|
411
+ file.write(chunk)
412
+ received += chunk.bytesize
413
+ reporter&.update(received)
414
+ reporter&.maybe_report_periodic
415
+ end
416
+ end
417
+ reporter&.finish
418
+ when Net::HTTPRedirection
419
+ download_file(response["location"], dest_path, reporter: reporter)
420
+ else
421
+ raise "Failed to download #{url}: #{response.code} #{response.message}"
422
+ end
423
+ end
424
+ end
425
+
426
+ # Extract Content-Length safely. Some servers omit it (chunked
427
+ # transfer encoding); caller treats nil as "size unknown".
428
+ # @param response [Net::HTTPResponse]
429
+ # @return [Integer, nil]
430
+ def content_length_from(response)
431
+ raw = response["Content-Length"]
432
+ return nil if raw.nil? || raw.strip.empty?
433
+
434
+ Integer(raw)
435
+ rescue ArgumentError
436
+ nil
437
+ end
438
+
439
+ # Write metadata to file.
440
+ #
441
+ # @param path [String] Metadata file path
442
+ # @param metadata [Hash] Metadata to write
443
+ def write_metadata(path, metadata)
444
+ FileUtils.mkdir_p(File.dirname(path))
445
+ File.write(path, JSON.pretty_generate(metadata))
446
+ end
447
+
448
+ # Check if cached file exists.
449
+ #
450
+ # @param metadata_path [String] Path to metadata file
451
+ # @return [Boolean] True if cached
452
+ def cached?(metadata_path)
453
+ File.exist?(metadata_path)
454
+ end
455
+
456
+ # Check if cached file is expired.
457
+ #
458
+ # @param metadata_path [String] Path to metadata file
459
+ # @return [Boolean] True if expired
460
+ def expired?(metadata_path)
461
+ return true unless File.exist?(metadata_path)
462
+
463
+ metadata = read_metadata(metadata_path)
464
+ return true unless metadata
465
+
466
+ cached_time_str = metadata["cached_at"] || metadata["version"]
467
+ return true unless cached_time_str
468
+
469
+ begin
470
+ cached_time = Time.iso8601(cached_time_str)
471
+ Time.now.utc - cached_time > @cache_ttl
472
+ rescue StandardError
473
+ true
474
+ end
475
+ end
476
+
477
+ # Calculate checksum of content.
478
+ #
479
+ # @param content [String] Content to checksum
480
+ # @return [String] SHA256 checksum
481
+ def checksum(content)
482
+ Digest::SHA256.hexdigest(content)
483
+ end
484
+
485
+ # Verify downloaded content against the manifest and log to audit.
486
+ #
487
+ # If a manifest is published for this cache's content repo, the content's
488
+ # SHA-256 is checked against the manifest entry for `relative_path`. A
489
+ # mismatch raises {Kotoshu::IntegrityError} — callers MUST remove the
490
+ # corrupt bytes from disk so the next call re-downloads. When no manifest
491
+ # entry exists (kotoshu/dictionaries hasn't shipped one yet), the content
492
+ # is logged as `"unverified"` and accepted — graceful degradation.
493
+ #
494
+ # @param url [String] Source URL (for audit log)
495
+ # @param relative_path [String] Manifest lookup key (e.g., "en/spelling/index.dic")
496
+ # @param content [String] Downloaded bytes
497
+ # @param resource_id [String, nil] Caller-supplied resource identifier
498
+ # @return [void]
499
+ def verify_and_audit(url:, relative_path:, content:, resource_id: nil)
500
+ sha = Digest::SHA256.hexdigest(content)
501
+ entry = manifest_entry_for(relative_path)
502
+
503
+ if entry.nil?
504
+ @audit_log.record(
505
+ url: url, status: "unverified", size: content.bytesize,
506
+ sha256: sha, manifest_sha256: nil, resource_id: resource_id
507
+ )
508
+ return
509
+ end
510
+
511
+ if sha == entry.sha256
512
+ @audit_log.record(
513
+ url: url, status: "verified", size: content.bytesize,
514
+ sha256: sha, manifest_sha256: entry.sha256, resource_id: resource_id
515
+ )
516
+ else
517
+ @audit_log.record(
518
+ url: url, status: "mismatch", size: content.bytesize,
519
+ sha256: sha, manifest_sha256: entry.sha256, resource_id: resource_id
520
+ )
521
+ raise Kotoshu::IntegrityError.new(
522
+ relative_path, expected: entry.sha256, actual: sha, url: url
523
+ )
524
+ end
525
+ end
526
+
527
+ # Pin used in URL templates (default "main"; override via constructor
528
+ # or KOTOSHU_RESOURCE_PIN env var through Configuration).
529
+ #
530
+ # @return [String]
531
+ attr_reader :resource_pin
532
+
533
+ private
534
+
535
+ # Look up a manifest entry by relative path. Loads the manifest
536
+ # lazily on first call; treats HTTP 404/410 as "no manifest" (returns
537
+ # nil) so verification is gracefully skipped.
538
+ def manifest_entry_for(relative_path)
539
+ load_manifest! unless @manifest_loaded
540
+ @manifest&.fetch(relative_path)
541
+ end
542
+
543
+ # Fetch the manifest once per cache instance. Sets @manifest_loaded
544
+ # regardless of outcome so we don't retry on every download.
545
+ def load_manifest!
546
+ @manifest_loaded = true
547
+ url = manifest_url
548
+ return unless url
549
+
550
+ begin
551
+ @manifest = Kotoshu::Integrity::Manifest.load(url)
552
+ rescue StandardError => e
553
+ warn "Manifest fetch failed for #{url}: #{e.message}" if $VERBOSE
554
+ @manifest = nil
555
+ end
556
+ end
557
+
558
+ # Default manifest URL — subclasses override to point at their repo's
559
+ # manifest.json. Returns nil to opt out of manifest verification.
560
+ def manifest_url
561
+ @manifest_url
562
+ end
563
+
564
+ # Get cache size in bytes.
565
+ #
566
+ # @return [Integer] Total size in bytes
567
+ def cache_size
568
+ total = 0
569
+ Dir.glob(File.join(@cache_path, "**", "*")).each do |path|
570
+ total += File.size(path) if File.file?(path)
571
+ end
572
+ total
573
+ end
574
+
575
+ # Get oldest cached entry timestamp.
576
+ #
577
+ # @return [String, nil] ISO8601 timestamp or nil
578
+ def oldest_entry
579
+ oldest = nil
580
+
581
+ Dir.glob(File.join(@cache_path, "**", "metadata.json")).each do |metadata_path|
582
+ metadata = read_metadata(metadata_path)
583
+ next unless metadata
584
+
585
+ timestamp = metadata["cached_at"] || metadata["version"]
586
+ next unless timestamp
587
+
588
+ oldest = timestamp if oldest.nil? || timestamp < oldest
589
+ end
590
+
591
+ oldest
592
+ end
593
+
594
+ # Clean expired cache entries.
595
+ #
596
+ # @return [Integer] Number of entries removed
597
+ def clean_expired
598
+ count = 0
599
+
600
+ Dir.glob(File.join(@cache_path, "**", "metadata.json")).each do |metadata_path|
601
+ next unless expired?(metadata_path)
602
+
603
+ dir_path = File.dirname(metadata_path)
604
+ FileUtils.rm_rf(dir_path)
605
+ count += 1
606
+ end
607
+
608
+ count
609
+ end
610
+
611
+ # Clean cache entries by size.
612
+ #
613
+ # @return [Integer] Bytes reclaimed
614
+ def clean_by_size
615
+ 0 # Override in subclass if needed
616
+ end
617
+
618
+ # Walk the cache directory and build one eviction-entry record
619
+ # per discovered metadata.json. Each record carries the resource
620
+ # directory path (what to remove), total size of files under it,
621
+ # and the +cached_at+ timestamp from metadata. Entries whose
622
+ # metadata cannot be parsed still appear (with nil +cached_at+)
623
+ # so they are not stranded forever.
624
+ #
625
+ # @return [Array<Hash>] { path:, size:, cached_at: }
626
+ def collect_eviction_entries
627
+ Dir.glob(File.join(@cache_path, "**", "metadata.json")).map do |metadata_path|
628
+ dir = File.dirname(metadata_path)
629
+ metadata = read_metadata(metadata_path) || {}
630
+ {
631
+ path: dir,
632
+ size: dir_size(dir),
633
+ cached_at: metadata["cached_at"] || metadata["version"]
634
+ }
635
+ end
636
+ end
637
+
638
+ # Sum the size of every regular file under +path+, recursively.
639
+ #
640
+ # @param path [String] directory to measure
641
+ # @return [Integer] total bytes
642
+ def dir_size(path)
643
+ total = 0
644
+ Dir.glob(File.join(path, "**", "*")).each do |p|
645
+ total += File.size(p) if File.file?(p)
646
+ end
647
+ total
648
+ end
649
+
650
+ # Parse resource identifier into components.
651
+ #
652
+ # @param resource_id [String] The resource identifier (e.g., "en:spelling" or "en:fasttext")
653
+ # @return [Array<String>, nil] Array of parts or nil if invalid
654
+ def parse_resource_id(resource_id)
655
+ parts = resource_id.split(":")
656
+ return nil unless parts.size == 2
657
+
658
+ parts
659
+ end
660
+
661
+ # Extract language code from resource identifier.
662
+ #
663
+ # @param resource_id [String] The resource identifier
664
+ # @return [String, nil] Language code or nil if invalid
665
+ def extract_language(resource_id)
666
+ parts = parse_resource_id(resource_id)
667
+ return nil unless parts
668
+
669
+ parts[0]
670
+ end
671
+
672
+ # Extract resource type from resource identifier.
673
+ #
674
+ # @param resource_id [String] The resource identifier
675
+ # @return [String, nil] Resource type or nil if invalid
676
+ def extract_type(resource_id)
677
+ parts = parse_resource_id(resource_id)
678
+ return nil unless parts
679
+
680
+ parts[1]
681
+ end
682
+
683
+ # Default cache path: $XDG_CACHE_HOME/kotoshu
684
+ #
685
+ # @return [String] Default cache path
686
+ def default_cache_path
687
+ Kotoshu::Paths.cache_path
688
+ end
689
+
690
+ # Default URL base.
691
+ #
692
+ # @return [String] Default URL base
693
+ def default_url_base
694
+ Kotoshu::SourceRegistry::DEFAULT_BASE_URL
695
+ end
696
+
697
+ # Default source registry — pulls from global Configuration so
698
+ # ENV (KOTOSHU_REPOS_BASE_URL, KOTOSHU_DICTIONARIES_PIN, etc.)
699
+ # and programmatic config reach the cache layer automatically.
700
+ #
701
+ # @return [Kotoshu::SourceRegistry]
702
+ def default_source_registry
703
+ Kotoshu::Configuration.instance.source_registry
704
+ end
705
+
706
+ # Registry for an explicitly-passed url_base. The deprecated
707
+ # `url_base:` constructor parameter predates SourceRegistry; when
708
+ # it is given it must still drive downloads, so derive a registry
709
+ # rooted at that base instead of silently ignoring it.
710
+ #
711
+ # @param url_base [String, nil] Explicit base URL from the caller
712
+ # @return [Kotoshu::SourceRegistry]
713
+ def source_registry_for(url_base)
714
+ return default_source_registry unless url_base
715
+
716
+ Kotoshu::SourceRegistry.new(base_url: url_base)
717
+ end
718
+
719
+ # Default GitHub URL.
720
+ #
721
+ # @return [String] Default GitHub URL
722
+ def default_github_url
723
+ "https://github.com/kotoshu"
724
+ end
725
+
726
+ # Default cache TTL (7 days).
727
+ #
728
+ # @return [Integer] Default TTL in seconds
729
+ def default_cache_ttl
730
+ 604_800
731
+ end
732
+
733
+ # Default maximum cache size — pulled from
734
+ # +Kotoshu::Configuration.instance.max_cache_size+ so ENV
735
+ # overrides (+KOTOSHU_MAX_CACHE_SIZE+) and programmatic settings
736
+ # flow through to {#evict} naturally.
737
+ #
738
+ # @return [Integer] Default max cache size in bytes
739
+ def default_max_cache_size
740
+ Kotoshu::Configuration.instance.max_cache_size
741
+ end
742
+
743
+ # Build a default audit log bound to the configured rotation
744
+ # policy. Pulls +audit_max_bytes+ and +audit_rotations+ from
745
+ # +Kotoshu::Configuration.instance+ so ENV overrides
746
+ # (+KOTOSHU_AUDIT_MAX_BYTES+, +KOTOSHU_AUDIT_ROTATIONS+) and
747
+ # programmatic settings flow through naturally.
748
+ #
749
+ # @return [Kotoshu::Integrity::AuditLog]
750
+ def default_audit_log
751
+ config = Kotoshu::Configuration.instance
752
+ policy = Kotoshu::Integrity::RotationPolicy.new(
753
+ max_bytes: config.audit_max_bytes,
754
+ rotations: config.audit_rotations
755
+ )
756
+ Kotoshu::Integrity::AuditLog.new(rotation_policy: policy)
757
+ end
758
+
759
+ public
760
+
761
+ # Read metadata from file.
762
+ #
763
+ # Public because the CLI cache subcommand and other inspection
764
+ # paths consume it to display cache state without going through
765
+ # the full download-and-load pipeline.
766
+ #
767
+ # @param path [String] Metadata file path
768
+ # @return [Hash, nil] Metadata or nil
769
+ def read_metadata(path)
770
+ return nil unless File.exist?(path)
771
+
772
+ JSON.parse(File.read(path))
773
+ rescue JSON::ParserError
774
+ nil
775
+ end
776
+ end
777
+ end
778
+ end