kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,1147 @@
1
+ # frozen_string_literal: true
2
+
3
+ # base_cache autoloaded via Kotoshu::Cache
4
+ require "zlib"
5
+ require "open-uri"
6
+ require "open3"
7
+
8
+ module Kotoshu
9
+ module Cache
10
+ # Manages embedding model downloads from FastText CDN and GitHub.
11
+ #
12
+ # Extends BaseCache to support FastText .vec files and ONNX models.
13
+ # Downloads FastText models from Facebook's public CDN.
14
+ #
15
+ # @example Downloading a FastText model
16
+ # cache = ModelCache.new
17
+ # vec_file = cache.get_fasttext_model('en')
18
+ # model = FastTextModel.from_file(vec_file)
19
+ #
20
+ # @example Downloading an ONNX model
21
+ # onnx_file = cache.get_onnx_model('en')
22
+ class ModelCache < BaseCache
23
+ # Available models in FastText CDN and models-fasttext-onnx repository
24
+ AVAILABLE_MODELS = {
25
+ # FastText crawl vectors (300D) from Facebook Research
26
+ # https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/
27
+ # Selected high-resource languages
28
+ fasttext: {
29
+ de: { file: "cc.de.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
30
+ en: { file: "cc.en.300.vec.gz", size: 2_000_000, source: "FastText Common Crawl" },
31
+ es: { file: "cc.es.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
32
+ fr: { file: "cc.fr.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
33
+ pt: { file: "cc.pt.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
34
+ ru: { file: "cc.ru.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" }
35
+ },
36
+ # ONNX models (active set) from models-fasttext-onnx repository.
37
+ # Sizes synced with manifest.json in kotoshu/models-fasttext-onnx.
38
+ # The repo holds .onnx for 158 languages but only the 9 below are
39
+ # tracked and exposed — to promote a language, see
40
+ # models-fasttext-onnx/.gitignore and re-sync this constant.
41
+ # https://github.com/kotoshu/models-fasttext-onnx
42
+ onnx: {
43
+ de: { file: "fasttext.de.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
44
+ en: { file: "fasttext.en.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
45
+ es: { file: "fasttext.es.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
46
+ fr: { file: "fasttext.fr.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
47
+ pt: { file: "fasttext.pt.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
48
+ ru: { file: "fasttext.ru.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
49
+ zh: { file: "fasttext.zh.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
50
+ ja: { file: "fasttext.ja.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
51
+ ko: { file: "fasttext.ko.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
52
+ }
53
+ }.freeze
54
+
55
+ # Model tiers published by kotoshu/models-fasttext-onnx (registry
56
+ # ids `kotoshu://models/{lang}/{tier}`). `full` is today's layout
57
+ # and behavior; `fluency` and `mini` are the tiered tiers resolved
58
+ # through the registry (TODO.impl/67 M1).
59
+ TIERS = %i[full fluency mini].freeze
60
+
61
+ # First line of a Git LFS pointer stub — what raw.githubusercontent.com
62
+ # serves for LFS-tracked files (*.onnx) instead of the object content.
63
+ # Used to detect pointer-stub downloads/caches by content: the legacy
64
+ # download path records the checksum of whatever it downloaded, so a
65
+ # pointer stub passes checksum verification forever.
66
+ LFS_POINTER_PREFIX = "version https://git-lfs.github.com/spec/v1"
67
+
68
+ # Deterministic tier preference order, smallest footprint first.
69
+ # This is NOT a fallback chain — tiers are never silently
70
+ # substituted. It only fixes the ordering in which cached tiers
71
+ # are reported (e.g. ambiguity errors from `tier: :any`), so
72
+ # output is stable across machines and runs.
73
+ TIER_PREFERENCE = %i[mini fluency full].freeze
74
+
75
+ # The language-identification artifact pair (plan 106): registry
76
+ # resource `kotoshu://models/lid/lid-176` — the ONNX-converted
77
+ # lid.176 model (int8-per-row, dim 16) plus its vocab.json
78
+ # sidecar. Lives in the same registry as the embedding tiers but
79
+ # is NOT a language tier: it has no language of its own and the
80
+ # tier machinery (resource ids, TIERS) does not apply. It is
81
+ # stored under a top-level `lid/` directory (not
82
+ # `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
83
+ # reporting per-language ids only.
84
+ LID_LANGUAGE = "lid"
85
+ LID_TIER = "lid-176"
86
+ LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
87
+
88
+ # Directory (under the cache root) holding the LID artifact pair.
89
+ LID_DIRECTORY = "lid"
90
+
91
+ # Normalize and validate a tier name.
92
+ #
93
+ # @param tier [String, Symbol] "full", "fluency", or "mini"
94
+ # @return [Symbol] the normalized tier symbol
95
+ # @raise [ArgumentError] for any other value (covers config
96
+ # `model_tier` / `KOTOSHU_MODEL_TIER` and caller typos)
97
+ def self.normalize_tier(tier)
98
+ symbol = tier.to_s.downcase.strip.to_sym
99
+ unless TIERS.include?(symbol)
100
+ raise ArgumentError,
101
+ "unknown model tier #{tier.inspect} (expected one of: #{TIERS.join(', ')})"
102
+ end
103
+
104
+ symbol
105
+ end
106
+
107
+ # Get or download FastText model for a language.
108
+ #
109
+ # @param language_code [String] ISO 639-1 language code
110
+ # @param force_download [Boolean] Force re-download
111
+ # @return [String, nil] Path to downloaded .vec file
112
+ def get_fasttext_model(language_code, force_download: false)
113
+ resource_id = "#{language_code}:fasttext"
114
+ result = get(resource_id, force_download: force_download)
115
+
116
+ result&.dig(:model_path)
117
+ end
118
+
119
+ # Get or download ONNX model for a language.
120
+ #
121
+ # @param language_code [String] ISO 639-1 language code
122
+ # @param force_download [Boolean] Force re-download
123
+ # @return [String, nil] Path to downloaded .onnx file
124
+ def get_onnx_model(language_code, force_download: false)
125
+ resource_id = "#{language_code}:onnx"
126
+ result = get(resource_id, force_download: force_download)
127
+
128
+ result&.dig(:model_path)
129
+ end
130
+
131
+ # ---- Tier-aware model resources (TODO.impl/67 M1) ----
132
+
133
+ # Resource identifier for a (language, tier) model.
134
+ #
135
+ # The `full` tier maps to today's two-part id ("{lang}:onnx") and
136
+ # therefore to today's on-disk layout — backwards compatible.
137
+ # `fluency`/`mini` get a third path segment so tiers coexist:
138
+ #
139
+ # models/{lang}/models/onnx/ # full (unchanged)
140
+ # models/{lang}/models/onnx/mini/ # mini
141
+ # models/{lang}/models/onnx/fluency/ # fluency
142
+ #
143
+ # @param language_code [String] ISO 639-1 language code
144
+ # @param tier [String, Symbol] one of {TIERS}
145
+ # @return [String] resource id ("en:onnx" or "en:onnx:mini")
146
+ def tier_resource_id(language_code, tier)
147
+ tier = self.class.normalize_tier(tier)
148
+ lang = language_code.to_s
149
+ tier == :full ? "#{lang}:onnx" : "#{lang}:onnx:#{tier}"
150
+ end
151
+
152
+ # Whether a resource id carries a tier segment ("{lang}:onnx:{tier}").
153
+ #
154
+ # @param resource_id [String] resource identifier
155
+ # @return [Boolean] true for well-formed tiered ids
156
+ def tiered_resource_id?(resource_id)
157
+ parts = resource_id.to_s.split(":")
158
+ return false unless parts.size == 3
159
+
160
+ _lang, type, tier = parts
161
+ type == "onnx" && TIERS.include?(tier.to_sym)
162
+ end
163
+
164
+ # Tier of a tiered resource id ("en:onnx:mini" -> :mini).
165
+ # Returns nil for legacy two-part ids.
166
+ #
167
+ # @param resource_id [String] resource identifier
168
+ # @return [Symbol, nil]
169
+ def tier_from_resource_id(resource_id)
170
+ return nil unless tiered_resource_id?(resource_id)
171
+
172
+ resource_id.split(":")[2].to_sym
173
+ end
174
+
175
+ # Model tiers currently cached for a language (disk-only lookup;
176
+ # never touches the network, so it is resolve-safe).
177
+ #
178
+ # The legacy layout (`{lang}/models/onnx/`) counts as :full.
179
+ #
180
+ # @param language_code [String] ISO 639-1 language code
181
+ # @return [Array<Symbol>] cached tiers in {TIER_PREFERENCE} order
182
+ def cached_tiers(language_code)
183
+ lang = language_code.to_s
184
+ onnx_dir = File.join(@cache_path, lang, "models", "onnx")
185
+ TIER_PREFERENCE.select do |tier|
186
+ dir = tier == :full ? onnx_dir : File.join(onnx_dir, tier.to_s)
187
+ File.exist?(File.join(dir, "metadata.json"))
188
+ end
189
+ end
190
+
191
+ # Look up the registry entry for a (language, tier) model.
192
+ #
193
+ # Reads the cached registry.json; downloads it once (per cache
194
+ # instance) when absent or stale — unless offline mode is active,
195
+ # in which case a stale cached copy is reused and a missing one
196
+ # raises instead of ever hitting the network.
197
+ #
198
+ # @param language_code [String] ISO 639-1 language code
199
+ # @param tier [String, Symbol] one of {TIERS}
200
+ # @param force [Boolean] re-fetch the registry first
201
+ # @return [ModelRegistry::Resource, nil] nil when the registry has
202
+ # no entry for the pair
203
+ # @raise [Kotoshu::Error] when offline with no cached registry
204
+ def registry_entry_for(language_code, tier, force: false)
205
+ registry(force: force)&.find(language_code.to_s, tier.to_s)
206
+ end
207
+
208
+ # Download and verify a tiered model from the registry.
209
+ #
210
+ # Downloads `urls.primary`, falling back to `urls.mirror`, plus the
211
+ # vocab sibling; verifies the model's SHA-256 against the registry
212
+ # entry before accepting it into the cache. Unlike {#get}, download
213
+ # failures and checksum mismatches RAISE rather than degrading to
214
+ # nil — setup callers rely on that to report :unavailable.
215
+ #
216
+ # @param language_code [String] ISO 639-1 language code
217
+ # @param tier [String, Symbol] "mini" or "fluency" (":full" is
218
+ # delegated to the legacy download path)
219
+ # @param force_download [Boolean] re-fetch the registry first
220
+ # @return [Hash] { model_path:, metadata:, vocab_path? }
221
+ # @raise [Kotoshu::Error] no registry entry / offline without cache
222
+ # @raise [Kotoshu::IntegrityError] SHA-256 mismatch (corrupt bytes
223
+ # are removed before raising)
224
+ def download_tiered_model(language_code, tier:, force_download: false)
225
+ tier = self.class.normalize_tier(tier)
226
+ lang = language_code.to_s
227
+ return get("#{lang}:onnx", force_download: force_download) if tier == :full
228
+
229
+ entry = registry_entry_for(lang, tier, force: force_download)
230
+ unless entry
231
+ raise Kotoshu::Error,
232
+ "no registry entry for #{ModelRegistry::Resource.id_for(lang, tier)}"
233
+ end
234
+
235
+ resource_id = tier_resource_id(lang, tier)
236
+ dest_path = resource_dir_for(resource_id)
237
+ FileUtils.mkdir_p(dest_path)
238
+
239
+ filename = filename_from_url(entry.urls.primary)
240
+ model_file = File.join(dest_path, filename)
241
+ used_url = download_primary_or_mirror!(entry, model_file)
242
+
243
+ verify_registry_sha256!(entry, model_file, resource_id, used_url)
244
+
245
+ vocab_path = download_tiered_vocab(entry, dest_path)
246
+
247
+ metadata = {
248
+ version: entry.version,
249
+ url: used_url,
250
+ language: lang,
251
+ type: "onnx",
252
+ tier: tier.to_s,
253
+ file: filename,
254
+ checksum: entry.sha256,
255
+ registry_id: ModelRegistry::Resource.id_for(lang, tier),
256
+ size_bytes: entry.size_bytes,
257
+ cached_at: Time.now.utc.iso8601,
258
+ source: "registry"
259
+ }
260
+ write_metadata(File.join(dest_path, "metadata.json"), metadata)
261
+
262
+ result = { model_path: model_file, metadata: metadata }
263
+ result[:vocab_path] = vocab_path if vocab_path
264
+ result
265
+ end
266
+
267
+ # ---- LID artifact pair (plan 106) ----
268
+
269
+ # Whether the LID artifact pair is cached and complete.
270
+ #
271
+ # @return [Boolean]
272
+ def lid_cached?
273
+ !load_cached_lid.nil?
274
+ end
275
+
276
+ # Resolve the LID artifact pair from the cache only — never
277
+ # touches the network (the resolve half of the two-stage model).
278
+ #
279
+ # @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
280
+ # cached, nil when not set up
281
+ # @raise [Kotoshu::IntegrityError] cached bytes fail their
282
+ # recorded checksum
283
+ def load_cached_lid
284
+ dir = File.join(@cache_path, LID_DIRECTORY)
285
+ metadata_path = File.join(dir, "metadata.json")
286
+ return nil unless File.exist?(metadata_path)
287
+
288
+ metadata = read_metadata(metadata_path)
289
+ return nil unless metadata
290
+
291
+ onnx_file = File.join(dir, metadata["file"].to_s)
292
+ vocab_file = File.join(dir, metadata["vocab_file"].to_s)
293
+ return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
294
+ return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
295
+
296
+ verify_lid_integrity!(metadata, onnx_file, vocab_file)
297
+
298
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
299
+ end
300
+
301
+ # Download the LID artifact pair from the models registry
302
+ # (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
303
+ # against the registry sha256 before accepting them (the setup
304
+ # half of the two-stage model; same contract as
305
+ # {#download_tiered_model}).
306
+ #
307
+ # @param force [Boolean] re-fetch the registry first
308
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
309
+ # @raise [Kotoshu::Error] no registry entry, or offline without
310
+ # a cached registry (the registry is fetched once per cache
311
+ # instance; see {#registry})
312
+ # @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
313
+ # bytes are removed before raising)
314
+ def download_lid(force: false)
315
+ entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
316
+ unless entry
317
+ raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
318
+ end
319
+
320
+ dir = File.join(@cache_path, LID_DIRECTORY)
321
+ FileUtils.mkdir_p(dir)
322
+
323
+ onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
324
+ used_url = download_primary_or_mirror!(entry, onnx_file)
325
+ verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
326
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
327
+
328
+ # The vocab sidecar is required, not optional: the reader
329
+ # cannot score without its labels and Huffman counts.
330
+ vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
331
+ begin
332
+ download_file(entry.vocab_url, vocab_file)
333
+ rescue StandardError => e
334
+ File.delete(onnx_file) if File.exist?(onnx_file)
335
+ raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
336
+ end
337
+
338
+ metadata = {
339
+ version: entry.version,
340
+ url: used_url,
341
+ language: LID_LANGUAGE,
342
+ type: "lid",
343
+ tier: LID_TIER,
344
+ file: File.basename(onnx_file),
345
+ vocab_file: File.basename(vocab_file),
346
+ checksum: entry.sha256,
347
+ registry_id: LID_REGISTRY_ID,
348
+ size_bytes: entry.size_bytes,
349
+ cached_at: Time.now.utc.iso8601,
350
+ source: "registry"
351
+ }
352
+ write_metadata(File.join(dir, "metadata.json"), metadata)
353
+
354
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
355
+ end
356
+
357
+ # Get available model types for a language.
358
+ #
359
+ # @param language_code [String] ISO 639-1 language code
360
+ # @return [Array<Symbol>] Available model types (:fasttext, :onnx)
361
+ def available_models_for(language_code)
362
+ lang = language_code.to_sym
363
+ types = []
364
+ types << :fasttext if AVAILABLE_MODELS[:fasttext][lang]
365
+ types << :onnx if AVAILABLE_MODELS[:onnx][lang]
366
+ types
367
+ end
368
+
369
+ # Get model info for a language and type.
370
+ #
371
+ # @param language_code [String] ISO 639-1 language code
372
+ # @param model_type [Symbol] Model type (:fasttext, :onnx)
373
+ # @return [Hash, nil] Model info or nil if not available
374
+ def model_info(language_code, model_type)
375
+ AVAILABLE_MODELS.dig(model_type, language_code.to_sym)
376
+ end
377
+
378
+ # List all available models across all languages.
379
+ #
380
+ # @return [Hash] Mapping of language to available model types
381
+ def all_available_models
382
+ AVAILABLE_MODELS
383
+ end
384
+
385
+ # Check if a resource type is supported.
386
+ #
387
+ # @param resource_id [String] The resource identifier (e.g., "en:fasttext")
388
+ # @return [Boolean] True if supported
389
+ def supports_resource?(resource_id)
390
+ return true if tiered_resource_id?(resource_id)
391
+
392
+ parts = resource_id.split(":")
393
+ return false unless parts.size == 2
394
+
395
+ language, type = parts
396
+ AVAILABLE_MODELS[type.to_sym]&.key?(language.to_sym)
397
+ end
398
+
399
+ # List all cached resources.
400
+ #
401
+ # Tiered entries appear as "{lang}:onnx:{tier}"; the registry
402
+ # storage directory is not a model resource and is skipped.
403
+ #
404
+ # @return [Array<String>] List of cached resource identifiers
405
+ def cached_resources
406
+ Dir.glob(File.join(@cache_path, "**", "metadata.json")).filter_map do |path|
407
+ relative = Pathname.new(path).relative_path_from(Pathname.new(@cache_path))
408
+ parts = relative.to_s.split("/")
409
+ next if parts.first == "registry" # registry storage, not a model
410
+
411
+ case parts.length
412
+ when 4 then "#{parts[0]}:#{parts[2]}" # lang/models/type/metadata.json (full)
413
+ when 5 then "#{parts[0]}:#{parts[2]}:#{parts[3]}" # lang/models/type/tier/metadata.json
414
+ end
415
+ end.uniq
416
+ end
417
+
418
+ # Load cached resource data (implements abstract method).
419
+ #
420
+ # Public — this is the cache-only reader BaseCache declares
421
+ # publicly; resolve paths use it to read without any download.
422
+ #
423
+ # Verifies the cached model file's SHA-256 against the checksum
424
+ # recorded at download time. A mismatch — truncated file, disk
425
+ # corruption, manual edit — raises {Kotoshu::IntegrityError} with
426
+ # a remediation hint pointing at the cache subcommand, matching
427
+ # the Phase C contract of TODO.impl/38-onnx-semantic-gating.md.
428
+ # Caches written before checksums were recorded are accepted as
429
+ # "unverified" (graceful degradation).
430
+ #
431
+ # @param resource_id [String] The resource identifier
432
+ # @return [Hash, nil] Loaded model info
433
+ # @raise [Kotoshu::IntegrityError] if the cached file's checksum
434
+ # does not match the recorded checksum
435
+ def load_cached(resource_id)
436
+ return load_tiered_cached(resource_id) if tiered_resource_id?(resource_id)
437
+
438
+ language = extract_language(resource_id)
439
+ type = extract_type(resource_id)
440
+ return nil unless language && type
441
+
442
+ model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
443
+ return nil unless model_info
444
+
445
+ metadata_path = metadata_path_for(resource_id)
446
+ return nil unless File.exist?(metadata_path)
447
+
448
+ metadata = read_metadata(metadata_path)
449
+ return nil unless metadata
450
+
451
+ # For .gz files, the decompressed version is stored without .gz extension
452
+ filename = model_info[:file].sub('.gz', '')
453
+ model_file = File.join(resource_dir_for(resource_id), filename)
454
+
455
+ return nil unless File.exist?(model_file)
456
+
457
+ verify_cached_integrity!(resource_id, metadata, model_file)
458
+
459
+ { model_path: model_file, metadata: metadata }
460
+ end
461
+
462
+ protected
463
+
464
+ # Download a specific resource (implements abstract method).
465
+ #
466
+ # @param resource_id [String] The resource identifier
467
+ # @param dest_path [String] Destination directory
468
+ # @return [Hash] Downloaded model info
469
+ def download_resource(resource_id, dest_path)
470
+ if tiered_resource_id?(resource_id)
471
+ download_tiered_model(extract_language(resource_id),
472
+ tier: tier_from_resource_id(resource_id))
473
+ else
474
+ download_legacy_resource(resource_id, dest_path)
475
+ end
476
+ end
477
+
478
+ # Download a legacy (untiered) resource: FastText .vec or the
479
+ # full-tier ONNX from the pinned git tree. This is today's
480
+ # behavior, unchanged.
481
+ #
482
+ # @param resource_id [String] The resource identifier
483
+ # @param dest_path [String] Destination directory
484
+ # @return [Hash] Downloaded model info
485
+ def download_legacy_resource(resource_id, dest_path)
486
+ language = extract_language(resource_id)
487
+ type = extract_type(resource_id)
488
+ return nil unless language && type
489
+
490
+ model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
491
+ return nil unless model_info
492
+
493
+ FileUtils.mkdir_p(dest_path)
494
+
495
+ filename = model_info[:file]
496
+
497
+ # Handle ONNX with try-download-first approach
498
+ if type == "onnx"
499
+ download_or_convert_onnx(language, dest_path, filename)
500
+ else
501
+ # Handle FastText download (existing logic)
502
+ url = model_url(language, type, filename)
503
+
504
+ # Remove .gz extension for final storage (we decompress gzip files)
505
+ final_filename = filename.sub('.gz', '')
506
+ model_file = File.join(dest_path, final_filename)
507
+
508
+ # Download (and decompress if needed)
509
+ if url.end_with?('.gz')
510
+ download_and_decompress(url, model_file)
511
+ else
512
+ download_file(url, model_file)
513
+ end
514
+
515
+ # Save metadata
516
+ metadata = build_model_metadata(language, type, final_filename, url, model_file)
517
+ write_metadata(File.join(dest_path, "metadata.json"), metadata)
518
+
519
+ { model_path: model_file, metadata: metadata }
520
+ end
521
+ end
522
+
523
+ # Load a cached tiered model. The file recorded in metadata.json is
524
+ # verified against the checksum recorded at download time (which,
525
+ # for registry downloads, is the registry's sha256).
526
+ #
527
+ # @param resource_id [String] Tiered resource identifier
528
+ # @return [Hash, nil] Loaded model info
529
+ def load_tiered_cached(resource_id)
530
+ metadata_path = metadata_path_for(resource_id)
531
+ return nil unless File.exist?(metadata_path)
532
+
533
+ metadata = read_metadata(metadata_path)
534
+ return nil unless metadata
535
+
536
+ model_file = File.join(resource_dir_for(resource_id), metadata["file"].to_s)
537
+ return nil unless File.exist?(model_file) && File.size(model_file).positive?
538
+
539
+ verify_cached_integrity!(resource_id, metadata, model_file)
540
+
541
+ result = { model_path: model_file, metadata: metadata }
542
+ vocab = File.join(resource_dir_for(resource_id), "fasttext.#{extract_language(resource_id)}.#{tier_from_resource_id(resource_id)}.vocab.json")
543
+ result[:vocab_path] = vocab if File.exist?(vocab)
544
+ result
545
+ end
546
+
547
+ # Get metadata file path for a resource.
548
+ #
549
+ # @param resource_id [String] The resource identifier
550
+ # @return [String] Metadata file path
551
+ def metadata_path_for(resource_id)
552
+ File.join(resource_dir_for(resource_id), "metadata.json")
553
+ end
554
+
555
+ # Get resource directory path. Tiered ids nest under the onnx dir
556
+ # (`{lang}/models/onnx/{tier}`); everything else keeps today's
557
+ # layout (`{lang}/models/{type}`).
558
+ #
559
+ # @param resource_id [String] The resource identifier
560
+ # @return [String] Resource directory path
561
+ def resource_dir_for(resource_id)
562
+ if tiered_resource_id?(resource_id)
563
+ language, _type, tier = resource_id.split(":")
564
+ return File.join(@cache_path, language, "models", "onnx", tier)
565
+ end
566
+
567
+ language = extract_language(resource_id)
568
+ type = extract_type(resource_id)
569
+ File.join(@cache_path, language, "models", type)
570
+ end
571
+
572
+ # Check if all resource files exist.
573
+ #
574
+ # @param resource_id [String] The resource identifier
575
+ # @return [Boolean] True if all files exist
576
+ def resource_files_exist?(resource_id)
577
+ if tiered_resource_id?(resource_id)
578
+ metadata = read_metadata(metadata_path_for(resource_id))
579
+ return false unless metadata
580
+
581
+ model_file = File.join(resource_dir_for(resource_id), metadata["file"].to_s)
582
+ return File.exist?(model_file) && File.size(model_file).positive?
583
+ end
584
+
585
+ language = extract_language(resource_id)
586
+ type = extract_type(resource_id)
587
+ return false unless language && type
588
+
589
+ model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
590
+ return false unless model_info
591
+
592
+ # For .gz files, check the decompressed version
593
+ filename = model_info[:file].sub('.gz', '')
594
+ model_file = File.join(resource_dir_for(resource_id), filename)
595
+ File.exist?(model_file) && File.size(model_file).positive?
596
+ end
597
+
598
+ private
599
+
600
+ # Parse resource identifier into components. Tiered model ids
601
+ # carry a third segment ("en:onnx:mini"); all others stay
602
+ # two-part. Overridden from BaseCache so {#extract_language}
603
+ # works for tiered ids.
604
+ #
605
+ # @param resource_id [String] The resource identifier
606
+ # @return [Array<String>, nil] Array of parts or nil if invalid
607
+ def parse_resource_id(resource_id)
608
+ parts = resource_id.to_s.split(":")
609
+ return nil unless [2, 3].include?(parts.size)
610
+
611
+ parts
612
+ end
613
+
614
+ # ---- Registry storage ----
615
+ #
616
+ # registry.json is downloaded ONCE per cache instance (and reused
617
+ # across setups while fresh). Its bytes are stored under the cache
618
+ # dir with a metadata.json recording their own sha256, so a
619
+ # corrupted or tampered registry is detected and re-fetched rather
620
+ # than trusted. Offline mode (KOTOSHU_OFFLINE=1) never fetches:
621
+ # a cached copy — even a stale one — is used, and a missing one
622
+ # raises. Resolve never reaches this code at all (cache-only).
623
+
624
+ # The parsed registry, fetched from cache or network at most once
625
+ # per cache instance.
626
+ #
627
+ # @param force [Boolean] re-fetch even if a fresh copy is cached
628
+ # @return [ModelRegistry]
629
+ # @raise [Kotoshu::Error] offline with no cached registry
630
+ def registry(force: false)
631
+ return @registry = load_registry(force: true) if force
632
+
633
+ @registry ||= load_registry(force: false)
634
+ end
635
+
636
+ def load_registry(force:)
637
+ cached = force ? nil : read_cached_registry
638
+ return cached if cached && !expired?(registry_metadata_path)
639
+
640
+ if offline?
641
+ return cached if cached
642
+
643
+ raise Kotoshu::Error,
644
+ "offline mode (KOTOSHU_OFFLINE=1) and no cached registry at " \
645
+ "#{registry_path}; run once online or copy registry.json there"
646
+ end
647
+
648
+ fetch_registry
649
+ end
650
+
651
+ # Read and sha-verify the cached registry. Returns nil when the
652
+ # bytes are absent, unparsable, or fail their recorded checksum.
653
+ def read_cached_registry
654
+ return nil unless File.exist?(registry_path) && File.exist?(registry_metadata_path)
655
+
656
+ bytes = File.binread(registry_path)
657
+ metadata = read_metadata(registry_metadata_path)
658
+ expected = metadata && metadata["sha256"]
659
+ return nil unless expected
660
+ return nil unless Digest::SHA256.hexdigest(bytes) == expected
661
+
662
+ ModelRegistry.from_json(bytes)
663
+ rescue StandardError
664
+ nil
665
+ end
666
+
667
+ def fetch_registry
668
+ url = @source_registry.url_for(:model_registry)
669
+ bytes = Kotoshu::Integrity::NetHTTP.get(url)
670
+ raise Kotoshu::Error, "registry not found at #{url}" if bytes.nil?
671
+
672
+ begin
673
+ model = ModelRegistry.from_json(bytes)
674
+ rescue StandardError => e
675
+ raise Kotoshu::IntegrityError.new(
676
+ "registry.json",
677
+ expected: "<valid registry JSON>",
678
+ actual: "<parse error: #{e.message}>",
679
+ url: url
680
+ )
681
+ end
682
+
683
+ FileUtils.mkdir_p(File.dirname(registry_path))
684
+ File.binwrite(registry_path, bytes)
685
+ write_metadata(registry_metadata_path,
686
+ "url" => url,
687
+ "sha256" => Digest::SHA256.hexdigest(bytes),
688
+ "cached_at" => Time.now.utc.iso8601)
689
+ model
690
+ end
691
+
692
+ def registry_path
693
+ File.join(@cache_path, "registry", "registry.json")
694
+ end
695
+
696
+ def registry_metadata_path
697
+ File.join(@cache_path, "registry", "metadata.json")
698
+ end
699
+
700
+ def offline?
701
+ Kotoshu.configuration.offline
702
+ end
703
+
704
+ # ---- Tiered download helpers ----
705
+
706
+ # Download from urls.primary, falling back to urls.mirror. Partial
707
+ # bytes from a failed attempt are removed first. Returns the URL
708
+ # that succeeded; re-raises the last error when both fail.
709
+ def download_primary_or_mirror!(entry, model_file)
710
+ urls = [entry.urls.primary, entry.urls.mirror].compact.uniq
711
+ last_error = nil
712
+ urls.each do |url|
713
+ File.delete(model_file) if File.exist?(model_file)
714
+ download_file(url, model_file)
715
+ return url
716
+ rescue StandardError => e
717
+ last_error = e
718
+ warn " download failed (#{url}): #{e.message}" if $VERBOSE
719
+ end
720
+
721
+ File.delete(model_file) if File.exist?(model_file)
722
+ raise last_error
723
+ end
724
+
725
+ # Whether a file is a Git LFS pointer stub rather than real content.
726
+ # Reads only the first line's prefix, so it stays cheap on 120MB models.
727
+ def lfs_pointer?(path)
728
+ File.open(path, "rb") do |f|
729
+ f.read(LFS_POINTER_PREFIX.bytesize) == LFS_POINTER_PREFIX
730
+ end
731
+ rescue StandardError
732
+ false
733
+ end
734
+
735
+ # Verify downloaded model bytes against the registry's sha256.
736
+ # Corrupt bytes are removed from disk before raising so the next
737
+ # attempt re-downloads (same contract as BaseCache#verify_and_audit).
738
+ # +remediation+ overrides the default re-download hint (the LID
739
+ # pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
740
+ def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
741
+ actual = Digest::SHA256.file(model_file).hexdigest
742
+ if actual == entry.sha256
743
+ @audit_log.record(
744
+ url: url, status: "verified", size: File.size(model_file),
745
+ sha256: actual, manifest_sha256: entry.sha256, resource_id: resource_id
746
+ )
747
+ return
748
+ end
749
+
750
+ @audit_log.record(
751
+ url: url, status: "mismatch", size: File.size(model_file),
752
+ sha256: actual, manifest_sha256: entry.sha256, resource_id: resource_id
753
+ )
754
+ File.delete(model_file)
755
+ raise Kotoshu::IntegrityError.new(
756
+ resource_id,
757
+ expected: entry.sha256,
758
+ actual: actual,
759
+ url: url,
760
+ remediation: remediation ||
761
+ "Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
762
+ )
763
+ end
764
+
765
+ # Pull the vocab sibling for a tiered model. Missing vocab is a
766
+ # warning, not a failure (mirrors the legacy full-tier behavior).
767
+ def download_tiered_vocab(entry, dest_path)
768
+ return nil if entry.vocab_url.to_s.empty?
769
+
770
+ vocab_file = File.join(dest_path, filename_from_url(entry.vocab_url))
771
+ begin
772
+ download_file(entry.vocab_url, vocab_file)
773
+ vocab_file
774
+ rescue StandardError => e
775
+ warn " vocab.json unavailable: #{e.message}" if $VERBOSE
776
+ nil
777
+ end
778
+ end
779
+
780
+ # Basename of a URL's path, e.g.
781
+ # "https://h/fasttext.en.mini.onnx" => "fasttext.en.mini.onnx".
782
+ def filename_from_url(url)
783
+ File.basename(URI.parse(url.to_s).path)
784
+ end
785
+
786
+ # Verify a cached model file's SHA-256 against the checksum
787
+ # recorded in its metadata. Raises {Kotoshu::IntegrityError} on
788
+ # mismatch so the caller surfaces a clear, actionable error.
789
+ # Caches written without a checksum field are accepted silently
790
+ # to preserve backward compatibility with pre-verification caches.
791
+ #
792
+ # Verify the cached LID pair against the checksum recorded at
793
+ # download time (which, for registry downloads, is the registry's
794
+ # sha256). A Git LFS pointer stub is refused by content first —
795
+ # same contract as {#verify_cached_integrity!}, against the
796
+ # fixed LID paths instead of a resource-id-derived layout.
797
+ #
798
+ # @param metadata [Hash] Parsed metadata.json (string keys)
799
+ # @param onnx_file [String] Path to the cached lid.176.onnx
800
+ # @param vocab_file [String] Path to the cached vocab sidecar
801
+ # @return [void]
802
+ # @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
803
+ # corrupt bytes are removed so the next setup re-downloads
804
+ def verify_lid_integrity!(metadata, onnx_file, vocab_file)
805
+ dir = File.join(@cache_path, LID_DIRECTORY)
806
+ if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
807
+ File.delete(onnx_file)
808
+ File.delete(vocab_file)
809
+ FileUtils.rm_f(File.join(dir, "metadata.json"))
810
+ raise Kotoshu::Error,
811
+ "Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
812
+ "artifact is a Git LFS pointer stub, not model content. Run " \
813
+ "Kotoshu.setup_lid(force: true) to re-download."
814
+ end
815
+
816
+ expected = metadata["checksum"]
817
+ return unless expected
818
+
819
+ actual = Digest::SHA256.file(onnx_file).hexdigest
820
+ return if actual == expected
821
+
822
+ raise Kotoshu::IntegrityError.new(
823
+ LID_REGISTRY_ID,
824
+ expected: expected,
825
+ actual: actual,
826
+ url: metadata["url"],
827
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
828
+ )
829
+ end
830
+
831
+ # @param resource_id [String] The resource identifier (e.g. "en:onnx")
832
+ # @param metadata [Hash] Parsed metadata.json (string keys)
833
+ # @param model_file [String] Path to the cached model file
834
+ # @return [void]
835
+ # @raise [Kotoshu::IntegrityError] if checksums do not match
836
+ #
837
+ def verify_cached_integrity!(resource_id, metadata, model_file)
838
+ if lfs_pointer?(model_file)
839
+ # Content, not checksum, betrays the stub: the legacy download
840
+ # path recorded the pointer's own sha256. Purge both files so
841
+ # the next get re-downloads (same contract as the tiered path).
842
+ File.delete(model_file)
843
+ metadata_file = metadata_path_for(resource_id)
844
+ File.delete(metadata_file) if File.exist?(metadata_file)
845
+ raise Kotoshu::Error,
846
+ "Integrity verification failed for #{resource_id}: cached model " \
847
+ "#{File.basename(model_file)} is a Git LFS pointer stub, not " \
848
+ "model content (raw.githubusercontent.com serves pointer stubs " \
849
+ "for LFS-tracked .onnx files). Run " \
850
+ "`kotoshu cache download :#{extract_language(resource_id)} --model` to re-download."
851
+ end
852
+
853
+ expected = metadata["checksum"]
854
+ return unless expected
855
+
856
+ actual = Digest::SHA256.file(model_file).hexdigest
857
+ return if actual == expected
858
+
859
+ remediation = "Run `kotoshu cache download :#{extract_language(resource_id)} --model` to re-download."
860
+ raise Kotoshu::IntegrityError.new(
861
+ resource_id,
862
+ expected: expected,
863
+ actual: actual,
864
+ url: metadata["url"],
865
+ remediation: remediation
866
+ )
867
+ end
868
+
869
+ # Build metadata hash for a model.
870
+ #
871
+ # @param language [String] Language code
872
+ # @param type [String] Model type
873
+ # @param filename [String] Model filename
874
+ # @param url [String] Download URL
875
+ # @param model_file [String] Path to downloaded model file
876
+ # @return [Hash] Metadata hash
877
+ def build_model_metadata(language, type, filename, url, model_file)
878
+ {
879
+ version: Time.now.utc.iso8601,
880
+ url: url,
881
+ language: language,
882
+ type: type,
883
+ file: filename,
884
+ checksum: Digest::SHA256.file(model_file).hexdigest,
885
+ cached_at: Time.now.utc.iso8601
886
+ }
887
+ end
888
+
889
+ # Get URL for a model file.
890
+ #
891
+ # @param language [String] Language code
892
+ # @param type [String] Model type
893
+ # @param filename [String] Model filename
894
+ # @return [String, nil] Download URL
895
+ def model_url(language, type, filename)
896
+ case type
897
+ when "fasttext"
898
+ # Download from FastText CDN (Facebook Research)
899
+ # https://fasttext.cc/docs/en/english-vectors.html
900
+ "https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/#{filename}"
901
+ when "onnx"
902
+ # Download from models-fasttext-onnx GitHub repository.
903
+ # SourceRegistry owns the per-repo pin so we never accidentally
904
+ # fall back to the dictionaries pin.
905
+ @source_registry.url_for(:model, lang: language)
906
+ else
907
+ "#{@url_base}/dictionaries/main/#{language}/models/#{type}/#{filename}"
908
+ end
909
+ end
910
+
911
+ # URL for the vocab.json sibling file. The conversion script ships
912
+ # vocabularies alongside the .onnx so OnnxModel.from_file can resolve
913
+ # word→index without re-parsing the FastText .vec.
914
+ #
915
+ # @param language [String] Language code
916
+ # @return [String]
917
+ def vocab_url(language)
918
+ @source_registry.url_for(:model_vocab, lang: language)
919
+ end
920
+
921
+ # Download and decompress gzip file.
922
+ #
923
+ # @param url [String] URL to gzip file
924
+ # @param dest_path [String] Destination path (without .gz)
925
+ def download_and_decompress(url, dest_path)
926
+ # Download to temporary file first
927
+ temp_gz = "#{dest_path}.gz"
928
+
929
+ puts " Downloading from #{url.split('/').last}..." if $VERBOSE
930
+
931
+ downloaded_bytes = 0
932
+ URI.open(url, open_timeout: 30, read_timeout: 300) do |uri|
933
+ File.open(temp_gz, 'wb') do |f|
934
+ downloaded_bytes = f.write(uri.read)
935
+ end
936
+ end
937
+
938
+ puts " Downloaded: #{(downloaded_bytes.to_f / 1024 / 1024).round(2)} MB" if $VERBOSE
939
+
940
+ # Verify the download succeeded
941
+ unless File.exist?(temp_gz) && File.size(temp_gz).positive?
942
+ raise "Download failed: #{temp_gz} is empty or missing"
943
+ end
944
+
945
+ puts " Decompressing..." if $VERBOSE
946
+
947
+ # Remove existing file if present (handles partial downloads)
948
+ File.delete(dest_path) if File.exist?(dest_path)
949
+
950
+ # Decompress gzip with streaming
951
+ File.open(temp_gz, 'rb') do |gz_file|
952
+ Zlib::GzipReader.wrap(gz_file) do |gzip|
953
+ # Stream in chunks to avoid memory issues with large files
954
+ File.open(dest_path, 'wb') do |out_file|
955
+ chunk_size = 65_536 # 64KB chunks
956
+ while (chunk = gzip.read(chunk_size))
957
+ out_file.write(chunk)
958
+ # Print progress every 10MB
959
+ if $VERBOSE && out_file.pos % (10 * 1024 * 1024) < chunk_size
960
+ puts " Decompressed: #{(out_file.pos.to_f / 1024 / 1024).round(1)} MB..."
961
+ end
962
+ end
963
+ end
964
+ end
965
+ end
966
+
967
+ # Verify the decompression succeeded
968
+ unless File.exist?(dest_path) && File.size(dest_path).positive?
969
+ raise "Decompression failed: #{dest_path} is empty or missing"
970
+ end
971
+
972
+ # Clean up gz file
973
+ File.delete(temp_gz)
974
+
975
+ puts " ✓ Downloaded and decompressed" if $VERBOSE
976
+ end
977
+
978
+ # Convert FastText .vec file to ONNX format.
979
+ #
980
+ # @param language [String] Language code
981
+ # @param dest_path [String] Destination directory
982
+ # @param onnx_filename [String] Output ONNX filename
983
+ # @return [Hash] Converted model info
984
+ def convert_to_onnx(language, dest_path, onnx_filename)
985
+ puts "Converting FastText to ONNX for #{language}..." if $VERBOSE
986
+
987
+ # First, ensure we have the FastText .vec file
988
+ fasttext_resource_id = "#{language}:fasttext"
989
+ fasttext_result = get(fasttext_resource_id, force_download: false)
990
+
991
+ unless fasttext_result
992
+ raise "Failed to get FastText model for #{language} needed for ONNX conversion"
993
+ end
994
+
995
+ vec_file = fasttext_result[:model_path]
996
+
997
+ # Verify the .vec file exists
998
+ unless File.exist?(vec_file)
999
+ raise "FastText .vec file not found: #{vec_file}"
1000
+ end
1001
+
1002
+ # Output ONNX file path
1003
+ onnx_file = File.join(dest_path, onnx_filename)
1004
+
1005
+ # Get the conversion script path
1006
+ script_path = File.expand_path('../scripts/fasttext_to_onnx.py', __dir__)
1007
+
1008
+ unless File.exist?(script_path)
1009
+ raise "ONNX conversion script not found: #{script_path}"
1010
+ end
1011
+
1012
+ # Build conversion command
1013
+ # Use --vocab-size to limit vocabulary size and reduce conversion time
1014
+ vocab_size = fasttext_result.dig(:metadata, "vocab_size")&.to_i || 100_000
1015
+
1016
+ cmd = [
1017
+ 'python3',
1018
+ script_path,
1019
+ vec_file,
1020
+ onnx_file,
1021
+ '--vocab-size', vocab_size.to_s
1022
+ ]
1023
+
1024
+ puts " Running conversion: #{shell_join(cmd)}" if $VERBOSE
1025
+
1026
+ # Run conversion
1027
+ require 'open3'
1028
+ stdout, stderr, status = Open3.capture3(*cmd)
1029
+
1030
+ unless status.success?
1031
+ raise "ONNX conversion failed:\n#{stdout}\n#{stderr}"
1032
+ end
1033
+
1034
+ puts stdout if $VERBOSE
1035
+
1036
+ # Build metadata for the ONNX file
1037
+ metadata = {
1038
+ version: Time.now.utc.iso8601,
1039
+ url: "converted:#{vec_file}",
1040
+ language: language,
1041
+ type: "onnx",
1042
+ file: onnx_filename,
1043
+ checksum: Digest::SHA256.file(onnx_file).hexdigest,
1044
+ cached_at: Time.now.utc.iso8601,
1045
+ source_model: File.basename(vec_file),
1046
+ conversion_method: "fasttext_to_onnx.py"
1047
+ }
1048
+
1049
+ # Save metadata
1050
+ write_metadata(File.join(dest_path, "metadata.json"), metadata)
1051
+
1052
+ puts " ✓ ONNX conversion complete" if $VERBOSE
1053
+
1054
+ { model_path: onnx_file, metadata: metadata }
1055
+ end
1056
+
1057
+ # Try to download ONNX from GitHub, fall back to conversion if download fails.
1058
+ #
1059
+ # @param language [String] Language code
1060
+ # @param dest_path [String] Destination directory
1061
+ # @param onnx_filename [String] ONNX filename
1062
+ # @return [Hash] Downloaded or converted model info
1063
+ def download_or_convert_onnx(language, dest_path, onnx_filename)
1064
+ url = model_url(language, "onnx", onnx_filename)
1065
+ onnx_file = File.join(dest_path, onnx_filename)
1066
+
1067
+ puts " Attempting download from GitHub..." if $VERBOSE
1068
+
1069
+ # Try downloading from GitHub first
1070
+ begin
1071
+ download_file(url, onnx_file)
1072
+
1073
+ # Verify the downloaded file
1074
+ unless File.exist?(onnx_file) && File.size(onnx_file).positive?
1075
+ raise "Download failed: empty file"
1076
+ end
1077
+
1078
+ # raw.githubusercontent.com serves LFS pointer stubs for
1079
+ # LFS-tracked .onnx files; refuse one so the fallback path runs
1080
+ # instead of poisoning the cache with a stub.
1081
+ if lfs_pointer?(onnx_file)
1082
+ raise "downloaded a Git LFS pointer stub instead of model bytes from #{url}"
1083
+ end
1084
+
1085
+ # Pull the matching vocab.json so OnnxModel.from_file can resolve
1086
+ # word→index without re-parsing the source FastText .vec.
1087
+ begin
1088
+ download_file(vocab_url(language),
1089
+ File.join(dest_path, "fasttext.#{language}.vocab.json"))
1090
+ rescue StandardError => e
1091
+ warn " vocab.json unavailable for #{language}: #{e.message}" if $VERBOSE
1092
+ end
1093
+
1094
+ puts " ✓ Downloaded from GitHub" if $VERBOSE
1095
+
1096
+ # Build metadata for downloaded file
1097
+ metadata = {
1098
+ version: Time.now.utc.iso8601,
1099
+ url: url,
1100
+ language: language,
1101
+ type: "onnx",
1102
+ file: onnx_filename,
1103
+ checksum: Digest::SHA256.file(onnx_file).hexdigest,
1104
+ cached_at: Time.now.utc.iso8601,
1105
+ source: "github"
1106
+ }
1107
+
1108
+ # Save metadata
1109
+ write_metadata(File.join(dest_path, "metadata.json"), metadata)
1110
+
1111
+ { model_path: onnx_file, metadata: metadata }
1112
+ rescue StandardError => e
1113
+ puts " GitHub download failed: #{e.message}" if $VERBOSE
1114
+ puts " Falling back to local conversion..." if $VERBOSE
1115
+
1116
+ # Remove partial download if any
1117
+ File.delete(onnx_file) if File.exist?(onnx_file)
1118
+
1119
+ # Fall back to local conversion
1120
+ convert_to_onnx(language, dest_path, onnx_filename)
1121
+ end
1122
+ end
1123
+
1124
+ # Join shell command arguments safely (for display purposes).
1125
+ #
1126
+ # @param args [Array<String>] Command arguments
1127
+ # @return [String] Joined command string
1128
+ def shell_join(args)
1129
+ args.map { |a| /\s/.match?(a) ? "'#{a}'" : a }.join(' ')
1130
+ end
1131
+
1132
+ # Default cache path: $XDG_CACHE_HOME/kotoshu/models
1133
+ #
1134
+ # @return [String] Default cache path
1135
+ def default_cache_path
1136
+ File.join(Kotoshu::Paths.cache_path, "models")
1137
+ end
1138
+
1139
+ # Default cache TTL (30 days for models).
1140
+ #
1141
+ # @return [Integer] Default TTL in seconds
1142
+ def default_cache_ttl
1143
+ 2_592_000 # 30 days
1144
+ end
1145
+ end
1146
+ end
1147
+ end