kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,328 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Embeddings
5
+ # Similarity search for embedding-based nearest neighbor lookup.
6
+ #
7
+ # Efficiently finds semantically similar words using cosine similarity.
8
+ # Supports both on-the-fly computation and pre-computed embedding matrices.
9
+ #
10
+ # @example Basic usage
11
+ # search = SimilaritySearch.new(
12
+ # vocabulary: vocab,
13
+ # model: model
14
+ # )
15
+ # neighbors = search.find_nearest('hello', k: 10)
16
+ #
17
+ # @example With pre-loaded embedding matrix (faster)
18
+ # search = SimilaritySearch.new(
19
+ # vocabulary: vocab,
20
+ # model: model,
21
+ # preload_embeddings: true
22
+ # )
23
+ # neighbors = search.find_nearest('hello', k: 10)
24
+ class SimilaritySearch
25
+ # @return [Vocabulary] The vocabulary
26
+ attr_reader :vocabulary
27
+
28
+ # @return [OnnxRuntimeModel] The ONNX model
29
+ attr_reader :model
30
+
31
+ # @return [Boolean] Whether embeddings are pre-loaded
32
+ attr_reader :embeddings_loaded
33
+
34
+ # Create a new similarity search instance.
35
+ #
36
+ # @param vocabulary [Vocabulary] Word vocabulary
37
+ # @param model [OnnxRuntimeModel] ONNX model for embeddings
38
+ # @param preload_embeddings [Boolean] Whether to preload all embeddings
39
+ # @param max_cache_size [Integer] Maximum embeddings to cache (if not preloading)
40
+ def initialize(vocabulary:, model:, preload_embeddings: false, max_cache_size: 1000)
41
+ @vocabulary = vocabulary
42
+ @model = model
43
+ @preload_embeddings = preload_embeddings
44
+ @max_cache_size = max_cache_size
45
+
46
+ # Embedding cache (word -> vector)
47
+ @embedding_cache = {}
48
+
49
+ # Pre-loaded embedding matrix (for faster search)
50
+ @embedding_matrix = nil
51
+
52
+ # Track whether embeddings are preloaded
53
+ @embeddings_loaded = false
54
+
55
+ # Load embeddings if requested
56
+ preload_embeddings! if preload_embeddings
57
+ end
58
+
59
+ # Find k nearest neighbors for a word.
60
+ #
61
+ # @param query_word [String] The query word
62
+ # @param k [Integer] Number of neighbors to return
63
+ # @param exclude_self [Boolean] Whether to exclude the query word itself
64
+ # @param min_similarity [Float] Minimum similarity threshold (0.0 to 1.0)
65
+ # @return [Array<Hash>] Array of {word, similarity} hashes
66
+ def find_nearest(query_word, k: 10, exclude_self: true, min_similarity: 0.0)
67
+ # Get query embedding
68
+ query_vec = get_embedding(query_word)
69
+ return [] unless query_vec
70
+
71
+ # Find neighbors
72
+ if @embedding_matrix
73
+ nearest_from_matrix(query_vec, k, exclude_self, min_similarity)
74
+ else
75
+ nearest_from_cache(query_vec, k, exclude_self, min_similarity)
76
+ end
77
+ end
78
+
79
+ # Find k nearest neighbors for multiple words.
80
+ #
81
+ # @param query_words [Array<String>] Query words
82
+ # @param k [Integer] Number of neighbors per word
83
+ # @return [Hash<String, Array<Hash>>] Word to neighbors mapping
84
+ def find_nearest_batch(query_words, k: 10)
85
+ query_words.each_with_object({}) do |word, result|
86
+ result[word] = find_nearest(word, k: k)
87
+ end
88
+ end
89
+
90
+ # Compute similarity between two words.
91
+ #
92
+ # @param word1 [String] First word
93
+ # @param word2 [String] Second word
94
+ # @return [Float] Cosine similarity (-1.0 to 1.0, or nil if either word not found)
95
+ def similarity(word1, word2)
96
+ vec1 = get_embedding(word1)
97
+ vec2 = get_embedding(word2)
98
+
99
+ return nil unless vec1 && vec2
100
+
101
+ cosine_similarity(vec1, vec2)
102
+ end
103
+
104
+ # Compute similarity between two embedding vectors.
105
+ #
106
+ # @param vec1 [Array<Float>] First vector
107
+ # @param vec2 [Array<Float>] Second vector
108
+ # @return [Float] Cosine similarity (-1.0 to 1.0)
109
+ def cosine_similarity(vec1, vec2)
110
+ return 0.0 if vec1.nil? || vec2.nil?
111
+
112
+ # Compute dot product
113
+ dot = vec1.zip(vec2).sum { |a, b| a * b }
114
+
115
+ # Compute magnitudes
116
+ norm1 = Math.sqrt(vec1.sum { |x| x * x })
117
+ norm2 = Math.sqrt(vec2.sum { |x| x * x })
118
+
119
+ return 0.0 if norm1.zero? || norm2.zero?
120
+
121
+ dot / (norm1 * norm2)
122
+ end
123
+
124
+ # Preload all embeddings into memory for faster search.
125
+ #
126
+ # @return [Boolean] True if loaded successfully
127
+ def preload_embeddings!
128
+ return false if @embedding_matrix
129
+
130
+ # Get all indices
131
+ all_indices = (0...@vocabulary.size).to_a
132
+
133
+ # Batch load embeddings
134
+ vectors = @model.get_embeddings(all_indices)
135
+ return false if vectors.nil? || vectors.empty?
136
+
137
+ # Store as hash for now (could use Numo::SFloat for efficiency)
138
+ @embedding_matrix = {}
139
+ all_indices.zip(vectors).each do |idx, vec|
140
+ @embedding_matrix[idx] = vec
141
+ end
142
+
143
+ @embeddings_loaded = true
144
+ true
145
+ rescue StandardError => e
146
+ warn "Failed to preload embeddings: #{e.message}"
147
+ false
148
+ end
149
+
150
+ # Clear the embedding cache.
151
+ #
152
+ # @return [self] Self for chaining
153
+ def clear_cache
154
+ @embedding_cache.clear
155
+ @embedding_matrix = nil
156
+ @embeddings_loaded = false
157
+ self
158
+ end
159
+
160
+ # Get cache statistics.
161
+ #
162
+ # @return [Hash] Cache statistics
163
+ def cache_stats
164
+ stats = {
165
+ size: @embedding_cache.size,
166
+ max_size: @max_cache_size
167
+ }
168
+ stats[:hit_rate] = @cache_hits.to_f / (@cache_hits + @cache_misses) if defined?(@cache_hits)
169
+ stats
170
+ end
171
+
172
+ # String representation.
173
+ #
174
+ # @return [String] String representation
175
+ def to_s
176
+ "SimilaritySearch(vocab_size: #{@vocabulary.size}, loaded: #{@embeddings_loaded})"
177
+ end
178
+ alias inspect to_s
179
+
180
+ private
181
+
182
+ # Get embedding for a word (with caching).
183
+ #
184
+ # @param word [String] The word
185
+ # @return [Array<Float>, nil] Embedding vector or nil if not found
186
+ def get_embedding(word)
187
+ # Check cache first
188
+ if @embedding_cache.key?(word)
189
+ @cache_hits += 1 if defined?(@cache_hits)
190
+ return @embedding_cache[word]
191
+ end
192
+
193
+ @cache_misses ||= 0
194
+ @cache_hits ||= 0
195
+ @cache_misses += 1
196
+
197
+ # Get from model
198
+ index = @vocabulary.lookup(word)
199
+ return nil unless index
200
+
201
+ vec = if @embedding_matrix
202
+ @embedding_matrix[index]
203
+ else
204
+ @model.get_embedding(index)
205
+ end
206
+
207
+ return nil unless vec
208
+
209
+ # Cache if not preloading (preload has all in memory already)
210
+ unless @preload_embeddings
211
+ # Evict oldest if cache is full
212
+ if @embedding_cache.size >= @max_cache_size
213
+ @embedding_cache.shift
214
+ end
215
+ @embedding_cache[word] = vec
216
+ end
217
+
218
+ vec
219
+ end
220
+
221
+ # Find nearest neighbors using pre-loaded matrix.
222
+ #
223
+ # @param query_vec [Array<Float>] Query embedding
224
+ # @param k [Integer] Number of neighbors
225
+ # @param exclude_self [Boolean] Whether to exclude exact matches
226
+ # @param min_similarity [Float] Minimum similarity
227
+ # @return [Array<Hash>] Nearest neighbors
228
+ def nearest_from_matrix(query_vec, k, exclude_self, min_similarity)
229
+ similarities = []
230
+
231
+ @vocabulary.words.each do |word|
232
+ index = @vocabulary.lookup(word)
233
+ vec = @embedding_matrix[index]
234
+
235
+ next unless vec
236
+
237
+ sim = cosine_similarity(query_vec, vec)
238
+
239
+ # Skip exact match if requested
240
+ next if exclude_self && sim >= 0.9999
241
+
242
+ # Skip below threshold
243
+ next if sim < min_similarity
244
+
245
+ similarities << { word: word, similarity: sim }
246
+ end
247
+
248
+ # Sort by similarity (descending) and take top k
249
+ similarities.sort_by { |s| -s[:similarity] }.first(k)
250
+ end
251
+
252
+ # Find nearest neighbors using cache (no pre-loading).
253
+ #
254
+ # @param query_vec [Array<Float>] Query embedding
255
+ # @param k [Integer] Number of neighbors
256
+ # @param exclude_self [Boolean] Whether to exclude exact matches
257
+ # @param min_similarity [Float] Minimum similarity
258
+ # @return [Array<Hash>] Nearest neighbors
259
+ def nearest_from_cache(query_vec, k, exclude_self, min_similarity)
260
+ similarities = []
261
+
262
+ # Sample from vocabulary for efficiency (or use common words)
263
+ sample_words = sample_vocabulary(k * 10)
264
+
265
+ sample_words.each do |word|
266
+ vec = get_embedding(word)
267
+ next unless vec
268
+
269
+ sim = cosine_similarity(query_vec, vec)
270
+
271
+ # Skip exact match if requested
272
+ next if exclude_self && sim >= 0.9999
273
+
274
+ # Skip below threshold
275
+ next if sim < min_similarity
276
+
277
+ similarities << { word: word, similarity: sim }
278
+ end
279
+
280
+ # Sort by similarity (descending) and take top k
281
+ similarities.sort_by { |s| -s[:similarity] }.first(k)
282
+ end
283
+
284
+ # Sample words from vocabulary for search.
285
+ #
286
+ # Prioritizes common words (first N in vocabulary).
287
+ #
288
+ # @param n [Integer] Number of words to sample
289
+ # @return [Array<String>] Sampled words
290
+ def sample_vocabulary(n)
291
+ # Use first N words (FastText orders by frequency)
292
+ # plus a random sample of the rest
293
+ common_size = [n / 2, 100].min
294
+ random_size = n - common_size
295
+
296
+ common = @vocabulary.common_words(n: common_size)
297
+
298
+ if @vocabulary.size > common_size
299
+ # Get a random sample from the rest
300
+ rest = @vocabulary.words.drop(common_size)
301
+ random_sample = rest.sample(random_size)
302
+ common + random_sample
303
+ else
304
+ common
305
+ end
306
+ end
307
+
308
+ # Create from cache.
309
+ #
310
+ # @param language_code [String] ISO 639-1 language code
311
+ # @param cache [Cache::ModelCache, nil] Optional cache instance
312
+ # @param preload [Boolean] Whether to preload embeddings
313
+ # @return [SimilaritySearch, nil] New search instance or nil if not available
314
+ def self.from_cache(language_code, cache: nil, preload: false)
315
+ vocab = Vocabulary.from_cache(language_code, cache: cache)
316
+ model = OnnxRuntimeModel.from_cache(language_code, cache: cache)
317
+
318
+ return nil unless vocab && model
319
+
320
+ new(
321
+ vocabulary: vocab,
322
+ model: model,
323
+ preload_embeddings: preload
324
+ )
325
+ end
326
+ end
327
+ end
328
+ end
@@ -0,0 +1,306 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ # Vocabulary - Word to index mapping
6
+ #
7
+ # Provides efficient lookup from words to integer indices for embedding retrieval.
8
+ # Supports JSON file loading and saving.
9
+ #
10
+ # @example Creating a vocabulary
11
+ # vocab = Kotoshu::Embeddings::Vocabulary.new(
12
+ # language_code: 'en',
13
+ # word_to_index: { 'hello' => 0, 'world' => 1 }
14
+ # )
15
+ #
16
+ # @example Loading from file
17
+ # vocab = Kotoshu::Embeddings::Vocabulary.from_file('/path/to/vocab.json', language_code: 'en')
18
+ #
19
+ module Kotoshu
20
+ module Embeddings
21
+ class Vocabulary
22
+ include VocabularyProtocol
23
+
24
+ # @return [String] ISO 639-1 language code
25
+ attr_reader :language_code
26
+
27
+ # @return [Hash{String => Integer}] Word to index mapping
28
+ attr_reader :word_to_index
29
+
30
+ # @return [Array<String>] Index to word mapping (sparse array)
31
+ attr_reader :index_to_word
32
+
33
+ # Create a new vocabulary
34
+ #
35
+ # @param language_code [String] ISO 639-1 language code
36
+ # @param word_to_index [Hash{String => Integer}] Word to index mapping
37
+ #
38
+ # @raise [ArgumentError] If word_to_index is empty
39
+ #
40
+ def initialize(language_code:, word_to_index:)
41
+ if word_to_index.nil? || word_to_index.empty?
42
+ raise ArgumentError,
43
+ 'word_to_index cannot be empty'
44
+ end
45
+
46
+ @language_code = language_code
47
+ @word_to_index = word_to_index.dup.freeze
48
+
49
+ # Build reverse index (index -> word)
50
+ @index_to_word = Array.new(@word_to_index.size)
51
+ @word_to_index.each do |word, index|
52
+ unless index.is_a?(Integer)
53
+ raise ArgumentError,
54
+ "word_to_index must map words to Integer indices " \
55
+ "(#{word.inspect} => #{index.inspect})"
56
+ end
57
+
58
+ @index_to_word[index] = word if index < @index_to_word.size
59
+ end
60
+ @index_to_word.freeze
61
+ end
62
+
63
+ # Look up word index
64
+ #
65
+ # @param word [String] The word to look up
66
+ # @return [Integer, nil] Index of the word, or nil if not found
67
+ #
68
+ def lookup(word)
69
+ @word_to_index[word]
70
+ end
71
+
72
+ # Get word by index
73
+ #
74
+ # @param index [Integer] The index to look up
75
+ # @return [String, nil] Word at the index, or nil if not found
76
+ #
77
+ def get_word(index)
78
+ @index_to_word[index]
79
+ end
80
+
81
+ # Check if word exists in vocabulary
82
+ #
83
+ # @param word [String] Word to check
84
+ # @return [Boolean] True if word exists
85
+ #
86
+ def include?(word)
87
+ @word_to_index.key?(word)
88
+ end
89
+
90
+ # Get vocabulary size
91
+ #
92
+ # @return [Integer] Number of words in vocabulary
93
+ #
94
+ def size
95
+ @word_to_index.size
96
+ end
97
+
98
+ # Check if index is valid
99
+ #
100
+ # @param index [Integer] Index to check
101
+ # @return [Boolean] True if index is valid
102
+ #
103
+ def valid_index?(index)
104
+ index.is_a?(Integer) && index >= 0 && index < @word_to_index.size
105
+ end
106
+
107
+ # Get common/most frequent words
108
+ #
109
+ # @param n [Integer] Number of words to return
110
+ # @return [Array<String>] Array of common words
111
+ #
112
+ def common_words(n: 10)
113
+ return [] if @word_to_index.empty?
114
+
115
+ @word_to_index.keys.first(n)
116
+ end
117
+
118
+ # Convert to Hash
119
+ #
120
+ # @return [Hash{String => Integer}] Copy of word_to_index mapping
121
+ #
122
+ def to_h
123
+ @word_to_index.dup
124
+ end
125
+
126
+ # Get all words as enumerator
127
+ #
128
+ # @return [Enumerator<String>] Enumerator of all words
129
+ #
130
+ def words
131
+ @word_to_index.each_key
132
+ end
133
+
134
+ # Load vocabulary from JSON file
135
+ #
136
+ # @param path [String] Path to JSON file
137
+ # @param language_code [String] Language code (auto-detected from filename if nil)
138
+ # @return [Vocabulary] New vocabulary instance
139
+ #
140
+ # @raise [ArgumentError] If file doesn't exist
141
+ # @raise [Json::ParserError] If file is not valid JSON
142
+ #
143
+ def self.from_file(path, language_code: nil)
144
+ raise ArgumentError, "File not found: #{path}" unless File.exist?(path)
145
+
146
+ language_code ||= detect_language_from_path(path)
147
+
148
+ data = JSON.parse(File.read(path))
149
+
150
+ case data
151
+ when Hash
152
+ word_to_index = word_to_index_from_document(data)
153
+ when Array
154
+ word_to_index = {}
155
+ data.each_with_index do |word, index|
156
+ word_to_index[word.freeze] = index
157
+ end
158
+ word_to_index.freeze
159
+ else
160
+ raise ArgumentError, 'Invalid vocabulary format: expected Hash or Array'
161
+ end
162
+
163
+ new(language_code: language_code, word_to_index: word_to_index)
164
+ end
165
+
166
+ # Normalize a parsed vocab.json document into a flat word-to-index hash.
167
+ #
168
+ # Two Hash shapes exist in the wild: a flat `{ "hello" => 0 }` map, and
169
+ # the wrapped `{ "vocab_size" => N, "word_to_idx" => { ... } }` document
170
+ # written by `scripts/fasttext_to_onnx.py` `save_vocabulary` and shipped
171
+ # by the models-fasttext-onnx repository. Both are accepted here so
172
+ # every vocab.json generation reads back regardless of which tool
173
+ # wrote it.
174
+ #
175
+ # @param document [Hash{String => Object}] Parsed JSON document
176
+ # @return [Hash{String => Integer}] Flat word-to-index mapping
177
+ def self.word_to_index_from_document(document)
178
+ inner = document['word_to_idx']
179
+ document = inner if inner.is_a?(Hash)
180
+
181
+ document.transform_keys(&:freeze)
182
+ end
183
+
184
+ # Create vocabulary from Array of words
185
+ #
186
+ # @param words [Array<String>] Array of words
187
+ # @param language_code [String] Language code
188
+ # @return [Vocabulary] New vocabulary instance
189
+ #
190
+ def self.from_words(words, language_code: 'en')
191
+ word_to_index = {}
192
+ words.each_with_index do |word, index|
193
+ word_to_index[word.freeze] = index
194
+ end
195
+ word_to_index.freeze
196
+
197
+ new(language_code: language_code, word_to_index: word_to_index)
198
+ end
199
+
200
+ # Save vocabulary to JSON file
201
+ #
202
+ # @param path [String] Path to save file
203
+ # @param format [Symbol] Format: :hash or :array
204
+ #
205
+ def save_to_file(path, format: :hash)
206
+ case format
207
+ when :hash
208
+ data = @word_to_index.dup
209
+ when :array
210
+ max_index = @index_to_word.compact.length
211
+ data = @index_to_word.compact.first(max_index)
212
+ else
213
+ raise ArgumentError, "Unknown format: #{format}"
214
+ end
215
+
216
+ File.write(path, JSON.pretty_generate(data))
217
+ end
218
+
219
+ # Check if vocabulary is empty
220
+ #
221
+ # @return [Boolean] True if empty
222
+ #
223
+ def empty?
224
+ @word_to_index.empty?
225
+ end
226
+
227
+ # Get a sample of words
228
+ #
229
+ # @param n [Integer] Number of words to sample
230
+ # @return [Array<String>] Sample of words
231
+ #
232
+ def sample(n: 10)
233
+ @word_to_index.keys.sample(n)
234
+ end
235
+
236
+ # Create a sub-vocabulary containing only specified words
237
+ #
238
+ # @param words [Array<String>] Words to include
239
+ # @return [Vocabulary] New vocabulary with subset of words
240
+ #
241
+ def sub_vocabulary(words)
242
+ filtered = @word_to_index.slice(*words)
243
+ self.class.new(language_code: @language_code, word_to_index: filtered)
244
+ end
245
+
246
+ # Find words starting with a prefix
247
+ #
248
+ # @param prefix [String] Prefix to match
249
+ # @return [Array<String>] Matching words
250
+ #
251
+ def words_starting_with(prefix)
252
+ pattern = /^#{Regexp.escape(prefix)}/
253
+ @word_to_index.keys.grep(pattern)
254
+ end
255
+
256
+ # String representation
257
+ #
258
+ # @return [String]
259
+ #
260
+ def to_s
261
+ "Vocabulary(language: #{@language_code}, size: #{@word_to_index.size})"
262
+ end
263
+ alias inspect to_s
264
+
265
+ # Detect language code from file path
266
+ #
267
+ # @param path [String] File path
268
+ # @return [String] Detected language code
269
+ #
270
+ def self.detect_language_from_path(path)
271
+ basename = File.basename(path)
272
+
273
+ if basename =~ /(\w+)\.vocab\.json\z/
274
+ return $1
275
+ end
276
+
277
+ if basename =~ /\.(\w+)\.vocab\.json\z/
278
+ return $1
279
+ end
280
+
281
+ 'unknown'
282
+ end
283
+
284
+ # Load vocabulary for a language from the model cache.
285
+ #
286
+ # Resolves the vocab.json sibling of the cached ONNX model. Returns nil
287
+ # when either the cache or the vocab file is unavailable so callers can
288
+ # degrade gracefully.
289
+ #
290
+ # @param language_code [String] ISO 639-1 language code
291
+ # @param cache [Kotoshu::Cache::ModelCache, nil] Optional cache instance
292
+ # @return [Vocabulary, nil]
293
+ def self.from_cache(language_code, cache: nil)
294
+ cache ||= Kotoshu::Cache::ModelCache.new
295
+ onnx_path = cache.get_onnx_model(language_code)
296
+ return nil unless onnx_path
297
+
298
+ vocab_path = File.join(File.dirname(onnx_path),
299
+ "fasttext.#{language_code}.vocab.json")
300
+ return nil unless File.file?(vocab_path)
301
+
302
+ from_file(vocab_path, language_code: language_code)
303
+ end
304
+ end
305
+ end
306
+ end