kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,411 @@
1
+ # frozen_string_literal: true
2
+
3
+ # OnnxRuntimeModel - ONNX Runtime wrapper for FastText embeddings
4
+ #
5
+ # Provides embedding inference using ONNX Runtime. Supports single lookups,
6
+ # batch inference, and vocabulary-aware operations.
7
+ #
8
+ # @example Single embedding lookup
9
+ # model = OnnxRuntimeModel.from_file('fasttext.en.onnx', language_code: 'en')
10
+ # model.load!
11
+ # embedding = model.get_embedding(1234)
12
+ #
13
+ # @example Batch lookup
14
+ # embeddings = model.get_embeddings([1, 2, 3, 4, 5])
15
+ #
16
+ # @example With vocabulary
17
+ # embedding = model.get_embedding_for_word('hello', vocabulary)
18
+ #
19
+ module Kotoshu
20
+ module Embeddings
21
+ class OnnxRuntimeModel
22
+ include EmbeddingModelProtocol
23
+
24
+ # Default dimension for FastText models
25
+ DEFAULT_DIMENSION = 300
26
+
27
+ # Batch size for batch inference
28
+ BATCH_SIZE = 32
29
+
30
+ # @return [String] Language code (ISO 639-1)
31
+ attr_reader :language_code
32
+
33
+ # @return [Integer] Embedding dimension
34
+ attr_reader :dimension
35
+
36
+ # @return [String] Path to ONNX model file
37
+ attr_reader :onnx_path
38
+
39
+ # @return [Boolean] Whether the model is loaded
40
+ def loaded?
41
+ @loaded
42
+ end
43
+
44
+ # @return [Integer] Number of inference calls
45
+ attr_reader :inference_count
46
+
47
+ # Create a new ONNX Runtime model
48
+ #
49
+ # @param language_code [String] ISO 639-1 language code
50
+ # @param onnx_path [String] Path to .onnx file
51
+ # @param dimension [Integer] Embedding dimension (default: 300)
52
+ #
53
+ def initialize(language_code:, onnx_path:, dimension: DEFAULT_DIMENSION)
54
+ @language_code = language_code
55
+ @onnx_path = onnx_path
56
+ @dimension = dimension
57
+ @session = nil
58
+ @loaded = false
59
+ @input_name = nil
60
+ @output_name = nil
61
+ @inference_count = 0
62
+ end
63
+
64
+ # Load the ONNX model into memory
65
+ #
66
+ # @return [self]
67
+ #
68
+ # @raise [Kotoshu::Models::OnnxUnavailable] if onnxruntime gem is missing
69
+ # @raise [ArgumentError] if model file doesn't exist
70
+ #
71
+ def load!
72
+ return self if @loaded
73
+
74
+ raise Kotoshu::Models::OnnxModel::OnnxUnavailable unless Kotoshu::Models::OnnxModel::ONNX_LOADED
75
+
76
+ unless File.exist?(@onnx_path)
77
+ raise ArgumentError,
78
+ "ONNX file not found: #{@onnx_path}"
79
+ end
80
+
81
+ begin
82
+ @session = OnnxRuntime::InferenceSession.new(@onnx_path)
83
+ rescue OnnxRuntime::Error => e
84
+ # Phase C of TODO.impl/38: a corrupt/truncated cached model must
85
+ # surface as an actionable error, not a raw "Protobuf parsing
86
+ # failed".
87
+ raise Kotoshu::Error,
88
+ "ONNX model failed to load (#{e.message}): #{@onnx_path}. " \
89
+ "The cached model may be corrupt or truncated — re-download " \
90
+ "it with `kotoshu cache download :#{@language_code} --model`."
91
+ end
92
+
93
+ # Detect input/output names
94
+ @input_name = detect_input_name
95
+ @output_name = detect_output_name
96
+
97
+ @loaded = true
98
+ self
99
+ end
100
+
101
+ # Unload the model from memory
102
+ #
103
+ # @return [self]
104
+ #
105
+ def unload!
106
+ @session = nil
107
+ @input_name = nil
108
+ @output_name = nil
109
+ @loaded = false
110
+ self
111
+ end
112
+
113
+ # Check if model is ready for inference
114
+ #
115
+ # @return [Boolean]
116
+ #
117
+ def ready?
118
+ @loaded && !@session.nil?
119
+ end
120
+
121
+ # Get embedding for a single word index
122
+ #
123
+ # @param index [Integer] Word index in vocabulary
124
+ # @return [Array<Float>] Embedding vector
125
+ #
126
+ # @raise [RuntimeError] if model is not loaded
127
+ # @raise [ArgumentError] if index is invalid
128
+ #
129
+ def get_embedding(index)
130
+ ensure_loaded
131
+
132
+ unless valid_index?(index)
133
+ raise ArgumentError,
134
+ "Invalid word index: #{index}"
135
+ end
136
+
137
+ output = @session.run(
138
+ [@output_name],
139
+ { @input_name => [index] },
140
+ )
141
+
142
+ @inference_count += 1
143
+
144
+ extract_embedding(output.first)
145
+ end
146
+
147
+ # Get embeddings for multiple indices (batched)
148
+ #
149
+ # More efficient than individual calls for batch operations.
150
+ #
151
+ # @param indices [Array<Integer>] Word indices
152
+ # @return [Array<Array<Float>>] Array of embedding vectors
153
+ #
154
+ def get_embeddings(indices)
155
+ ensure_loaded
156
+ return [] if indices.nil? || indices.empty?
157
+
158
+ valid_indices = indices.select { |i| valid_index?(i) }
159
+ return [] if valid_indices.empty?
160
+
161
+ # Process in batches for memory efficiency
162
+ valid_indices.each_slice(BATCH_SIZE).flat_map do |batch|
163
+ run_batch_inference(batch)
164
+ end
165
+ end
166
+
167
+ # Preload all embeddings into memory
168
+ #
169
+ # For small vocabularies, this provides O(1) lookup after loading.
170
+ #
171
+ # @param vocabulary [Vocabulary] Vocabulary with complete word list
172
+ # @return [Hash<Integer, Array<Float>>] Index to embedding mapping
173
+ #
174
+ def preload_embeddings!(vocabulary)
175
+ ensure_loaded
176
+
177
+ all_indices = (0...vocabulary.size).to_a
178
+ embeddings = get_embeddings(all_indices)
179
+
180
+ # Build index mapping
181
+ all_indices.zip(embeddings).to_h
182
+ end
183
+
184
+ # Get embedding for a word using vocabulary
185
+ #
186
+ # @param word [String] The word to lookup
187
+ # @param vocabulary [Vocabulary] Vocabulary for word-to-index mapping
188
+ # @return [Array<Float>, nil] Embedding vector or nil if word not found
189
+ #
190
+ def get_embedding_for_word(word, vocabulary)
191
+ index = vocabulary.lookup(word)
192
+ return nil unless index
193
+
194
+ get_embedding(index)
195
+ end
196
+
197
+ # Get embeddings for multiple words using vocabulary
198
+ #
199
+ # @param words [Array<String>] Words to lookup
200
+ # @param vocabulary [Vocabulary] Vocabulary for word-to-index mapping
201
+ # @return [Hash<String, Array<Float>>] Word to embedding mapping
202
+ #
203
+ def get_embeddings_for_words(words, vocabulary)
204
+ result = {}
205
+ words.each do |word|
206
+ embedding = get_embedding_for_word(word, vocabulary)
207
+ result[word] = embedding if embedding
208
+ end
209
+ result
210
+ end
211
+
212
+ # Check if batching is supported
213
+ #
214
+ # @return [Boolean]
215
+ #
216
+ def supports_batching?
217
+ true
218
+ end
219
+
220
+ # Get batch size for batch inference
221
+ #
222
+ # @return [Integer]
223
+ #
224
+ def batch_size
225
+ BATCH_SIZE
226
+ end
227
+
228
+ # Get model type identifier
229
+ #
230
+ # @return [String]
231
+ #
232
+ def model_type
233
+ 'onnx'
234
+ end
235
+
236
+ # Get model information
237
+ #
238
+ # @return [Hash]
239
+ #
240
+ def model_info
241
+ {
242
+ type: 'onnx',
243
+ language: @language_code,
244
+ dimension: @dimension,
245
+ path: @onnx_path,
246
+ loaded: @loaded,
247
+ inference_count: @inference_count,
248
+ }
249
+ end
250
+
251
+ # Create model from file
252
+ #
253
+ # @param onnx_path [String] Path to .onnx file
254
+ # @param language_code [String] Language code (auto-detected if nil)
255
+ # @param dimension [Integer] Embedding dimension
256
+ # @return [OnnxRuntimeModel]
257
+ #
258
+ def self.from_file(onnx_path, language_code: nil, dimension: nil)
259
+ unless File.exist?(onnx_path)
260
+ raise ArgumentError,
261
+ "ONNX file not found: #{onnx_path}"
262
+ end
263
+
264
+ language_code ||= detect_language_from_path(onnx_path)
265
+ dimension ||= DEFAULT_DIMENSION
266
+
267
+ new(
268
+ language_code: language_code,
269
+ onnx_path: onnx_path,
270
+ dimension: dimension,
271
+ )
272
+ end
273
+
274
+ # Create model from cache
275
+ #
276
+ # @param language_code [String] ISO 639-1 language code
277
+ # @param cache [Cache::ModelCache] Cache instance
278
+ # @return [OnnxRuntimeModel, nil]
279
+ #
280
+ def self.from_cache(language_code, cache: nil)
281
+ cache ||= Kotoshu::Cache::ModelCache.new
282
+
283
+ onnx_path = cache.get_onnx_model(language_code)
284
+ return nil unless onnx_path
285
+
286
+ from_file(onnx_path, language_code: language_code)
287
+ end
288
+
289
+ # String representation
290
+ #
291
+ # @return [String]
292
+ #
293
+ def to_s
294
+ "OnnxRuntimeModel(language: #{@language_code}, dimension: #{@dimension}, loaded: #{@loaded})"
295
+ end
296
+ alias inspect to_s
297
+
298
+ private
299
+
300
+ # Ensure model is loaded
301
+ #
302
+ def ensure_loaded
303
+ load! unless @loaded
304
+ end
305
+
306
+ # Check if index is valid
307
+ #
308
+ def valid_index?(index)
309
+ index.is_a?(Integer) && index >= 0
310
+ end
311
+
312
+ # Run batch inference for a batch of indices
313
+ #
314
+ # @param indices [Array<Integer>] Word indices
315
+ # @return [Array<Array<Float>>] Embedding vectors
316
+ #
317
+ def run_batch_inference(indices)
318
+ # Create input tensor
319
+ input_data = indices.flatten
320
+
321
+ output = @session.run(
322
+ [@output_name],
323
+ { @input_name => input_data },
324
+ )
325
+
326
+ @inference_count += 1
327
+
328
+ # Extract embeddings
329
+ result = output.first
330
+ if result.is_a?(Array)
331
+ result
332
+ else
333
+ # Handle OrtValue or other wrappers
334
+ Array.new(indices.length) { |i| extract_single_embedding(result, i) }
335
+ end
336
+ end
337
+
338
+ # Extract embedding from output
339
+ #
340
+ # @param output [Object] ONNX output
341
+ # @return [Array<Float>]
342
+ #
343
+ def extract_embedding(output)
344
+ case output
345
+ when Array
346
+ output
347
+ when NumpyArray, Numo::SFloat
348
+ output.to_a
349
+ when OnnxRuntime::OrtValue
350
+ output.to_a
351
+ else
352
+ # Idiomatic Array coercion: invokes to_ary / to_a when defined,
353
+ # otherwise wraps. Equivalent to the prior respond_to?(:to_a)
354
+ # branch without the duck-typing check.
355
+ Array(output)
356
+ end
357
+ end
358
+
359
+ # Extract single embedding from batch output
360
+ #
361
+ # @param output [Object] ONNX batch output
362
+ # @param index [Integer] Index in batch
363
+ # @return [Array<Float>]
364
+ #
365
+ def extract_single_embedding(output, index)
366
+ case output
367
+ when Array
368
+ output[index]
369
+ when NumpyArray, Numo::SFloat
370
+ output[index, true].to_a
371
+ else
372
+ # Default: assume array-like
373
+ output[index].to_a
374
+ end
375
+ end
376
+
377
+ # Detect input name from model
378
+ #
379
+ # @return [String]
380
+ #
381
+ def detect_input_name
382
+ inputs = @session.inputs
383
+ inputs&.first&.dig(:name) || 'word_index'
384
+ end
385
+
386
+ # Detect output name from model
387
+ #
388
+ # @return [String]
389
+ #
390
+ def detect_output_name
391
+ outputs = @session.outputs
392
+ outputs&.first&.dig(:name) || 'embedding'
393
+ end
394
+
395
+ # Detect language from file path
396
+ #
397
+ # @param path [String]
398
+ # @return [String]
399
+ #
400
+ def self.detect_language_from_path(path)
401
+ basename = File.basename(path)
402
+
403
+ if basename =~ /\.([a-z]{2})\./i
404
+ Regexp.last_match(1).downcase
405
+ else
406
+ 'en'
407
+ end
408
+ end
409
+ end
410
+ end
411
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+
5
+ module Kotoshu
6
+ module Embeddings
7
+ # Protocol - Ruby interface/contract system
8
+ #
9
+ # Provides a simple way to define interfaces with required and optional methods.
10
+ module Protocol
11
+ # Set of method names that conforming classes must implement.
12
+ # Populated via +required+ (the writer); read via +required_methods+.
13
+ def required_methods
14
+ @required_methods ||= Set.new
15
+ end
16
+
17
+ # Set of method names that conforming classes may optionally implement.
18
+ # Populated via +optional+ (the writer); read via +optional_methods+.
19
+ def optional_methods
20
+ @optional_methods ||= Set.new
21
+ end
22
+
23
+ # Declare one or more required methods on this protocol.
24
+ def required(*names)
25
+ names.each { |n| required_methods << n }
26
+ end
27
+
28
+ # Declare one or more optional methods on this protocol.
29
+ def optional(*names)
30
+ names.each { |n| optional_methods << n }
31
+ end
32
+
33
+ # Returns the subset of +required_methods+ that +klass+ does not
34
+ # implement as an instance method. Empty result means full
35
+ # conformance. +klass+ should be a Class (not an instance); we
36
+ # check +method_defined?+ because protocol-declared methods are
37
+ # mixed in via +include+, becoming instance methods of +klass+.
38
+ def compliance_errors(klass)
39
+ required_methods.reject { |m| klass.method_defined?(m) }
40
+ end
41
+
42
+ # Raise ProtocolError unless +klass+ implements every required method.
43
+ def assert_implemented_by!(klass)
44
+ errors = compliance_errors(klass)
45
+ return if errors.empty?
46
+
47
+ raise ProtocolError.new(klass, self, errors.to_a)
48
+ end
49
+ end
50
+
51
+ # Protocol error
52
+ class ProtocolError < StandardError
53
+ attr_reader :klass, :protocol, :missing_methods
54
+
55
+ def initialize(klass, protocol, missing_methods)
56
+ @klass = klass
57
+ @protocol = protocol
58
+ @missing_methods = missing_methods
59
+ super("#{klass} missing: #{missing_methods.join(', ')}")
60
+ end
61
+ end
62
+
63
+ # EmbeddingModel Protocol
64
+ module EmbeddingModelProtocol
65
+ extend Protocol
66
+
67
+ required :dimension, :language_code, :get_embedding, :get_embeddings
68
+ required :load!, :unload!, :loaded?, :ready?
69
+
70
+ optional :get_embeddings_batch, :batch_size, :preload_embeddings!
71
+ optional :supports_batching?, :model_type, :model_info
72
+ end
73
+
74
+ # SimilarityEngine Protocol
75
+ module SimilarityEngineProtocol
76
+ extend Protocol
77
+
78
+ required :cosine, :dot_product, :euclidean, :manhattan
79
+ required :pre_normalize, :normalize_and_compute
80
+
81
+ optional :cosine_batch, :compute_all_pairs
82
+ optional :is_normalized?, :normalization_required?
83
+ end
84
+
85
+ # Vocabulary Protocol
86
+ module VocabularyProtocol
87
+ extend Protocol
88
+
89
+ required :lookup, :get_word, :include?, :size, :words
90
+ required :valid_index?, :common_words, :to_h
91
+
92
+ optional :sample, :sub_vocabulary, :words_starting_with
93
+ optional :save_to_file, :language_code
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is intentionally a no-op stub.
4
+ #
5
+ # History: an earlier draft of the embeddings layer planned to split
6
+ # protocol definitions across `protocols/embedding_model.rb`,
7
+ # `protocols/similarity_engine.rb`, and `protocols/vocabulary.rb`.
8
+ # Those subfiles were never created; the protocols were inlined into
9
+ # `protocol.rb` (singular) instead, which is what the autoload table
10
+ # in `lib/kotoshu/embeddings.rb` points at.
11
+ #
12
+ # Loading the original version of this file raised LoadError because
13
+ # `require_relative 'protocols/embedding_model'` referenced a path
14
+ # that did not exist. Rather than delete the file (project policy
15
+ # preserves files contributors may have local context for), it is
16
+ # reduced to this stub so any future `require` of it succeeds
17
+ # harmlessly.
18
+ #
19
+ # For the actual protocol definitions see:
20
+ # Kotoshu::Embeddings::EmbeddingModelProtocol
21
+ # Kotoshu::Embeddings::SimilarityEngineProtocol
22
+ # Kotoshu::Embeddings::VocabularyProtocol
23
+ # all in lib/kotoshu/embeddings/protocol.rb.
@@ -0,0 +1,184 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Registry - Plugin system for embeddings components
4
+ #
5
+ # Provides a centralized registry for embedding models, similarity engines,
6
+ # and vocabularies. Allows registering and retrieving custom implementations.
7
+ #
8
+ # @example Registering a custom model
9
+ # Kotoshu::Embeddings::Registry.register_model(:my_model, MyCustomModel)
10
+ #
11
+ # @example Creating from registry
12
+ # model = Kotoshu::Embeddings::Registry.create_model(:my_model, vectors: my_vectors)
13
+ #
14
+ # @example Listing available implementations
15
+ # Kotoshu::Embeddings::Registry.models.keys # => [:onnx, :my_model]
16
+ #
17
+ module Kotoshu
18
+ module Embeddings
19
+ class Registry
20
+ class << self
21
+ # @return [Hash{Symbol => Class}] Registered models
22
+ attr_reader :models
23
+
24
+ # @return [Hash{Symbol => Class}] Registered engines
25
+ attr_reader :engines
26
+
27
+ # @return [Hash{Symbol => Class}] Registered vocabularies
28
+ attr_reader :vocabularies
29
+
30
+ # Initialize registry
31
+ #
32
+ def init
33
+ @models ||= {}
34
+ @engines ||= {}
35
+ @vocabularies ||= {}
36
+ end
37
+
38
+ # Register an embedding model
39
+ #
40
+ # @param name [Symbol] Model identifier
41
+ # @param klass [Class] Model class (must implement EmbeddingModel protocol)
42
+ #
43
+ def register_model(name, klass)
44
+ init
45
+ @models[name] = klass
46
+ klass
47
+ end
48
+
49
+ # Register a similarity engine
50
+ #
51
+ # @param name [Symbol] Engine identifier
52
+ # @param klass [Class] Engine class (must implement SimilarityEngine protocol)
53
+ #
54
+ def register_engine(name, klass)
55
+ init
56
+ @engines[name] = klass
57
+ klass
58
+ end
59
+
60
+ # Register a vocabulary
61
+ #
62
+ # @param name [Symbol] Vocabulary identifier
63
+ # @param klass [Class] Vocabulary class (must implement Vocabulary protocol)
64
+ #
65
+ def register_vocabulary(name, klass)
66
+ init
67
+ @vocabularies[name] = klass
68
+ klass
69
+ end
70
+
71
+ # Get registered model class
72
+ #
73
+ # @param name [Symbol] Model identifier
74
+ # @return [Class, nil]
75
+ #
76
+ def model(name)
77
+ init
78
+ @models[name]
79
+ end
80
+
81
+ # Get registered engine class
82
+ #
83
+ # @param name [Symbol] Engine identifier
84
+ # @return [Class, nil]
85
+ #
86
+ def engine(name)
87
+ init
88
+ @engines[name]
89
+ end
90
+
91
+ # Get registered vocabulary class
92
+ #
93
+ # @param name [Symbol] Vocabulary identifier
94
+ # @return [Class, nil]
95
+ #
96
+ def vocabulary(name)
97
+ init
98
+ @vocabularies[name]
99
+ end
100
+
101
+ # Create a model instance
102
+ #
103
+ # @param name [Symbol] Model identifier
104
+ # @param kwargs [Hash] Model constructor arguments
105
+ # @return [EmbeddingModel]
106
+ #
107
+ def create_model(name, **kwargs)
108
+ klass = model(name)
109
+ raise ArgumentError, "Unknown model: #{name}" unless klass
110
+
111
+ klass.new(**kwargs)
112
+ end
113
+
114
+ # Create an engine instance
115
+ #
116
+ # @param name [Symbol] Engine identifier
117
+ # @param kwargs [Hash] Engine constructor arguments
118
+ # @return [SimilarityEngine]
119
+ #
120
+ def create_engine(name, **kwargs)
121
+ klass = engine(name)
122
+ raise ArgumentError, "Unknown engine: #{name}" unless klass
123
+
124
+ klass.new(**kwargs)
125
+ end
126
+
127
+ # Create a vocabulary instance
128
+ #
129
+ # @param name [Symbol] Vocabulary identifier
130
+ # @param kwargs [Hash] Vocabulary constructor arguments
131
+ # @return [Vocabulary]
132
+ #
133
+ def create_vocabulary(name, **kwargs)
134
+ klass = vocabulary(name)
135
+ raise ArgumentError, "Unknown vocabulary: #{name}" unless klass
136
+
137
+ klass.new(**kwargs)
138
+ end
139
+
140
+ # List all registered models
141
+ #
142
+ # @return [Array<Symbol>]
143
+ #
144
+ def model_names
145
+ init
146
+ @models.keys
147
+ end
148
+
149
+ # List all registered engines
150
+ #
151
+ # @return [Array<Symbol>]
152
+ #
153
+ def engine_names
154
+ init
155
+ @engines.keys
156
+ end
157
+
158
+ # List all registered vocabularies
159
+ #
160
+ # @return [Array<Symbol>]
161
+ #
162
+ def vocabulary_names
163
+ init
164
+ @vocabularies.keys
165
+ end
166
+
167
+ # Clear all registrations
168
+ #
169
+ def reset!
170
+ @models = {}
171
+ @engines = {}
172
+ @vocabularies = {}
173
+ end
174
+ end
175
+ end
176
+
177
+ # Register built-in implementations. Triggering the autoloaded constants
178
+ # (OnnxRuntimeModel, SimilarityEngine, Vocabulary) here ensures the registry
179
+ # is populated the first time Kotoshu::Embeddings::Registry is referenced.
180
+ Registry.register_model(:onnx, OnnxRuntimeModel)
181
+ Registry.register_engine(:cosine, SimilarityEngine)
182
+ Registry.register_vocabulary(:json, Vocabulary)
183
+ end
184
+ end