kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,204 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Search - Brute force nearest neighbor search
4
+ #
5
+ # Performs exhaustive search over all vocabulary entries.
6
+ # Uses min-heap for efficient top-k selection (O(n log k) instead of O(n log n)).
7
+ #
8
+ # @example
9
+ # search = Kotoshu::Embeddings::Search.new(
10
+ # vocabulary: vocab,
11
+ # model: model,
12
+ # similarity_engine: engine
13
+ # )
14
+ # neighbors = search.find_nearest('hello', k: 5)
15
+ #
16
+ module Kotoshu
17
+ module Embeddings
18
+ class Search
19
+ # Min-heap for top-k selection
20
+ class MinHeap
21
+ def initialize(max_size)
22
+ @max_size = max_size
23
+ @heap = []
24
+ end
25
+
26
+ def push(item)
27
+ @heap << item
28
+ @heap.sort_by! { |i| i[:similarity] }
29
+ @heap.shift if @heap.size > @max_size
30
+ end
31
+
32
+ def empty?
33
+ @heap.empty?
34
+ end
35
+
36
+ def size
37
+ @heap.size
38
+ end
39
+
40
+ def each(&)
41
+ @heap.each(&)
42
+ end
43
+
44
+ def to_a
45
+ @heap.dup
46
+ end
47
+ end
48
+
49
+ # @return [Vocabulary]
50
+ attr_reader :vocabulary
51
+
52
+ # @return [EmbeddingModel]
53
+ attr_reader :model
54
+
55
+ # @return [SimilarityEngine]
56
+ attr_reader :similarity_engine
57
+
58
+ # @return [Boolean] Whether embeddings are preloaded
59
+ attr_reader :embeddings_loaded
60
+
61
+ # Create a new exact search
62
+ #
63
+ # @param vocabulary [Vocabulary] Word vocabulary
64
+ # @param model [EmbeddingModel] Embedding provider
65
+ # @param similarity_engine [SimilarityEngine] Similarity calculator
66
+ # @param pre_normalize [Boolean] Pre-normalize vectors for faster similarity
67
+ #
68
+ def initialize(vocabulary:, model:, similarity_engine:, pre_normalize: false)
69
+ @vocabulary = vocabulary
70
+ @model = model
71
+ @similarity_engine = similarity_engine
72
+ @pre_normalize = pre_normalize
73
+
74
+ @embedding_cache = {}
75
+ @embeddings_loaded = false
76
+ end
77
+
78
+ # Find k nearest neighbors for a word
79
+ #
80
+ # @param query_word [String] Query word
81
+ # @param k [Integer] Number of neighbors to return
82
+ # @param exclude_self [Boolean] Exclude query word from results
83
+ # @param min_similarity [Float] Minimum similarity threshold
84
+ # @return [Array<Hash>] Array of {word, similarity, index}
85
+ #
86
+ def find_nearest(query_word, k: 10, exclude_self: true, min_similarity: 0.0)
87
+ query_vec = get_embedding_for_word(query_word)
88
+ return [] unless query_vec
89
+
90
+ heap = MinHeap.new(k)
91
+
92
+ @vocabulary.words.each do |word|
93
+ next if exclude_self && word == query_word
94
+
95
+ vec = get_embedding_for_word(word)
96
+ next unless vec
97
+
98
+ similarity = @similarity_engine.cosine(query_vec, vec)
99
+ next if similarity < min_similarity
100
+
101
+ index = @vocabulary.lookup(word)
102
+ heap.push(word: word, similarity: similarity, index: index)
103
+ end
104
+
105
+ # Return sorted by similarity descending
106
+ heap.to_a.sort_by { |r| -r[:similarity] }
107
+ end
108
+
109
+ # Find nearest neighbors for multiple words
110
+ #
111
+ # @param query_words [Array<String>] Query words
112
+ # @param k [Integer] Number of neighbors per word
113
+ # @return [Hash<String, Array<Hash>>] Word to results mapping
114
+ #
115
+ def find_nearest_batch(query_words, k: 10)
116
+ query_words.each_with_object({}) do |word, results|
117
+ results[word] = find_nearest(word, k: k)
118
+ end
119
+ end
120
+
121
+ # Compute similarity between two words
122
+ #
123
+ # @param word1 [String] First word
124
+ # @param word2 [String] Second word
125
+ # @return [Float, nil] Similarity or nil if either word not found
126
+ #
127
+ def similarity(word1, word2)
128
+ vec1 = get_embedding_for_word(word1)
129
+ vec2 = get_embedding_for_word(word2)
130
+ return nil unless vec1 && vec2
131
+
132
+ @similarity_engine.cosine(vec1, vec2)
133
+ end
134
+
135
+ # Preload all embeddings into memory
136
+ #
137
+ # @return [self]
138
+ #
139
+ def preload_embeddings!
140
+ all_indices = (0...@vocabulary.size).to_a
141
+ embeddings = @model.get_embeddings(all_indices)
142
+
143
+ @vocabulary.words.each_with_index do |word, i|
144
+ @embedding_cache[word] = embeddings[i]
145
+ end
146
+
147
+ @embeddings_loaded = true
148
+ self
149
+ end
150
+
151
+ # Clear embedding cache
152
+ #
153
+ # @return [self]
154
+ #
155
+ def clear_cache
156
+ @embedding_cache.clear
157
+ @embeddings_loaded = false
158
+ self
159
+ end
160
+
161
+ # Number of cached embeddings
162
+ #
163
+ # @return [Integer]
164
+ #
165
+ def cache_size
166
+ @embedding_cache.size
167
+ end
168
+
169
+ # String representation
170
+ #
171
+ # @return [String]
172
+ #
173
+ def to_s
174
+ "ExactSearch(vocab: #{@vocabulary.size}, loaded: #{@embeddings_loaded})"
175
+ end
176
+ alias inspect to_s
177
+
178
+ private
179
+
180
+ # Get embedding for a word (with caching)
181
+ #
182
+ # @param word [String] Word
183
+ # @return [Array<Float>, nil]
184
+ #
185
+ def get_embedding_for_word(word)
186
+ # Check cache first
187
+ if @embedding_cache.key?(word)
188
+ return @embedding_cache[word]
189
+ end
190
+
191
+ index = @vocabulary.lookup(word)
192
+ return nil unless index
193
+
194
+ vec = @model.get_embedding(index)
195
+ return nil unless vec
196
+
197
+ # Cache if not preloaded (to avoid repeated lookups)
198
+ @embedding_cache[word] = vec unless @embeddings_loaded
199
+
200
+ vec
201
+ end
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,278 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SimilarityEngine - Compute similarity between embedding vectors
4
+ #
5
+ # Provides various similarity/distance metrics with optimizations like
6
+ # norm caching and pre-normalized vector support.
7
+ #
8
+ # @example Basic usage
9
+ # engine = Kotoshu::Embeddings::SimilarityEngine.new
10
+ # engine.cosine([1.0, 0.0], [1.0, 0.0]) # => 1.0
11
+ #
12
+ # @example Pre-normalized vectors (faster)
13
+ # engine = Kotoshu::Embeddings::SimilarityEngine.new(pre_normalize: true)
14
+ # engine.pre_normalize([1.0, 0.0]) # => [1.0, 0.0]
15
+ #
16
+ module Kotoshu
17
+ module Embeddings
18
+ class SimilarityEngine
19
+ include SimilarityEngineProtocol
20
+
21
+ # Default embedding dimension for norm cache initialization
22
+ DEFAULT_CACHE_SIZE = 10_000
23
+
24
+ # @return [Boolean] Whether vectors are pre-normalized
25
+ def pre_normalize?
26
+ @pre_normalize
27
+ end
28
+
29
+ # @return [Integer] Number of cache hits
30
+ attr_reader :cache_hits
31
+
32
+ # @return [Integer] Number of cache misses
33
+ attr_reader :cache_misses
34
+
35
+ # Create a new similarity engine
36
+ #
37
+ # @param pre_normalize [Boolean] Whether to pre-normalize vectors
38
+ # @param cache_norms [Boolean] Whether to cache vector norms
39
+ #
40
+ def initialize(pre_normalize: false, cache_norms: true)
41
+ @pre_normalize = pre_normalize
42
+ @cache_norms = cache_norms
43
+ @norm_cache = cache_norms ? {} : nil
44
+ @cache_hits = 0
45
+ @cache_misses = 0
46
+ end
47
+
48
+ # Compute cosine similarity between two vectors
49
+ #
50
+ # Cosine similarity = dot(v1, v2) / (||v1|| * ||v2||)
51
+ # Range: -1.0 (opposite) to 1.0 (identical)
52
+ #
53
+ # @param vec1 [Array<Float>] First vector
54
+ # @param vec2 [Array<Float>] Second vector
55
+ # @return [Float] Cosine similarity, or 0.0 if either vector is nil/empty
56
+ #
57
+ def cosine(vec1, vec2)
58
+ return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
59
+
60
+ validate_dimensions!(vec1, vec2)
61
+
62
+ norm1 = get_norm(vec1)
63
+ norm2 = get_norm(vec2)
64
+
65
+ return 0.0 if norm1.zero? || norm2.zero?
66
+
67
+ dot = dot_product(vec1, vec2)
68
+ dot / (norm1 * norm2)
69
+ end
70
+
71
+ # Compute dot product between two vectors
72
+ #
73
+ # @param vec1 [Array<Float>] First vector
74
+ # @param vec2 [Array<Float>] Second vector
75
+ # @return [Float] Dot product
76
+ #
77
+ def dot_product(vec1, vec2)
78
+ return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
79
+
80
+ validate_dimensions!(vec1, vec2)
81
+
82
+ vec1.zip(vec2).sum { |a, b| a * b }
83
+ end
84
+
85
+ # Compute Euclidean distance between two vectors
86
+ #
87
+ # @param vec1 [Array<Float>] First vector
88
+ # @param vec2 [Array<Float>] Second vector
89
+ # @return [Float] Euclidean distance
90
+ #
91
+ def euclidean(vec1, vec2)
92
+ return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
93
+ return 0.0 if vec1.equal?(vec2)
94
+
95
+ validate_dimensions!(vec1, vec2)
96
+
97
+ sum = 0.0
98
+ vec1.zip(vec2) do |a, b|
99
+ diff = a - b
100
+ sum += diff * diff
101
+ end
102
+ Math.sqrt(sum)
103
+ end
104
+
105
+ # Compute Manhattan (L1) distance between two vectors
106
+ #
107
+ # @param vec1 [Array<Float>] First vector
108
+ # @param vec2 [Array<Float>] Second vector
109
+ # @return [Float] Manhattan distance
110
+ #
111
+ def manhattan(vec1, vec2)
112
+ return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
113
+
114
+ validate_dimensions!(vec1, vec2)
115
+
116
+ vec1.zip(vec2).sum { |a, b| (a - b).abs }
117
+ end
118
+
119
+ # Pre-normalize a vector to unit length
120
+ #
121
+ # @param vec [Array<Float>] Vector to normalize
122
+ # @return [Array<Float>] Normalized vector
123
+ #
124
+ def pre_normalize(vec)
125
+ return vec.dup if vec.nil? || vec.empty?
126
+
127
+ norm = get_norm(vec)
128
+ return vec.dup if norm.zero?
129
+
130
+ vec.map { |x| x / norm }
131
+ end
132
+
133
+ # Normalize and compute similarity in one pass
134
+ #
135
+ # For pre-normalized vectors, this is just dot product (much faster).
136
+ #
137
+ # @param vec1 [Array<Float>] First vector
138
+ # @param vec2 [Array<Float>] Second vector
139
+ # @return [Float] Cosine similarity
140
+ #
141
+ def normalize_and_compute(vec1, vec2)
142
+ return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
143
+
144
+ if @pre_normalize
145
+ # For normalized vectors, cosine similarity = dot product
146
+ dot_product(vec1, vec2)
147
+ else
148
+ cosine(vec1, vec2)
149
+ end
150
+ end
151
+
152
+ # Check if vectors are normalized (unit length)
153
+ #
154
+ # @param vec [Array<Float>] Vector to check
155
+ # @return [Boolean] True if vector is normalized
156
+ #
157
+ def is_normalized?(vec)
158
+ return true if vec.nil? || vec.empty?
159
+
160
+ norm = get_norm(vec)
161
+ (norm - 1.0).abs < Float::EPSILON * 10
162
+ end
163
+
164
+ # Check if normalization is required for accurate similarity
165
+ #
166
+ # @return [Boolean] True if normalization should be applied
167
+ #
168
+ def normalization_required?
169
+ !@pre_normalize
170
+ end
171
+
172
+ # Clear the norm cache
173
+ #
174
+ # @return [self]
175
+ #
176
+ def clear_cache
177
+ @norm_cache&.clear
178
+ @cache_hits = 0
179
+ @cache_misses = 0
180
+ self
181
+ end
182
+
183
+ # Get cache statistics
184
+ #
185
+ # @return [Hash] Cache statistics
186
+ #
187
+ def cache_stats
188
+ total = @cache_hits + @cache_misses
189
+ {
190
+ hits: @cache_hits,
191
+ misses: @cache_misses,
192
+ hit_rate: total.zero? ? 0.0 : @cache_hits.to_f / total,
193
+ cache_size: @norm_cache&.size || 0
194
+ }
195
+ end
196
+
197
+ # Compute similarity for a batch of vector pairs
198
+ #
199
+ # More efficient than calling cosine() repeatedly.
200
+ #
201
+ # @param pairs [Array<Array<Array<Float>>>] Array of [vec1, vec2] pairs
202
+ # @return [Array<Float>] Array of similarities
203
+ #
204
+ def cosine_batch(pairs)
205
+ pairs.map { |v1, v2| cosine(v1, v2) }
206
+ end
207
+
208
+ # Compute all pairwise similarities for a set of vectors
209
+ #
210
+ # @param vectors [Array<Array<Float>>>] Array of vectors
211
+ # @return [Array<Array<Float>>] Similarity matrix
212
+ #
213
+ def compute_all_pairs(vectors)
214
+ n = vectors.length
215
+ matrix = Array.new(n) { Array.new(n, 0.0) }
216
+
217
+ (0...n).each do |i|
218
+ matrix[i][i] = 1.0
219
+ ((i + 1)...n).each do |j|
220
+ sim = cosine(vectors[i], vectors[j])
221
+ matrix[i][j] = sim
222
+ matrix[j][i] = sim
223
+ end
224
+ end
225
+
226
+ matrix
227
+ end
228
+
229
+ private
230
+
231
+ # Raise ArgumentError if two non-empty vectors have different lengths.
232
+ #
233
+ # Array#zip pads the shorter vector with nil, which produces a
234
+ # confusing TypeError deep inside the arithmetic. Surface the
235
+ # mismatch at the API boundary instead.
236
+ #
237
+ # @param vec1 [Array<Float>] First vector
238
+ # @param vec2 [Array<Float>] Second vector
239
+ # @return [void]
240
+ # @raise [ArgumentError] if lengths differ
241
+ #
242
+ def validate_dimensions!(vec1, vec2)
243
+ return if vec1.length == vec2.length
244
+
245
+ raise ArgumentError,
246
+ "vector dimension mismatch: #{vec1.length} vs #{vec2.length}"
247
+ end
248
+
249
+ # Get norm with caching
250
+ #
251
+ # @param vec [Array<Float>] Vector
252
+ # @return [Float] Vector norm (magnitude)
253
+ #
254
+ def get_norm(vec)
255
+ return 0.0 if vec.nil? || vec.empty?
256
+
257
+ if @norm_cache && @norm_cache.key?(vec.object_id)
258
+ @cache_hits += 1
259
+ return @norm_cache[vec.object_id]
260
+ end
261
+
262
+ @cache_misses += 1 if @norm_cache
263
+
264
+ norm = Math.sqrt(vec.sum { |x| x * x })
265
+
266
+ if @norm_cache
267
+ # Avoid memory leaks by limiting cache size
268
+ if @norm_cache.size >= 100_000
269
+ @norm_cache.shift
270
+ end
271
+ @norm_cache[vec.object_id] = norm
272
+ end
273
+
274
+ norm
275
+ end
276
+ end
277
+ end
278
+ end