kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,238 @@
1
+ # frozen_string_literal: true
2
+
3
+ # EmbeddingPipeline - Unified API for embedding-based similarity search
4
+ #
5
+ # Provides a simple, unified interface for loading vocabulary and models,
6
+ # and performing similarity search. This is the recommended entry point.
7
+ #
8
+ # @example Simple usage (one line)
9
+ # pipeline = Kotoshu::Embeddings::EmbeddingPipeline.from_cache(language: 'en')
10
+ #
11
+ # @example Full configuration
12
+ # pipeline = Kotoshu::Embeddings::EmbeddingPipeline.new(
13
+ # vocabulary: vocab,
14
+ # model: model,
15
+ # preload: true
16
+ # )
17
+ #
18
+ # @example Finding similar words
19
+ # neighbors = pipeline.find_nearest('semantic', k: 5)
20
+ # neighbors.each { |r| puts "#{r[:word]}: #{r[:similarity].round(4)}" }
21
+ #
22
+ module Kotoshu
23
+ module Embeddings
24
+ class EmbeddingPipeline
25
+ # @return [Vocabulary]
26
+ attr_reader :vocabulary
27
+
28
+ # @return [EmbeddingModel]
29
+ attr_reader :model
30
+
31
+ # @return [SimilarityEngine]
32
+ attr_reader :similarity_engine
33
+
34
+ # @return [Search]
35
+ attr_reader :search
36
+
37
+ # Create pipeline from cache (one-line initialization)
38
+ #
39
+ # @param language [String] ISO 639-1 language code
40
+ # @param cache [Cache::ModelCache] Cache instance
41
+ # @param preload [Boolean] Preload embeddings into memory
42
+ # @param index [:exact, :auto] Search index type
43
+ # @return [EmbeddingPipeline]
44
+ #
45
+ # @raise [ArgumentError] If no cached model found for language
46
+ #
47
+ def self.from_cache(language:, cache: nil, preload: false, index: :exact)
48
+ cache ||= Kotoshu::Cache::ModelCache.new
49
+
50
+ vocab_path = cache.find_vocab(language)
51
+ model_path = cache.find_model(language, :onnx)
52
+
53
+ unless vocab_path && model_path
54
+ raise ArgumentError, "No cached model for language: #{language}. " \
55
+ "Run: ruby scripts/extract_vocabularies.rb --languages=#{language}"
56
+ end
57
+
58
+ from_files(
59
+ vocab_path: vocab_path,
60
+ model_path: model_path,
61
+ language: language,
62
+ preload: preload,
63
+ index: index
64
+ )
65
+ end
66
+
67
+ # Create pipeline from files
68
+ #
69
+ # @param vocab_path [String] Path to vocabulary JSON file
70
+ # @param model_path [String] Path to ONNX model file
71
+ # @param language [String] Language code
72
+ # @param preload [Boolean] Preload embeddings
73
+ # @param index [:exact, :auto] Search index type
74
+ # @return [EmbeddingPipeline]
75
+ #
76
+ def self.from_files(vocab_path:, model_path:, language:, preload: false, index: :exact)
77
+ vocab = Vocabulary.from_file(vocab_path, language_code: language)
78
+ model = OnnxRuntimeModel.from_file(model_path, language_code: language)
79
+
80
+ new(
81
+ vocabulary: vocab,
82
+ model: model,
83
+ preload: preload,
84
+ index: index
85
+ )
86
+ end
87
+
88
+ # Create pipeline with full configuration
89
+ #
90
+ # @param vocabulary [Vocabulary] Vocabulary instance
91
+ # @param model [EmbeddingModel] Model instance
92
+ # @param preload [Boolean] Preload embeddings
93
+ # @param index [:exact, :ann] Search index type (:exact = brute force, :ann = FAISS/HNSW)
94
+ # @param pre_normalize [Boolean] Pre-normalize vectors
95
+ # @param cache_size [Integer] Embedding cache size
96
+ #
97
+ def initialize(vocabulary:, model:, preload: false, index: :exact, pre_normalize: false, cache_size: 1000)
98
+ @vocabulary = vocabulary
99
+ @model = model
100
+ @similarity_engine = SimilarityEngine.new(pre_normalize: pre_normalize)
101
+ @cache_size = cache_size
102
+
103
+ # Create search engine
104
+ @search = Search.new(
105
+ vocabulary: vocabulary,
106
+ model: model,
107
+ similarity_engine: @similarity_engine,
108
+ pre_normalize: pre_normalize
109
+ )
110
+
111
+ preload_embeddings! if preload
112
+ end
113
+
114
+ # Find k nearest neighbors for a word
115
+ #
116
+ # @param word [String] Query word
117
+ # @param k [Integer] Number of neighbors
118
+ # @param exclude_self [Boolean] Exclude query word
119
+ # @param min_similarity [Float] Minimum similarity threshold
120
+ # @return [Array<Hash>] Array of {word, similarity, index}
121
+ #
122
+ def find_nearest(word, k: 10, exclude_self: true, min_similarity: 0.0)
123
+ @search.find_nearest(word, k: k, exclude_self: exclude_self, min_similarity: min_similarity)
124
+ end
125
+
126
+ # Find nearest neighbors for multiple words
127
+ #
128
+ # @param words [Array<String>] Query words
129
+ # @param k [Integer] Neighbors per word
130
+ # @return [Hash<String, Array<Hash>>]
131
+ #
132
+ def find_nearest_batch(words, k: 10)
133
+ @search.find_nearest_batch(words, k: k)
134
+ end
135
+
136
+ # Compute similarity between two words
137
+ #
138
+ # @param word1 [String] First word
139
+ # @param word2 [String] Second word
140
+ # @return [Float, nil] Similarity or nil if either word not found
141
+ #
142
+ def similarity(word1, word2)
143
+ @search.similarity(word1, word2)
144
+ end
145
+
146
+ # Get embedding for a word
147
+ #
148
+ # @param word [String] Word
149
+ # @return [Array<Float>, nil]
150
+ #
151
+ def get_embedding(word)
152
+ @model.get_embedding_for_word(word, @vocabulary)
153
+ end
154
+
155
+ # Get embedding by index
156
+ #
157
+ # @param index [Integer] Word index
158
+ # @return [Array<Float>, nil]
159
+ #
160
+ def get_embedding_by_index(index)
161
+ @model.get_embedding(index)
162
+ end
163
+
164
+ # Check if word exists in vocabulary
165
+ #
166
+ # @param word [String] Word
167
+ # @return [Boolean]
168
+ #
169
+ def include?(word)
170
+ @vocabulary.include?(word)
171
+ end
172
+
173
+ # Preload all embeddings into memory
174
+ #
175
+ # @return [self]
176
+ #
177
+ def preload_embeddings!
178
+ @model.load!
179
+ @search.preload_embeddings!
180
+ self
181
+ end
182
+
183
+ # Unload model from memory
184
+ #
185
+ # @return [self]
186
+ #
187
+ def unload!
188
+ @model.unload!
189
+ @search.clear_cache
190
+ self
191
+ end
192
+
193
+ # Get pipeline statistics
194
+ #
195
+ # @return [Hash]
196
+ #
197
+ def stats
198
+ {
199
+ language: @vocabulary.language_code,
200
+ vocabulary_size: @vocabulary.size,
201
+ embedding_dimension: @model.dimension,
202
+ model_loaded: @model.loaded?,
203
+ embeddings_preloaded: @search.embeddings_loaded,
204
+ cache_stats: @search.cache_size
205
+ }
206
+ end
207
+
208
+ # Get model information
209
+ #
210
+ # @return [Hash]
211
+ #
212
+ def model_info
213
+ @model.model_info
214
+ end
215
+
216
+ # String representation
217
+ #
218
+ # @return [String]
219
+ #
220
+ def to_s
221
+ "EmbeddingPipeline(language: #{@vocabulary.language_code}, " \
222
+ "vocab_size: #{@vocabulary.size}, " \
223
+ "dimension: #{@model.dimension}, " \
224
+ "loaded: #{@model.loaded?})"
225
+ end
226
+ alias inspect to_s
227
+
228
+ # Convenience class methods
229
+ class << self
230
+ # Create a pipeline for a specific language (shortcut for
231
+ # {.from_cache}; same parameters and return value).
232
+ #
233
+ # @see .from_cache
234
+ alias :[] :from_cache
235
+ end
236
+ end
237
+ end
238
+ end
@@ -0,0 +1,244 @@
1
+ # frozen_string_literal: true
2
+
3
+ # LruCache - Least Recently Used Cache
4
+ #
5
+ # Provides efficient O(1) LRU caching with optional TTL support.
6
+ # Used for caching embeddings during similarity search.
7
+ #
8
+ # @example Basic usage
9
+ # cache = LruCache.new(max_size: 1000)
10
+ # cache[:key] = value
11
+ # cache[:key] # => value
12
+ #
13
+ # @example With TTL
14
+ # cache = LruCache.new(max_size: 1000, ttl: 300) # 5 minutes
15
+ #
16
+ module Kotoshu
17
+ module Embeddings
18
+ # LRU cache for embedding results.
19
+ class LruCache
20
+ # @return [Integer] Maximum number of entries
21
+ attr_reader :max_size
22
+
23
+ # @return [Integer, nil] TTL in seconds
24
+ attr_reader :ttl
25
+
26
+ # @return [Integer] Number of cache hits
27
+ attr_reader :hits
28
+
29
+ # @return [Integer] Number of cache misses
30
+ attr_reader :misses
31
+
32
+ # Create a new LRU cache
33
+ #
34
+ # @param max_size [Integer] Maximum number of entries (default: 1000)
35
+ # @param ttl [Integer, nil] Time-to-live in seconds (default: nil = no expiry)
36
+ #
37
+ def initialize(max_size: 1000, ttl: nil)
38
+ @max_size = max_size
39
+ @ttl = ttl
40
+ @cache = {} # key -> {value: v, accessed_at: t, created_at: t}
41
+ @order = [] # Ordered list of keys (most recently used first)
42
+ @hits = 0
43
+ @misses = 0
44
+ end
45
+
46
+ # Get value for key
47
+ #
48
+ # @param key [Object] Cache key
49
+ # @return [Object, nil] Cached value or nil if not found/expired
50
+ #
51
+ def [](key)
52
+ entry = @cache[key]
53
+ return nil unless entry
54
+
55
+ # Check TTL
56
+ if @ttl && (Time.now - entry[:created_at]) > @ttl
57
+ delete(key)
58
+ @misses += 1
59
+ return nil
60
+ end
61
+
62
+ # Update access order (move to front = most recently used)
63
+ @order.delete(key)
64
+ @order.unshift(key)
65
+ entry[:accessed_at] = Time.now
66
+
67
+ @hits += 1
68
+ entry[:value]
69
+ end
70
+
71
+ # Set value for key
72
+ #
73
+ # @param key [Object] Cache key
74
+ # @param value [Object] Value to cache
75
+ # @return [Object] The value
76
+ #
77
+ def []=(key, value)
78
+ # Evict LRU if at capacity
79
+ if @cache.key?(key)
80
+ # Update existing entry
81
+ @cache[key][:value] = value
82
+ @cache[key][:accessed_at] = Time.now
83
+ # Move to front
84
+ @order.delete(key)
85
+ @order.unshift(key)
86
+ return value
87
+ end
88
+
89
+ if @cache.size >= @max_size
90
+ evict_lru
91
+ end
92
+
93
+ @cache[key] = {
94
+ value: value,
95
+ accessed_at: Time.now,
96
+ created_at: Time.now
97
+ }
98
+ @order.unshift(key)
99
+
100
+ value
101
+ end
102
+
103
+ # Check if key exists
104
+ #
105
+ # @param key [Object] Cache key
106
+ # @return [Boolean] True if key exists and not expired
107
+ #
108
+ def key?(key)
109
+ entry = @cache[key]
110
+ return false unless entry
111
+
112
+ if @ttl && (Time.now - entry[:created_at]) > @ttl
113
+ delete(key)
114
+ return false
115
+ end
116
+
117
+ true
118
+ end
119
+
120
+ # Delete key from cache
121
+ #
122
+ # @param key [Object] Cache key
123
+ # @return [Object, nil] Deleted value or nil
124
+ #
125
+ def delete(key)
126
+ entry = @cache.delete(key)
127
+ @order.delete(key)
128
+ entry&.[](:value)
129
+ end
130
+
131
+ # Clear all entries
132
+ #
133
+ # @return [self]
134
+ #
135
+ def clear
136
+ @cache.clear
137
+ @order.clear
138
+ self
139
+ end
140
+
141
+ # Get current size
142
+ #
143
+ # @return [Integer] Number of entries
144
+ #
145
+ def size
146
+ @cache.size
147
+ end
148
+
149
+ # Check if empty
150
+ #
151
+ # @return [Boolean]
152
+ #
153
+ def empty?
154
+ @cache.empty?
155
+ end
156
+
157
+ # Get least recently used key-value pair
158
+ #
159
+ # @return [Array<Object, Object>, nil]
160
+ #
161
+ def lru
162
+ return nil if @order.empty?
163
+
164
+ key = @order.last
165
+ [key, @cache[key][:value]]
166
+ end
167
+
168
+ # Get most recently used key-value pair
169
+ #
170
+ # @return [Array<Object, Object>, nil]
171
+ #
172
+ def mru
173
+ return nil if @order.empty?
174
+
175
+ key = @order.first
176
+ [key, @cache[key][:value]]
177
+ end
178
+
179
+ # Get all keys
180
+ #
181
+ # @return [Array<Object>] Array of keys
182
+ #
183
+ def keys
184
+ @order.dup
185
+ end
186
+
187
+ # Get all values
188
+ #
189
+ # @return [Array<Object>] Array of values
190
+ #
191
+ def values
192
+ @order.map { |key| @cache[key][:value] }
193
+ end
194
+
195
+ # Get cache statistics
196
+ #
197
+ # @return [Hash] Statistics
198
+ #
199
+ def stats
200
+ total = @hits + @misses
201
+ {
202
+ size: size,
203
+ max_size: @max_size,
204
+ hits: @hits,
205
+ misses: @misses,
206
+ hit_rate: total.zero? ? 0.0 : @hits.to_f / total,
207
+ ttl: @ttl
208
+ }
209
+ end
210
+
211
+ # Fetch with block (cache-aside pattern)
212
+ #
213
+ # Yields the key to the block on a miss, matching Ruby's
214
+ # +Hash#fetch+ convention. The block is invoked at most once per
215
+ # call.
216
+ #
217
+ # @param key [Object] Cache key
218
+ # @yieldparam key [Object] The key being fetched
219
+ # @yieldreturn [Object] Value to cache and return on miss
220
+ # @return [Object] Cached value or block result
221
+ #
222
+ def fetch(key)
223
+ result = self[key]
224
+ return result if result || key?(key)
225
+
226
+ value = yield(key)
227
+ self[key] = value
228
+ value
229
+ end
230
+
231
+ private
232
+
233
+ # Evict least recently used entry
234
+ #
235
+ def evict_lru
236
+ return if @order.empty?
237
+
238
+ lru_key = @order.last
239
+ @cache.delete(lru_key)
240
+ @order.pop
241
+ end
242
+ end
243
+ end
244
+ end