kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Results
5
+ # Result pattern for explicit error handling.
6
+ #
7
+ # Provides a type-safe way to handle operations that can fail
8
+ # without using exceptions. Based on functional programming patterns.
9
+ #
10
+ # @example Using Success
11
+ # result = Result::Success.new("value")
12
+ # result.success? # => true
13
+ # result.value # => "value"
14
+ #
15
+ # @example Using Failure
16
+ # result = Result::Failure.new(error)
17
+ # result.failure? # => true
18
+ # result.error # => the error
19
+ #
20
+ # @example Chaining operations
21
+ # result = Result::Success.new(5)
22
+ # .and_then { |v| Success.new(v * 2) } # Only called if success
23
+ # .or_else { |e| Success.new(0) } # Only called if failure
24
+ module Result
25
+ # Base result class.
26
+ #
27
+ # @abstract
28
+ class Base
29
+ # Check if result is successful.
30
+ #
31
+ # @return [Boolean] True if successful
32
+ def success?
33
+ is_a?(Success)
34
+ end
35
+
36
+ # Check if result is a failure.
37
+ #
38
+ # @return [Boolean] True if failed
39
+ def failure?
40
+ is_a?(Failure)
41
+ end
42
+
43
+ # Map the value if successful.
44
+ #
45
+ # @yield [value] The wrapped value
46
+ # @return [Result::Success, Result::Failure] Mapped result
47
+ def map
48
+ return self if failure?
49
+
50
+ Success.new(yield value)
51
+ rescue StandardError => e
52
+ Failure.new(e)
53
+ end
54
+
55
+ # Chain operations if successful.
56
+ #
57
+ # @yield [value] The wrapped value
58
+ # @return [Result::Success, Result::Failure] Chained result
59
+ def and_then
60
+ return self if failure?
61
+
62
+ result = yield value
63
+
64
+ # Ensure we get a Result back
65
+ result.is_a?(Base) ? result : Success.new(result)
66
+ rescue StandardError => e
67
+ Failure.new(e)
68
+ end
69
+
70
+ # Recover from failure.
71
+ #
72
+ # @yield [error] The wrapped error
73
+ # @return [Result::Success, Result::Failure] Recovered result
74
+ def or_else
75
+ return self if success?
76
+
77
+ result = yield error
78
+
79
+ # Ensure we get a Result back
80
+ result.is_a?(Base) ? result : Success.new(result)
81
+ end
82
+
83
+ # Unwrap the value or raise error.
84
+ #
85
+ # @return [Object] The wrapped value
86
+ # @raise [Error] The wrapped error if this is a Failure
87
+ def unwrap
88
+ return value if success?
89
+
90
+ raise error
91
+ end
92
+
93
+ # Get the wrapped value (nil for Failure).
94
+ #
95
+ # @return [Object, nil] The wrapped value or nil
96
+ def value
97
+ raise NotImplementedError
98
+ end
99
+
100
+ # Get the wrapped error (nil for Success).
101
+ #
102
+ # @return [StandardError, nil] The wrapped error or nil
103
+ def error
104
+ raise NotImplementedError
105
+ end
106
+ end
107
+
108
+ # Represents a successful operation.
109
+ #
110
+ class Success < Base
111
+ # @return [Object] The wrapped value
112
+ attr_reader :value
113
+
114
+ # Create a new Success result.
115
+ #
116
+ # @param value [Object] The wrapped value
117
+ def initialize(value)
118
+ @value = value
119
+ end
120
+
121
+ # Get the error (always nil for Success).
122
+ #
123
+ # @return [nil] Always nil
124
+ def error
125
+ nil
126
+ end
127
+ end
128
+
129
+ # Represents a failed operation.
130
+ #
131
+ class Failure < Base
132
+ # @return [StandardError] The wrapped error
133
+ attr_reader :error
134
+
135
+ # Create a new Failure result.
136
+ #
137
+ # @param error [StandardError] The wrapped error
138
+ def initialize(error)
139
+ @error = error
140
+ end
141
+
142
+ # Map does nothing for Failure.
143
+ #
144
+ # @return [Failure] Self
145
+ def map
146
+ self
147
+ end
148
+
149
+ # and_then does nothing for Failure.
150
+ #
151
+ # @return [Failure] Self
152
+ def and_then
153
+ self
154
+ end
155
+
156
+ # Get the value (always nil for Failure).
157
+ #
158
+ # @return [nil] Always nil
159
+ def value
160
+ nil
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Generic result types shared across the codebase.
5
+ module Results
6
+ autoload :Result, "kotoshu/results/result"
7
+ end
8
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rspec/matchers"
4
+
5
+ module Kotoshu
6
+ # RSpec integration (plan 89): word and document matchers with
7
+ # real failure messages.
8
+ #
9
+ # RSpec is a soft dependency of the gem itself; this file requires
10
+ # rspec/matchers (present in any suite that would use it).
11
+ #
12
+ # @example Wiring
13
+ # # spec_helper.rb
14
+ # require "kotoshu/rspec"
15
+ # RSpec.configure do |config|
16
+ # config.include Kotoshu::Rspec::Matchers
17
+ # end
18
+ #
19
+ # @example Word matcher
20
+ # expect_words("helo", "world").to all_be_spelled_correctly
21
+ # expect("helo").not_to be_spelled_correctly
22
+ #
23
+ # @example Document matcher
24
+ # expect_document("README.adoc").to be_spelled_correctly
25
+ # expect("helo wrold").not_to be_spelled_correctly(in: "en")
26
+ module Rspec
27
+ # Matchers exposed to specs via config.include.
28
+ module Matchers
29
+ # Expectation target over discrete words: returns the real
30
+ # RSpec target so `.to all_be_spelled_correctly` works. Requires
31
+ # RSpec::Matchers to be in scope (it is, in any suite including
32
+ # this module).
33
+ #
34
+ # @param words [Array<String>] words to check
35
+ # @return [RSpec::Expectations::ExpectationTarget]
36
+ def expect_words(*words)
37
+ expect(words.flatten)
38
+ end
39
+
40
+ # Expectation target over a documents contents.
41
+ #
42
+ # @param path [String] document path
43
+ # @return [RSpec::Expectations::ExpectationTarget]
44
+ def expect_document(path)
45
+ expect(File.read(path))
46
+ end
47
+
48
+ # Matcher: every item of the target is a correctly spelled
49
+ # word. Pairs with expect_words.
50
+ def all_be_spelled_correctly
51
+ AllWordsSpelledCorrectlyMatcher.new
52
+ end
53
+
54
+ # Matcher: a single word is correct, or a whole text is free of
55
+ # misspellings (any whitespace makes it a document).
56
+ def be_spelled_correctly(in: nil)
57
+ SpelledCorrectlyMatcher.new(language: binding.local_variable_get(:in))
58
+ end
59
+
60
+ # Checks each word of an enumerable.
61
+ class AllWordsSpelledCorrectlyMatcher
62
+ def matches?(words)
63
+ @words = Array(words)
64
+ @misspelled = @words.reject { |word| Kotoshu.correct?(word) }
65
+ @misspelled.empty?
66
+ end
67
+
68
+ def failure_message
69
+ "expected all words to be spelled correctly, but found " \
70
+ "#{formatted_misspellings}"
71
+ end
72
+
73
+ def failure_message_when_negated
74
+ "expected some words to be misspelled, but all were correct"
75
+ end
76
+
77
+ def description
78
+ "all be spelled correctly"
79
+ end
80
+
81
+ private
82
+
83
+ # One "word -> suggestion" pair per misspelling.
84
+ #
85
+ # @return [String]
86
+ def formatted_misspellings
87
+ @misspelled.map do |word|
88
+ suggestions = Kotoshu.suggest(word).top(3).map(&:word)
89
+ detail = suggestions.empty? ? "no suggestions" : suggestions.join(", ")
90
+ "#{word.inspect} (#{detail})"
91
+ end.join(", ")
92
+ end
93
+ end
94
+
95
+ # Checks one word, or a whole document when the string contains
96
+ # whitespace.
97
+ class SpelledCorrectlyMatcher
98
+ def initialize(language: nil)
99
+ @language = language
100
+ end
101
+
102
+ def matches?(target)
103
+ @target = target.to_s
104
+ @errors = document? ? document_errors : word_errors
105
+ @errors.empty?
106
+ end
107
+
108
+ def failure_message
109
+ if document?
110
+ "expected the document to be spelled correctly, but found " \
111
+ "#{formatted_errors}"
112
+ else
113
+ "expected #{@target.inspect} to be spelled correctly"
114
+ end
115
+ end
116
+
117
+ def failure_message_when_negated
118
+ if document?
119
+ "expected the document to have misspellings, but it is clean"
120
+ else
121
+ "expected #{@target.inspect} to be misspelled"
122
+ end
123
+ end
124
+
125
+ def description
126
+ "be spelled correctly"
127
+ end
128
+
129
+ private
130
+
131
+ # Whether the target is a document rather than one word.
132
+ def document?
133
+ @target.match?(/\s/)
134
+ end
135
+
136
+ # Document misspellings via the check pipeline.
137
+ def document_errors
138
+ Kotoshu.check(@target, language: @language).errors
139
+ end
140
+
141
+ # Single-word misspelling via correct?.
142
+ def word_errors
143
+ Kotoshu.correct?(@target, language: @language) ? [] : [@target]
144
+ end
145
+
146
+ # One "word -> suggestions" pair per misspelling.
147
+ #
148
+ # @return [String]
149
+ def formatted_errors
150
+ @errors.map do |error|
151
+ if error.is_a?(String)
152
+ "#{error.inspect} (no suggestions)"
153
+ else
154
+ suggestions = error.top_suggestions(3)
155
+ detail = suggestions.empty? ? "no suggestions" : suggestions.join(", ")
156
+ "#{error.word.inspect} (#{detail})"
157
+ end
158
+ end.join(", ")
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -0,0 +1,275 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ FastText to ONNX Converter
4
+
5
+ Converts FastText .vec files to ONNX format for efficient deployment.
6
+ The ONNX model contains an embedding lookup layer with the word vectors.
7
+
8
+ Usage:
9
+ python fasttext_to_onnx.py <vec_file> <output_onnx> [--vocab_size N]
10
+
11
+ Example:
12
+ python fasttext_to_onnx.py cc.en.300.vec fasttext.en.onnx --vocab_size 50000
13
+ """
14
+
15
+ import sys
16
+ import argparse
17
+ import json
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+ import onnx
22
+ from onnx import helper, numpy_helper
23
+ from onnx import TensorProto
24
+
25
+
26
+ def parse_fasttext_vec(vec_file_path, vocab_size=None):
27
+ """
28
+ Parse FastText .vec file and extract vocabulary and embeddings.
29
+
30
+ Args:
31
+ vec_file_path: Path to FastText .vec file
32
+ vocab_size: Maximum vocabulary size (None for all words)
33
+
34
+ Returns:
35
+ tuple: (vocab_dict, embeddings_array, metadata)
36
+ - vocab_dict: Dictionary mapping word to index
37
+ - embeddings_array: NumPy array of shape [vocab_size, embedding_dim]
38
+ - metadata: Dictionary with file metadata
39
+ """
40
+ print(f"Parsing FastText file: {vec_file_path}")
41
+
42
+ word_to_idx = {}
43
+ embeddings = []
44
+ metadata = {
45
+ "source_format": "fasttext_vec",
46
+ "embedding_dim": None,
47
+ "vocab_size": 0,
48
+ "total_words_in_file": 0
49
+ }
50
+
51
+ with open(vec_file_path, 'r', encoding='utf-8') as f:
52
+ # First line: vocab_size and dimension
53
+ first_line = f.readline().strip()
54
+ parts = first_line.split()
55
+ total_vocab = int(parts[0])
56
+ embedding_dim = int(parts[1])
57
+
58
+ metadata["embedding_dim"] = embedding_dim
59
+ metadata["total_words_in_file"] = total_vocab
60
+
61
+ print(f" Total words in file: {total_vocab}")
62
+ print(f" Embedding dimension: {embedding_dim}")
63
+
64
+ # Limit vocab size if specified
65
+ if vocab_size is None or vocab_size > total_vocab:
66
+ vocab_size = total_vocab
67
+
68
+ metadata["vocab_size"] = vocab_size
69
+
70
+ print(f" Loading {vocab_size} words...")
71
+
72
+ # Read embeddings
73
+ for idx in range(vocab_size):
74
+ line = f.readline()
75
+ if not line:
76
+ break
77
+
78
+ parts = line.strip().split()
79
+ word = parts[0]
80
+ vector = np.array([float(x) for x in parts[1:]], dtype=np.float32)
81
+
82
+ word_to_idx[word] = idx
83
+ embeddings.append(vector)
84
+
85
+ if (idx + 1) % 10000 == 0:
86
+ print(f" Loaded {idx + 1} words...")
87
+
88
+ # Stack embeddings into a matrix
89
+ embeddings_matrix = np.vstack(embeddings).astype(np.float32)
90
+
91
+ print(f" Embeddings matrix shape: {embeddings_matrix.shape}")
92
+ print(f" Matrix size in MB: {embeddings_matrix.nbytes / (1024 * 1024):.2f}")
93
+
94
+ return word_to_idx, embeddings_matrix, metadata
95
+
96
+
97
+ def create_onnx_model(embeddings_matrix, word_to_idx, model_name="fasttext"):
98
+ """
99
+ Create ONNX model with embedding lookup layer.
100
+
101
+ Args:
102
+ embeddings_matrix: NumPy array of word embeddings [vocab_size, embedding_dim]
103
+ word_to_idx: Dictionary mapping words to indices
104
+ model_name: Name for the ONNX model
105
+
106
+ Returns:
107
+ onnx.ModelProto: ONNX model
108
+ """
109
+ vocab_size, embedding_dim = embeddings_matrix.shape
110
+
111
+ print(f"Creating ONNX model...")
112
+ print(f" Input: word_index (int64)")
113
+ print(f" Output: embedding (float32, shape=[{embedding_dim}])")
114
+
115
+ # Create input (word index)
116
+ input_tensor = helper.make_tensor_value_info(
117
+ 'word_index',
118
+ TensorProto.INT64,
119
+ [1] # Scalar input (single word index)
120
+ )
121
+
122
+ # Create output (embedding vector)
123
+ output_tensor = helper.make_tensor_value_info(
124
+ 'embedding',
125
+ TensorProto.FLOAT,
126
+ [embedding_dim]
127
+ )
128
+
129
+ # Create initializers for the embedding matrix
130
+ embedding_initializer = numpy_helper.from_array(embeddings_matrix, name='word_embeddings')
131
+
132
+ # Create Constant node for the embedding matrix
133
+ embedding_constant = helper.make_node(
134
+ 'Constant',
135
+ inputs=[],
136
+ outputs=['embeddings_matrix'],
137
+ value=embedding_initializer
138
+ )
139
+
140
+ # Create Gather node to lookup embedding by index
141
+ gather_node = helper.make_node(
142
+ 'Gather',
143
+ inputs=['embeddings_matrix', 'word_index'],
144
+ outputs=['embedding_flat'],
145
+ axis=0
146
+ )
147
+
148
+ # Create Squeeze node to remove the batch dimension
149
+ squeeze_node = helper.make_node(
150
+ 'Squeeze',
151
+ inputs=['embedding_flat'],
152
+ outputs=['embedding'],
153
+ axes=[0] # Remove first dimension
154
+ )
155
+
156
+ # Create graph
157
+ graph = helper.make_graph(
158
+ [embedding_constant, gather_node, squeeze_node],
159
+ f'{model_name}_embedding',
160
+ [input_tensor],
161
+ [output_tensor]
162
+ )
163
+
164
+ # Create model
165
+ model = helper.make_model(
166
+ graph,
167
+ producer_name='kotoshu-fasttext-converter',
168
+ producer_version='1.0.0',
169
+ opset_imports=[helper.make_operatorsetid('', 11)], # ONNX opset 11 for compatibility
170
+ ir_version=11 # Set IR version to match opset
171
+ )
172
+
173
+ # Add metadata
174
+ from onnx import StringStringEntryProto
175
+ model.metadata_props.append(StringStringEntryProto(key='vocabulary_size', value=str(vocab_size)))
176
+ model.metadata_props.append(StringStringEntryProto(key='embedding_dimension', value=str(embedding_dim)))
177
+ model.metadata_props.append(StringStringEntryProto(key='model_type', value='fasttext_embedding'))
178
+
179
+ print(f" ONNX model created successfully")
180
+
181
+ return model
182
+
183
+
184
+ def save_vocabulary(word_to_idx, vocab_file_path):
185
+ """
186
+ Save vocabulary dictionary to JSON file.
187
+
188
+ Args:
189
+ word_to_idx: Dictionary mapping words to indices
190
+ vocab_file_path: Path to save vocabulary file
191
+ """
192
+ print(f"Saving vocabulary to: {vocab_file_path}")
193
+
194
+ vocab_data = {
195
+ "vocab_size": len(word_to_idx),
196
+ "word_to_idx": word_to_idx
197
+ }
198
+
199
+ with open(vocab_file_path, 'w', encoding='utf-8') as f:
200
+ json.dump(vocab_data, f, ensure_ascii=False, indent=2)
201
+
202
+ print(f" Vocabulary saved: {len(word_to_idx)} words")
203
+
204
+
205
+ def main():
206
+ parser = argparse.ArgumentParser(
207
+ description='Convert FastText .vec file to ONNX format'
208
+ )
209
+ parser.add_argument(
210
+ 'vec_file',
211
+ type=str,
212
+ help='Path to FastText .vec file'
213
+ )
214
+ parser.add_argument(
215
+ 'output_onnx',
216
+ type=str,
217
+ help='Path to output ONNX file'
218
+ )
219
+ parser.add_argument(
220
+ '--vocab-size',
221
+ type=int,
222
+ default=None,
223
+ help='Maximum vocabulary size (default: all words)'
224
+ )
225
+ parser.add_argument(
226
+ '--save-vocab',
227
+ type=str,
228
+ default=None,
229
+ help='Path to save vocabulary JSON file (optional)'
230
+ )
231
+
232
+ args = parser.parse_args()
233
+
234
+ # Validate input file
235
+ vec_file = Path(args.vec_file)
236
+ if not vec_file.exists():
237
+ print(f"Error: Input file not found: {vec_file}")
238
+ sys.exit(1)
239
+
240
+ # Parse FastText file
241
+ word_to_idx, embeddings_matrix, metadata = parse_fasttext_vec(
242
+ vec_file,
243
+ vocab_size=args.vocab_size
244
+ )
245
+
246
+ # Create ONNX model
247
+ model = create_onnx_model(embeddings_matrix, word_to_idx)
248
+
249
+ # Save ONNX model
250
+ output_path = Path(args.output_onnx)
251
+ output_path.parent.mkdir(parents=True, exist_ok=True)
252
+
253
+ print(f"Saving ONNX model to: {output_path}")
254
+ onnx.save(model, str(output_path))
255
+ print(f" Model saved successfully")
256
+
257
+ # Calculate file sizes
258
+ vec_size_mb = vec_file.stat().st_size / (1024 * 1024)
259
+ onnx_size_mb = output_path.stat().st_size / (1024 * 1024)
260
+
261
+ print(f"\nFile size comparison:")
262
+ print(f" Input .vec file: {vec_size_mb:.2f} MB")
263
+ print(f" Output .onnx file: {onnx_size_mb:.2f} MB")
264
+ print(f" Compression ratio: {(vec_size_mb / onnx_size_mb):.2f}x")
265
+
266
+ # Save vocabulary if requested
267
+ if args.save_vocab:
268
+ save_vocabulary(word_to_idx, args.save_vocab)
269
+
270
+ print(f"\nConversion complete!")
271
+ print(f" Model metadata: {metadata}")
272
+
273
+
274
+ if __name__ == '__main__':
275
+ main()