kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,444 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Dictionary Validation Script
5
+ #
6
+ # This script validates all dictionaries in the kotoshu/dictionaries catalog
7
+ # by loading them from GitHub and testing basic functionality.
8
+ #
9
+ # Usage:
10
+ # ruby script/validate_all_dictionaries.rb [--full] [--lang LANG] [--code CODE]
11
+ #
12
+ # Options:
13
+ # --full Run full validation including suggestion tests (slow)
14
+ # --lang LANG Only test dictionaries for this language (e.g., en, de, fr)
15
+ # --code CODE Only test this specific dictionary code (e.g., en-GB, de-AT)
16
+ # --format FMT Only test dictionaries with this format (hunspell, plain_text)
17
+
18
+ require_relative "../lib/kotoshu"
19
+ require "optparse"
20
+ require "benchmark"
21
+ require "json"
22
+
23
+ # ANSI color codes for terminal output
24
+ module Colors
25
+ RESET = "\e[0m"
26
+ RED = "\e[31m"
27
+ GREEN = "\e[32m"
28
+ YELLOW = "\e[33m"
29
+ BLUE = "\e[34m"
30
+ MAGENTA = "\e[35m"
31
+ CYAN = "\e[36m"
32
+ BOLD = "\e[1m"
33
+ end
34
+
35
+ # Validation result for a single dictionary
36
+ class ValidationResult
37
+ attr_reader :code, :status, :load_time, :size, :test_results, :error
38
+
39
+ def initialize(code)
40
+ @code = code
41
+ @status = :pending # :pending, :success, :warning, :error
42
+ @load_time = nil
43
+ @size = nil
44
+ @test_results = {}
45
+ @error = nil
46
+ end
47
+
48
+ def success!(load_time, size)
49
+ @status = :success
50
+ @load_time = load_time
51
+ @size = size
52
+ end
53
+
54
+ def warning!(load_time, size, message)
55
+ @status = :warning
56
+ @load_time = load_time
57
+ @size = size
58
+ @test_results[:warning] = message
59
+ end
60
+
61
+ def error!(error)
62
+ @status = :error
63
+ @error = error
64
+ end
65
+
66
+ def add_test_result(name, passed, details = nil)
67
+ @test_results[name] = { passed: passed, details: details }
68
+ end
69
+
70
+ def success?
71
+ @status == :success
72
+ end
73
+
74
+ def error?
75
+ @status == :error
76
+ end
77
+
78
+ def warning?
79
+ @status == :warning
80
+ end
81
+
82
+ def to_h
83
+ {
84
+ code: @code,
85
+ status: @status,
86
+ load_time: @load_time,
87
+ size: @size,
88
+ test_results: @test_results,
89
+ error: @error&.message
90
+ }
91
+ end
92
+ end
93
+
94
+ # Main validator class
95
+ class DictionaryValidator
96
+ attr_reader :options, :results
97
+
98
+ def initialize(options = {})
99
+ @options = options
100
+ @results = []
101
+ @catalog = Kotoshu::Dictionaries::Catalog
102
+ end
103
+
104
+ # Run validation
105
+ def run
106
+ print_header
107
+
108
+ dictionaries = select_dictionaries
109
+
110
+ print "Validating #{dictionaries.size} dictionaries...\n\n"
111
+
112
+ dictionaries.each_with_index do |entry, index|
113
+ validate_dictionary(entry, index + 1, dictionaries.size)
114
+ end
115
+
116
+ print_summary
117
+
118
+ write_report if @options[:report]
119
+
120
+ exit_with_code
121
+ end
122
+
123
+ private
124
+
125
+ def select_dictionaries
126
+ dicts = @catalog.all
127
+
128
+ dicts = dicts.select { |d| d.language == @options[:lang] } if @options[:lang]
129
+ dicts = [dicts.find { |d| d.code.casecmp(@options[:code]).zero? }].compact if @options[:code]
130
+ dicts = dicts.select { |d| d.format == @options[:format].to_sym } if @options[:format]
131
+
132
+ dicts
133
+ end
134
+
135
+ def validate_dictionary(entry, index, total)
136
+ result = ValidationResult.new(entry.code)
137
+
138
+ print_status(entry, index, total, result)
139
+
140
+ begin
141
+ # Load dictionary with timing
142
+ dict = nil
143
+ load_time = Benchmark.realtime do
144
+ dict = entry.load
145
+ end
146
+
147
+ # Basic validation
148
+ size = dict.size
149
+
150
+ if size.zero?
151
+ result.warning!(load_time, size, "Dictionary has zero words")
152
+ elsif size < 100
153
+ result.warning!(load_time, size, "Dictionary has fewer than 100 words")
154
+ else
155
+ result.success!(load_time, size)
156
+ end
157
+
158
+ # Run tests if --full
159
+ run_full_tests(dict, entry, result) if @options[:full] && result.success?
160
+ rescue StandardError => e
161
+ result.error!(e)
162
+ end
163
+
164
+ @results << result
165
+ print_result(entry, result)
166
+ end
167
+
168
+ def run_full_tests(dict, entry, result)
169
+ # Test 1: Lookup basic word (varies by language)
170
+ test_word = basic_test_word(entry.language)
171
+ if dict.lookup?(test_word)
172
+ result.add_test_result(:basic_lookup, true, test_word)
173
+ else
174
+ result.add_test_result(:basic_lookup, false, "Could not find '#{test_word}'")
175
+ end
176
+
177
+ # Test 2: Lookup non-existent word
178
+ nonsense_word = nonsense_test_word(entry.language)
179
+ if !dict.lookup?(nonsense_word)
180
+ result.add_test_result(:nonexistent_lookup, true, nonsense_word)
181
+ else
182
+ result.add_test_result(:nonexistent_lookup, false, "Incorrectly found '#{nonsense_word}'")
183
+ end
184
+
185
+ # Test 3: Suggestions (if supported)
186
+ begin
187
+ misspelled = misspelled_test_word(entry.language)
188
+ suggestions = dict.suggest(misspelled, max_suggestions: 5)
189
+ if suggestions&.any?
190
+ result.add_test_result(:suggestions, true, "Found #{suggestions.size} suggestions for '#{misspelled}'")
191
+ else
192
+ result.add_test_result(:suggestions, false, "No suggestions for '#{misspelled}'")
193
+ end
194
+ rescue StandardError => e
195
+ result.add_test_result(:suggestions, false, e.message)
196
+ end
197
+
198
+ # Test 4: Case sensitivity (if not case-sensitive)
199
+ return if dict.case_sensitive?
200
+
201
+ if dict.lookup?(test_word.upcase) || dict.lookup?(test_word.downcase)
202
+ result.add_test_result(:case_insensitive, true, "Case-insensitive lookup works")
203
+ else
204
+ result.add_test_result(:case_insensitive, false, "Case-insensitive lookup failed")
205
+ end
206
+ end
207
+
208
+ def basic_test_word(language)
209
+ # Common words in different languages
210
+ {
211
+ "en" => "the",
212
+ "de" => "der",
213
+ "es" => "el",
214
+ "fr" => "le",
215
+ "it" => "il",
216
+ "pt" => "o",
217
+ "ru" => "и",
218
+ "nl" => "de",
219
+ "pl" => "i",
220
+ "cs" => "a",
221
+ "sv" => "och",
222
+ "da" => "og",
223
+ "no" => "og",
224
+ "fi" => "ja",
225
+ "tr" => "ve",
226
+ "ko" => "그",
227
+ "vi" => "là",
228
+ "ja" => "は",
229
+ "zh" => "的",
230
+ "ar" => "في",
231
+ "he" => "ו",
232
+ "el" => "το",
233
+ "hu" => "a",
234
+ "ro" => "şi",
235
+ "bg" => "и",
236
+ "uk" => "і",
237
+ "ga" => "an",
238
+ "cy" => "y",
239
+ "is" => "og",
240
+ "mt" => "u",
241
+ "lv" => "un",
242
+ "et" => "ja",
243
+ "lt" => "ir",
244
+ "sk" => "a",
245
+ "sl" => "in",
246
+ "hr" => "i",
247
+ "sr" => "и",
248
+ "sq" => "dhe",
249
+ "be" => "і",
250
+ "mk" => "и",
251
+ "hy" => "և",
252
+ "ka" => "და",
253
+ "fa" => "و",
254
+ "ur" => "اور",
255
+ "hi" => "और",
256
+ "bn" => "এবং",
257
+ "th" => "และ",
258
+ "id" => "dan",
259
+ "ms" => "dan",
260
+ "sw" => "na",
261
+ "af" => "en",
262
+ "ca" => "i",
263
+ "gl" => "e",
264
+ "eu" => "eta",
265
+ "lb" => "an",
266
+ "fy" => "en",
267
+ "ku" => "û",
268
+ "eo" => "kaj",
269
+ "ia" => "e"
270
+ }.fetch(language, "a")
271
+ end
272
+
273
+ def nonsense_test_word(_language)
274
+ # Nonsense words that shouldn't exist
275
+ "zzzzzzzzz"
276
+ end
277
+
278
+ def misspelled_test_word(language)
279
+ # Common misspellings in different languages
280
+ {
281
+ "en" => "helo",
282
+ "de" => "hallo",
283
+ "es" => "ola",
284
+ "fr" => "bonjur",
285
+ "it" => "ciao",
286
+ "pt" => "ola",
287
+ "ru" => "привет",
288
+ "nl" => "halo",
289
+ "pl" => "czesc"
290
+ }.fetch(language, "teest")
291
+ end
292
+
293
+ def print_header
294
+ print "#{Colors::BOLD}Kotoshu Dictionary Validator#{Colors::RESET}\n"
295
+ print "#{"=" * 60}\n\n"
296
+
297
+ stats = @catalog.statistics
298
+ print "Catalog Statistics:\n"
299
+ print " Total dictionaries: #{stats[:total]}\n"
300
+ print " Hunspell dictionaries: #{stats[:hunspell]}\n"
301
+ print " Plain text dictionaries: #{stats[:plain_text]}\n"
302
+ print " Languages: #{stats[:languages]}\n"
303
+ print " Total words: #{stats[:total_words].round}\n"
304
+ print "\n"
305
+ print "#{"=" * 60}\n\n"
306
+ end
307
+
308
+ def print_status(entry, index, total, _result)
309
+ print "[#{index}/#{total}] #{Colors::CYAN}#{entry.code}#{Colors::RESET} - #{entry.description}\n"
310
+ print " Format: #{entry.format}, License: #{entry.license}\n"
311
+ end
312
+
313
+ def print_result(_entry, result)
314
+ if result.success?
315
+ print " #{Colors::GREEN}✓ PASS#{Colors::RESET}"
316
+ print " - #{result.size.round} words, #{(result.load_time * 1000).round(1)}ms"
317
+ print " - Tests: #{result.test_results.size}" if @options[:full]
318
+ elsif result.warning?
319
+ print " #{Colors::YELLOW}⚠ WARN#{Colors::RESET}"
320
+ print " - #{result.size.round} words, #{(result.load_time * 1000).round(1)}ms"
321
+ print " - #{result.test_results[:warning]}"
322
+ else
323
+ print " #{Colors::RED}✗ FAIL#{Colors::RESET}"
324
+ print " - #{result.error.class}: #{result.error.message}"
325
+ end
326
+ print "\n"
327
+
328
+ # Print test results details
329
+ if @options[:full] && result.test_results.any?
330
+ result.test_results.each do |name, test_result|
331
+ next if name == :warning
332
+
333
+ status = test_result[:passed] ? "#{Colors::GREEN}✓#{Colors::RESET}" : "#{Colors::RED}✗#{Colors::RESET}"
334
+ print " #{status} #{name}: #{test_result[:details]}\n"
335
+ end
336
+ end
337
+
338
+ print "\n"
339
+ end
340
+
341
+ def print_summary
342
+ print "#{"=" * 60}\n"
343
+ print "#{Colors::BOLD}Validation Summary#{Colors::RESET}\n"
344
+ print "#{"=" * 60}\n\n"
345
+
346
+ total = @results.size
347
+ success = @results.count(&:success?)
348
+ warnings = @results.count(&:warning?)
349
+ errors = @results.count(&:error?)
350
+
351
+ print "Total: #{total}\n"
352
+ print "#{Colors::GREEN}✓ Passed: #{success}#{Colors::RESET}\n"
353
+ print "#{Colors::YELLOW}⚠ Warnings: #{warnings}#{Colors::RESET}\n"
354
+ print "#{Colors::RED}✗ Failed: #{errors}#{Colors::RESET}\n"
355
+ print "\n"
356
+
357
+ if success.positive?
358
+ avg_load_time = @results.select(&:success?).map(&:load_time).sum / success
359
+ avg_size = @results.select(&:success?).map(&:size).sum / success
360
+ print "Average load time: #{(avg_load_time * 1000).round(1)}ms\n"
361
+ print "Average size: #{avg_size.round} words\n"
362
+ print "\n"
363
+ end
364
+
365
+ if errors.positive?
366
+ print "#{Colors::BOLD}Failed Dictionaries:#{Colors::RESET}\n"
367
+ @results.select(&:error?).each do |result|
368
+ print " #{Colors::RED}#{result.code}#{Colors::RESET}: #{result.error.message}\n"
369
+ end
370
+ print "\n"
371
+ end
372
+
373
+ return unless warnings.positive?
374
+
375
+ print "#{Colors::BOLD}Warnings:#{Colors::RESET}\n"
376
+ @results.select(&:warning?).each do |result|
377
+ print " #{Colors::YELLOW}#{result.code}#{Colors::RESET}: #{result.test_results[:warning]}\n"
378
+ end
379
+ print "\n"
380
+ end
381
+
382
+ def write_report
383
+ report_path = "dictionary_validation_report.json"
384
+ File.write(report_path, JSON.pretty_generate({
385
+ timestamp: Time.now.iso8601,
386
+ summary: {
387
+ total: @results.size,
388
+ success: @results.count(&:success?),
389
+ warnings: @results.count(&:warning?),
390
+ errors: @results.count(&:error?)
391
+ },
392
+ results: @results.map(&:to_h)
393
+ }))
394
+ print "Report written to: #{report_path}\n"
395
+ end
396
+
397
+ def exit_with_code
398
+ # Exit with error code if any failures
399
+ exit 1 if @results.any?(&:error?)
400
+ exit 0
401
+ end
402
+ end
403
+
404
+ # Parse options
405
+ options = {
406
+ full: false,
407
+ lang: nil,
408
+ code: nil,
409
+ format: nil,
410
+ report: false
411
+ }
412
+
413
+ OptionParser.new do |opts|
414
+ opts.banner = "Usage: ruby script/validate_all_dictionaries.rb [options]"
415
+
416
+ opts.on("--full", "Run full validation including tests") do
417
+ options[:full] = true
418
+ end
419
+
420
+ opts.on("--lang LANG", "Filter by language (e.g., en, de, fr)") do |lang|
421
+ options[:lang] = lang
422
+ end
423
+
424
+ opts.on("--code CODE", "Filter by dictionary code (e.g., en-GB)") do |code|
425
+ options[:code] = code
426
+ end
427
+
428
+ opts.on("--format FORMAT", "Filter by format (hunspell, plain_text)") do |fmt|
429
+ options[:format] = fmt
430
+ end
431
+
432
+ opts.on("--report", "Write JSON report file") do
433
+ options[:report] = true
434
+ end
435
+
436
+ opts.on("-h", "--help", "Show this message") do
437
+ puts opts
438
+ exit
439
+ end
440
+ end.parse!
441
+
442
+ # Run validator
443
+ validator = DictionaryValidator.new(options)
444
+ validator.run
@@ -0,0 +1,45 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Build a precompiled platform gem (plan 133): assemble kotoshu's
5
+ # gemspec around the ALREADY-COMPILED extension binary, exactly the
6
+ # surgery rb_sys's cross-compiling blocks perform — the rb_sys build
7
+ # dependency goes away (nothing builds at install), the Rust sources,
8
+ # Cargo files, and extconf drop out of the file list, and the platform
9
+ # tag makes RubyGems resolve this gem on matching platforms while the
10
+ # pure-Ruby gem keeps serving everything else.
11
+ #
12
+ # Usage: ruby scripts/build_native_gem.rb [platform]
13
+ # platform defaults to Gem::Platform.local; pass e.g. x86_64-linux
14
+ # when the binary was produced for another host. The compiled
15
+ # extension must already sit at lib/kotoshu/kotoshu_native.<dlext>
16
+ # (rake compile on the matching platform).
17
+
18
+ require "rubygems/package"
19
+ require "rbconfig"
20
+
21
+ platform = ARGV[0]
22
+ dlext = RbConfig::CONFIG["DLEXT"]
23
+ binary = File.join("lib", "kotoshu", "kotoshu_native.#{dlext}")
24
+
25
+ unless File.exist?(binary)
26
+ abort "missing #{binary} — run `rake compile` on the target platform first"
27
+ end
28
+
29
+ spec = Gem::Specification.load("kotoshu.gemspec")
30
+ spec.platform = platform ? Gem::Platform.new(platform) : Gem::Platform.local
31
+ # The darwin deployment-target suffix (-23, -25, ...) differs per build
32
+ # machine; RubyGems matches a versionless darwin tag on any macOS of
33
+ # the same CPU, so normalize it off (the convention shipped binary
34
+ # gems use). Linux and mingw tags carry no version to begin with.
35
+ if spec.platform.os == "darwin" && spec.platform.version
36
+ spec.platform = Gem::Platform.new("#{spec.platform.cpu}-darwin")
37
+ end
38
+ spec.files += [binary]
39
+ spec.extensions = []
40
+ spec.dependencies.reject! { |d| d.name == "rb_sys" }
41
+ spec.files.reject! { |f| f.start_with?("ext/") }
42
+
43
+ path = Gem::Package.build(spec)
44
+ puts "built #{path}"
45
+ path
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # One-off generator for spec/fixtures/models_eval_grids.json.
4
+ # Not part of the gem or the suite; kept next to the fixture it makes.
5
+ #
6
+ # Reads the models repo eval noise.py and extracts the tr/uk/el key
7
+ # grids into the JSON fixture used by the drift-check spec.
8
+ #
9
+ # Usage: ruby scripts/extract_eval_grids.rb PATH_TO_NOISE_PY
10
+
11
+ require "json"
12
+
13
+ source = ARGV[0]
14
+ abort "usage: ruby #{__FILE__} PATH_TO_models-fasttext-onnx/eval/noise.py" unless source
15
+ abort "noise.py not found: #{source}" unless File.exist?(source)
16
+
17
+ src = File.read(source)
18
+ grids = {}
19
+ %w[
20
+ _TR_Q _UK_JCUKEN _EL_PHONETIC _AR_101 _FA_STD _HE_SI1452 _BG_BDS
21
+ _SR_CYR _HRSL _NE_INSCRIPT
22
+ ].each do |name|
23
+ body = src[/^#{name} = \{(.*?)^\}/m, 1]
24
+ abort "grid #{name} not found in #{source}" unless body
25
+ grid = {}
26
+ body.scan(/"((?:[^"\\]|\\.)*)"\s*:\s*\((\d+),\s*(\d+)\)/) do |key, row, col|
27
+ key = key.gsub('\\"', '"').gsub("\\\\", "\\")
28
+ grid[key] = [Integer(row), Integer(col)]
29
+ end
30
+ abort "grid #{name} empty" if grid.empty?
31
+
32
+ grids[name] = grid
33
+ end
34
+
35
+ out = File.expand_path("../spec/fixtures/models_eval_grids.json", __dir__)
36
+ File.write(out, JSON.pretty_generate({ "source" => source, "grids" => grids }) + "\n")
37
+ puts "wrote #{out}: #{grids.map { |k, v| "#{k}=#{v.size}" }.join(' ')}"
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Generates lib/kotoshu/data/frozen_kelly.rb from the Rust engine's
4
+ # frozen tier tables (kotoshu-rs kotoshu/src/suggest/frequency_data.rs)
5
+ # so both engines carry byte-identical frequency data for `en`
6
+ # (plan 119). The rs tables are themselves frozen from the Kelly
7
+ # en.json sha256 97535823f41aa3f0… — this script preserves that
8
+ # provenance rather than re-deriving from the raw Kelly data.
9
+ #
10
+ # Usage: ruby scripts/generate_frozen_tiers.rb <path-to-frequency_data.rs>
11
+ #
12
+ # The generator is deterministic: same input file, same output bytes.
13
+
14
+ require "json"
15
+
16
+ SRC = ARGV[0]
17
+ abort "usage: #{$PROGRAM_NAME} <kotoshu-rs frequency_data.rs>" unless SRC
18
+ abort "not found: #{SRC}" unless File.exist?(SRC)
19
+
20
+ def parse_tier(source, name)
21
+ match = source.match(/pub const #{name}: &\[&str\] = &\[(.*?)\];/m)
22
+ abort "tier #{name} not found in #{SRC}" unless match
23
+
24
+ match[1].scan(/"([^"]+)"/).flatten
25
+ end
26
+
27
+ source = File.read(SRC)
28
+ tiers = { "top_50" => parse_tier(source, "TOP_50"),
29
+ "top_200" => parse_tier(source, "TOP_200"),
30
+ "top_1000" => parse_tier(source, "TOP_1000") }
31
+ abort "unparsed" if tiers.values.any?(&:empty?)
32
+
33
+ data = {
34
+ "provenance" => {
35
+ "source" => "kotoshu-rs kotoshu/src/suggest/frequency_data.rs",
36
+ "upstream" => "kotoshu/frequency-list-kelly data/en.json",
37
+ "upstream_sha256" => "97535823f41aa3f06c2087dfc162aec0a591412b8b95c0594fcbb2cea79b0e61"
38
+ },
39
+ "tiers" => tiers.transform_keys(&:downcase)
40
+ }
41
+
42
+ dest = File.expand_path("../lib/kotoshu/data/frozen_kelly/en.json", __dir__)
43
+ File.write(dest, JSON.pretty_generate(data) + "\n")
44
+ puts "wrote #{dest}: #{tiers.map { |k, v| "#{k}=#{v.size}" }.join(' ')}"