kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,287 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 6: Configuration and Advanced Usage
5
+ #
6
+ # This example demonstrates how to configure Kotoshu and use
7
+ # advanced features like custom words, multiple languages, etc.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 6: Configuration and Advanced Usage ==="
12
+ puts
13
+
14
+ # Example 1: Global Configuration
15
+ puts "1. Global Configuration"
16
+ puts "-" * 40
17
+
18
+ # Use system dictionary instead of non-existent path
19
+ dict_path = if File.exist?("/usr/share/dict/words")
20
+ "/usr/share/dict/words"
21
+ elsif File.exist?("dictionaries/unix_words/words")
22
+ "dictionaries/unix_words/words"
23
+ else
24
+ # Will use auto-detected system dictionary
25
+ nil
26
+ end
27
+
28
+ Kotoshu.configure do |config|
29
+ config.dictionary_type = :unix_words
30
+ config.dictionary_path = dict_path
31
+ config.language = "en-US"
32
+ config.max_suggestions = 15
33
+ config.case_sensitive = false
34
+ config.custom_words = %w[Kotoshu spellcheck]
35
+ end
36
+
37
+ puts "Configuration:"
38
+ config = Kotoshu.configuration
39
+ puts " Dictionary type: #{config.dictionary_type}"
40
+ puts " Dictionary path: #{config.dictionary_path}"
41
+ puts " Language: #{config.language}"
42
+ puts " Max suggestions: #{config.max_suggestions}"
43
+ puts " Case sensitive: #{config.case_sensitive}"
44
+ puts " Custom words: #{config.custom_words.inspect}"
45
+ puts
46
+
47
+ # Use the configured spellchecker
48
+ puts "Using configured spellchecker:"
49
+ puts " Has 'hello': #{Kotoshu.correct?("hello")}"
50
+ puts " Has 'Kotoshu': #{Kotoshu.correct?("Kotoshu")}"
51
+ suggestions = Kotoshu.suggest("helo")
52
+ puts " Suggestions for 'helo': #{suggestions.to_words.first(10).join(", ")}..."
53
+
54
+ puts
55
+ puts "=" * 40
56
+ puts
57
+
58
+ # Example 2: Spellchecker Instance with Custom Configuration
59
+ puts "2. Custom Spellchecker Instance"
60
+ puts "-" * 40
61
+
62
+ # Create a custom dictionary
63
+ custom_dict = Kotoshu::Dictionary::Custom.new(
64
+ words: %w[ruby gem rspec rake bundler],
65
+ language_code: "en"
66
+ )
67
+
68
+ # Create a spellchecker with the custom dictionary
69
+ custom_spellchecker = Kotoshu::Spellchecker.new(dictionary: custom_dict)
70
+
71
+ puts "Custom spellchecker with Ruby-related words:"
72
+ puts " Has 'ruby': #{custom_spellchecker.correct?("ruby")}"
73
+ puts " Has 'gem': #{custom_spellchecker.correct?("gem")}"
74
+ puts " Has 'rake': #{custom_spellchecker.correct?("rake")}"
75
+ puts " Has 'python': #{custom_spellchecker.correct?("python")}"
76
+ puts " Suggestions for 'rke': #{custom_spellchecker.suggest("rke").to_words.join(", ")}"
77
+
78
+ puts
79
+ puts "=" * 40
80
+ puts
81
+
82
+ # Example 3: Dictionary Repository
83
+ puts "3. Dictionary Repository"
84
+ puts "-" * 40
85
+
86
+ repo = Kotoshu::Dictionary::Repository.new
87
+
88
+ # Register multiple dictionaries
89
+ repo.register(:en_US, custom_dict)
90
+ repo.register(:programming, Kotoshu::Dictionary::PlainText.from_words(
91
+ %w[code function variable class module],
92
+ language_code: "en"
93
+ ))
94
+ repo.register(:tech, Kotoshu::Dictionary::PlainText.from_words(
95
+ %w[computer software hardware internet api],
96
+ language_code: "en"
97
+ ))
98
+
99
+ puts "Registered dictionaries:"
100
+ repo.each_key do |key|
101
+ dict = repo.get(key)
102
+ puts " #{key}: #{dict.size} words (#{dict.type})"
103
+ end
104
+
105
+ puts "\nFind by language 'en':"
106
+ found = repo.find_by_language("en")
107
+ found.each do |dict|
108
+ puts " #{dict.type}: #{dict.size} words"
109
+ end
110
+
111
+ puts
112
+ puts "=" * 40
113
+ puts
114
+
115
+ # Example 4: IndexedDictionary
116
+ puts "4. IndexedDictionary (Rich Query Interface)"
117
+ puts "-" * 40
118
+
119
+ index_dict = Kotoshu.dictionary(%w[
120
+ hello help held heap
121
+ world work word
122
+ test text toast
123
+ run running runner
124
+ code coding coded
125
+ ])
126
+
127
+ puts "IndexedDictionary: #{index_dict.size} words"
128
+ puts
129
+
130
+ puts "Query methods:"
131
+ puts " Words starting with 'he': #{index_dict.find_by_prefix("he").inspect}"
132
+ puts " Words ending with 'ld': #{index_dict.find_by_suffix("ld").inspect}"
133
+ puts " Words with length 3: #{index_dict.find_by_length(3).inspect}"
134
+ puts " Words matching pattern 't.*t': #{index_dict.find_by_pattern(/t.*t/).inspect}"
135
+ puts
136
+
137
+ puts "Statistics:"
138
+ stats = index_dict.statistics
139
+ stats.each do |key, value|
140
+ puts " #{key}: #{value}"
141
+ end
142
+
143
+ puts
144
+ puts "=" * 40
145
+ puts
146
+
147
+ # Example 5: WordResult and DocumentResult
148
+ puts "5. Result Objects"
149
+ puts "-" * 40
150
+
151
+ # Check a word
152
+ word_result = Kotoshu.spellchecker.check_word("hello")
153
+ puts "WordResult for 'hello':"
154
+ puts " Word: #{word_result.word}"
155
+ puts " Correct: #{word_result.correct?}"
156
+ puts " Has suggestions: #{word_result.has_suggestions?}"
157
+ puts
158
+
159
+ word_result2 = Kotoshu.spellchecker.check_word("helo")
160
+ puts "WordResult for 'helo':"
161
+ puts " Word: #{word_result2.word}"
162
+ puts " Correct: #{word_result2.correct?}"
163
+ puts " Suggestion count: #{word_result2.suggestion_count}"
164
+ puts " First suggestion: #{word_result2.first_suggestion}"
165
+ puts " Top 3: #{word_result2.top_suggestions(3).join(", ")}"
166
+ puts
167
+
168
+ # Check text
169
+ text_result = Kotoshu.spellchecker.check("Hello wrold! This is a tst.")
170
+ puts "DocumentResult:"
171
+ puts " Success: #{text_result.success?}"
172
+ puts " Word count: #{text_result.word_count}"
173
+ puts " Error count: #{text_result.error_count}"
174
+ puts " Unique errors: #{text_result.unique_error_count}"
175
+ puts
176
+ puts " Errors:"
177
+ text_result.errors.each do |error|
178
+ suggestions_str = if error.has_suggestions?
179
+ " (suggestions: #{error.top_suggestions(2).join(", ")})"
180
+ else
181
+ ""
182
+ end
183
+ puts " • #{error.word} at position #{error.position}#{suggestions_str}"
184
+ end
185
+
186
+ puts
187
+ puts "=" * 40
188
+ puts
189
+
190
+ # Example 6: Multiple File Checking
191
+ puts "6. Batch File Checking"
192
+ puts "-" * 40
193
+
194
+ # Check multiple files
195
+ fixtures_dir = "spec/fixtures/documents"
196
+ if Dir.exist?(fixtures_dir)
197
+ files = Dir.glob(File.join(fixtures_dir, "*.txt"))
198
+ puts "Checking #{files.size} files..."
199
+ puts
200
+
201
+ files.each do |file|
202
+ result = Kotoshu.check_file(file)
203
+ status = result.success? ? "✓" : "✗"
204
+ filename = File.basename(file)
205
+ puts "#{status} #{filename}: #{result.error_count} error(s), #{result.word_count} words"
206
+ end
207
+
208
+ puts
209
+
210
+ # Get all results at once
211
+ results = Kotoshu.check_files(files)
212
+ total_errors = results.sum(&:error_count)
213
+ total_words = results.sum(&:word_count)
214
+ failed_count = results.count(&:failed?)
215
+
216
+ puts "Summary:"
217
+ puts " Files checked: #{files.size}"
218
+ puts " Files with errors: #{failed_count}"
219
+ puts " Total errors: #{total_errors}"
220
+ puts " Total words: #{total_words}"
221
+ end
222
+
223
+ puts
224
+ puts "=" * 40
225
+ puts
226
+
227
+ # Example 7: Error Handling
228
+ puts "7. Error Handling"
229
+ puts "-" * 40
230
+
231
+ begin
232
+ # Try to load a non-existent dictionary
233
+ bad_config = Kotoshu::Configuration.new(
234
+ dictionary_type: :plain_text,
235
+ dictionary_path: "/nonexistent/path.txt"
236
+ )
237
+ bad_config.load_dictionary
238
+ rescue Kotoshu::DictionaryNotFoundError => e
239
+ puts "Caught DictionaryNotFoundError:"
240
+ puts " Message: #{e.message}"
241
+ puts " Path: #{e.path}"
242
+ end
243
+
244
+ puts
245
+
246
+ begin
247
+ # Try to use an invalid dictionary type
248
+ bad_config2 = Kotoshu::Configuration.new(
249
+ dictionary_type: :invalid_type
250
+ )
251
+ bad_config2.load_dictionary
252
+ rescue Kotoshu::ConfigurationError => e
253
+ puts "Caught ConfigurationError:"
254
+ puts " Message: #{e.message}"
255
+ puts " Key: #{e.key.inspect}"
256
+ end
257
+
258
+ puts
259
+ puts "=" * 40
260
+ puts
261
+
262
+ # Example 8: Thread Safety (each instance is independent)
263
+ puts "8. Thread Safety"
264
+ puts "-" * 40
265
+
266
+ # Create two independent spellcheckers
267
+ spell1 = Kotoshu::Spellchecker.new(
268
+ dictionary: Kotoshu::Dictionary::Custom.new(
269
+ words: %w[hello world],
270
+ language_code: "en"
271
+ )
272
+ )
273
+
274
+ spell2 = Kotoshu::Spellchecker.new(
275
+ dictionary: Kotoshu::Dictionary::Custom.new(
276
+ words: %w[ruby python],
277
+ language_code: "en"
278
+ )
279
+ )
280
+
281
+ puts "Spellchecker 1 words: #{spell1.dictionary.words.inspect}"
282
+ puts "Spellchecker 2 words: #{spell2.dictionary.words.inspect}"
283
+ puts
284
+ puts "Spellchecker 1 has 'hello': #{spell1.correct?("hello")}"
285
+ puts "Spellchecker 1 has 'ruby': #{spell1.correct?("ruby")}"
286
+ puts "Spellchecker 2 has 'hello': #{spell2.correct?("hello")}"
287
+ puts "Spellchecker 2 has 'ruby': #{spell2.correct?("ruby")}"
@@ -0,0 +1,278 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Example 7: Multi-Language Dictionaries
5
+ #
6
+ # This example demonstrates how to use Kotoshu with multiple languages
7
+ # by loading dictionaries from the kotoshu/dictionaries repository.
8
+
9
+ require_relative "../lib/kotoshu"
10
+
11
+ puts "=== Example 7: Multi-Language Dictionaries ==="
12
+ puts
13
+
14
+ # Example 1: Load a specific dictionary by code
15
+ puts "1. Loading a Dictionary by Code"
16
+ puts "-" * 40
17
+
18
+ catalog = Kotoshu::Dictionaries::Catalog
19
+
20
+ # Find and load British English dictionary
21
+ en_gb_entry = catalog.find("en-GB")
22
+ if en_gb_entry
23
+ puts "Found: #{en_gb_entry.description}"
24
+ puts "Source: #{en_gb_entry.source}"
25
+ puts "License: #{en_gb_entry.license}"
26
+ puts "URL: #{en_gb_entry.dic_url}"
27
+ puts
28
+
29
+ en_gb_dict = en_gb_entry.load
30
+ puts "Loaded #{en_gb_dict.size} words"
31
+ puts "Has 'colour': #{en_gb_dict.lookup?("colour")}"
32
+ puts "Has 'color': #{en_gb_dict.lookup?("color")}"
33
+ else
34
+ puts "Dictionary not found"
35
+ end
36
+
37
+ puts
38
+ puts "=" * 40
39
+ puts
40
+
41
+ # Example 2: List all dictionaries for a language
42
+ puts "2. All English Dictionaries"
43
+ puts "-" * 40
44
+
45
+ english_dicts = catalog.by_language("en")
46
+ puts "Found #{english_dicts.size} English dictionaries:"
47
+ english_dicts.each do |entry|
48
+ puts " #{entry.code}: #{entry.name} (#{entry.word_count} words)"
49
+ end
50
+
51
+ puts
52
+ puts "=" * 40
53
+ puts
54
+
55
+ # Example 3: List all available languages
56
+ puts "3. All Available Languages"
57
+ puts "-" * 40
58
+
59
+ languages = catalog.languages
60
+ puts "Supported languages (#{languages.size}):"
61
+ puts languages.join(", ")
62
+
63
+ puts
64
+ puts "=" * 40
65
+ puts
66
+
67
+ # Example 4: Create spellcheckers for different languages
68
+ puts "4. Multi-Language Spellcheckers"
69
+ puts "-" * 40
70
+
71
+ # Load dictionaries for multiple languages
72
+ languages_to_test = %w[en de es fr]
73
+
74
+ spellcheckers = {}
75
+ languages_to_test.each do |lang|
76
+ entry = catalog.find(lang)
77
+ next unless entry
78
+
79
+ begin
80
+ dict = entry.load
81
+ spellcheckers[lang] = Kotoshu::Spellchecker.new(dictionary: dict)
82
+ puts "✓ Loaded #{entry.name}: #{dict.size} words"
83
+ rescue StandardError => e
84
+ puts "✗ Failed to load #{entry.name}: #{e.message}"
85
+ end
86
+ end
87
+
88
+ puts
89
+ puts "Testing multi-language spellchecking:"
90
+ puts
91
+
92
+ # Test words in different languages
93
+ test_cases = {
94
+ "en" => { correct: "hello", incorrect: "helo" },
95
+ "de" => { correct: "hallo", incorrect: "hllo" },
96
+ "es" => { correct: "hola", incorrect: "hla" },
97
+ "fr" => { correct: "bonjour", incorrect: "bnjour" }
98
+ }
99
+
100
+ test_cases.each do |lang, words|
101
+ checker = spellcheckers[lang]
102
+ next unless checker
103
+
104
+ correct_result = checker.correct?(words[:correct])
105
+ incorrect_result = checker.check_word(words[:incorrect])
106
+
107
+ status = correct_result ? "✓" : "✗"
108
+ puts "#{status} #{lang.upcase} '#{words[:correct]}': #{correct_result}"
109
+
110
+ puts " Suggestions for '#{words[:incorrect]}': #{incorrect_result.top_suggestions(3).join(", ")}" if incorrect_result.has_suggestions?
111
+ end
112
+
113
+ puts
114
+ puts "=" * 40
115
+ puts
116
+
117
+ # Example 5: Hunspell vs Plain Text formats
118
+ puts "5. Dictionary Formats"
119
+ puts "-" * 40
120
+
121
+ hunspell_dicts = catalog.hunspell
122
+ plain_text_dicts = catalog.plain_text
123
+
124
+ puts "Hunspell dictionaries: #{hunspell_dicts.size}"
125
+ puts "Plain text dictionaries: #{plain_text_dicts.size}"
126
+ puts
127
+
128
+ # Show some examples of each
129
+ puts "Hunspell examples:"
130
+ hunspell_dicts.first(5).each do |entry|
131
+ puts " #{entry.code}: #{entry.description}"
132
+ end
133
+
134
+ puts
135
+ puts "Plain text examples:"
136
+ plain_text_dicts.each do |entry|
137
+ puts " #{entry.code}: #{entry.description}"
138
+ end
139
+
140
+ puts
141
+ puts "=" * 40
142
+ puts
143
+
144
+ # Example 6: Filter by license
145
+ puts "6. Dictionaries by License"
146
+ puts "-" * 40
147
+
148
+ public_domain = catalog.by_license("Public Domain")
149
+ gpl = catalog.by_license("GPL")
150
+
151
+ puts "Public Domain dictionaries: #{public_domain.size}"
152
+ public_domain.each do |entry|
153
+ puts " #{entry.code}: #{entry.name}"
154
+ end
155
+
156
+ puts
157
+ puts "GPL dictionaries: #{gpl.size}"
158
+ gpl.first(5).each do |entry|
159
+ puts " #{entry.code}: #{entry.name}"
160
+ end
161
+
162
+ puts
163
+ puts "=" * 40
164
+ puts
165
+
166
+ # Example 7: Catalog statistics
167
+ puts "7. Catalog Statistics"
168
+ puts "-" * 40
169
+
170
+ stats = catalog.statistics
171
+
172
+ puts "Total dictionaries: #{stats[:total]}"
173
+ puts " Hunspell: #{stats[:hunspell]}"
174
+ puts " Plain text: #{stats[:plain_text]}"
175
+ puts
176
+ puts "Languages: #{stats[:languages]}"
177
+ puts "Total words: #{stats[:total_words].round}"
178
+ puts
179
+ puts "By format:"
180
+ stats[:formats].each do |format, count|
181
+ puts " #{format}: #{count}"
182
+ end
183
+
184
+ puts
185
+ puts "By license:"
186
+ stats[:licenses].each do |license, count|
187
+ puts " #{license}: #{count}"
188
+ end
189
+
190
+ puts
191
+ puts "=" * 40
192
+ puts
193
+
194
+ # Example 8: Create spellcheckers with regional variants
195
+ puts "8. English Regional Variants"
196
+ puts "-" * 40
197
+
198
+ english_variants = %w[en en-GB en-CA en-AU en-ZA]
199
+
200
+ english_variants.each do |code|
201
+ entry = catalog.find(code)
202
+ next unless entry
203
+
204
+ begin
205
+ dict = entry.load
206
+ checker = Kotoshu::Spellchecker.new(dictionary: dict)
207
+
208
+ # Test a word with different spellings
209
+ colour_result = checker.correct?("colour")
210
+ color_result = checker.correct?("color")
211
+
212
+ puts "#{entry.name}:"
213
+ puts " 'colour': #{colour_result ? "✓" : "✗"}"
214
+ puts " 'color': #{color_result ? "✓" : "✗"}"
215
+ rescue StandardError => e
216
+ puts "#{entry.name}: ✗ Error - #{e.message}"
217
+ end
218
+ end
219
+
220
+ puts
221
+ puts "=" * 40
222
+ puts
223
+
224
+ # Example 9: Loading large dictionaries with performance
225
+ puts "9. Large Dictionary Performance"
226
+ puts "-" * 40
227
+
228
+ require "benchmark"
229
+
230
+ large_dicts = %w[en de es fr ru]
231
+
232
+ large_dicts.each do |lang|
233
+ entry = catalog.find(lang)
234
+ next unless entry
235
+
236
+ begin
237
+ load_time = Benchmark.realtime do
238
+ dict = entry.load
239
+ checker = Kotoshu::Spellchecker.new(dictionary: dict)
240
+ checker.correct?("hello")
241
+ end
242
+
243
+ puts "#{entry.name}: #{(load_time * 1000).round(1)}ms (load + check)"
244
+ rescue StandardError => e
245
+ puts "#{entry.name}: ✗ Error - #{e.message}"
246
+ end
247
+ end
248
+
249
+ puts
250
+ puts "=" * 40
251
+ puts
252
+
253
+ # Example 10: Dictionary metadata
254
+ puts "10. Dictionary Metadata"
255
+ puts "-" * 40
256
+
257
+ entry = catalog.find("ru")
258
+ if entry
259
+ puts "Code: #{entry.code}"
260
+ puts "Name: #{entry.name}"
261
+ puts "Language: #{entry.language}"
262
+ puts "Region: #{entry.region || "N/A"}"
263
+ puts "Format: #{entry.format}"
264
+ puts "Source: #{entry.source}"
265
+ puts "License: #{entry.license}"
266
+ puts "Word count: #{entry.word_count}"
267
+ puts "Dictionary URL: #{entry.dic_url}"
268
+ puts "Affix URL: #{entry.aff_url}" if entry.aff_url
269
+ puts "Metadata: #{entry.metadata.inspect}"
270
+ end
271
+
272
+ puts
273
+ puts "=" * 40
274
+ puts
275
+
276
+ puts "For more information, see:"
277
+ puts " https://github.com/kotoshu/dictionaries"
278
+ puts
data/exe/kotoshu ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "kotoshu"
5
+
6
+ Kotoshu::Cli::Cli.start(ARGV)