kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,476 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Japanese language implementation.
6
+ #
7
+ # Supports ja-JP with full CJK script support.
8
+ #
9
+ # Uses morphological analysis via Suika gem for tokenization and POS tagging.
10
+ # Japanese spell checking uses dictionary lookup with CJK character support.
11
+ class Japanese < Language::Base
12
+ # Japanese spell checker using dictionary lookup.
13
+ #
14
+ # Japanese uses morphological analysis rather than traditional Hunspell
15
+ # dictionaries. Spell checking is done through dictionary lookup of segmented
16
+ # words from the morphological analyzer.
17
+ class SpellChecker < Components::SpellChecker
18
+ attr_reader :dic_path, :script
19
+
20
+ def initialize(dic_path:, script: :cjk)
21
+ @dic_path = dic_path
22
+ @script = script
23
+ # Japanese dictionaries are typically in custom formats
24
+ # Load dictionary into memory for fast lookup
25
+ @dictionary = load_dictionary(dic_path)
26
+ end
27
+
28
+ def check(word)
29
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
30
+
31
+ # Check if word exists in dictionary
32
+ found = @dictionary.include?(word)
33
+
34
+ if found
35
+ { found: true, stem: word, flags: [] }
36
+ else
37
+ # For CJK text, we might want to check if it contains valid characters
38
+ # but not actual word validation
39
+ { found: false, stem: nil, flags: [] }
40
+ end
41
+ end
42
+
43
+ def suggest(word, max_suggestions: 10)
44
+ return [] if word.nil? || word.empty?
45
+ return [] if @dictionary.include?(word)
46
+
47
+ # Generate suggestions based on common Japanese errors
48
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
49
+ end
50
+
51
+ def correct?(word)
52
+ check(word)[:found]
53
+ end
54
+
55
+ private
56
+
57
+ def load_dictionary(path)
58
+ # Simple in-memory dictionary for Japanese words
59
+ # In production, this would use a proper CJK dictionary
60
+ @dictionary = Set.new
61
+ if File.exist?(path)
62
+ File.readlines(path, encoding: 'UTF-8').each do |line|
63
+ @dictionary.add(line.strip)
64
+ end
65
+ end
66
+ @dictionary
67
+ end
68
+
69
+ def generate_suggestions(word, max_suggestions)
70
+ variations = []
71
+
72
+ # Japanese character substitutions (common errors)
73
+ japanese_substitutions = {
74
+ 'あ' => %w[ああ],
75
+ 'い' => %w[いい],
76
+ 'う' => %w[うう],
77
+ 'え' => %w[ええ],
78
+ 'お' => %w[おお],
79
+ 'か' => %w[かが],
80
+ 'き' => %w[きぎ],
81
+ 'く' => %w[くぐ],
82
+ 'け' => %w[けげ],
83
+ 'こ' => %w[こご],
84
+ 'さ' => %w[さざ],
85
+ 'し' => %w[しじ],
86
+ 'す' => %w[すず],
87
+ 'せ' => %w[せぜ],
88
+ 'そ' => %w[そぞ],
89
+ 'た' => %w[ただ],
90
+ 'ち' => %w[ちぢ],
91
+ 'つ' => %w[つづ],
92
+ 'て' => %w[てで],
93
+ 'と' => %w[とど],
94
+ 'は' => %w[はば],
95
+ 'ひ' => %w[ひび],
96
+ 'ふ' => %w[ふぶ],
97
+ 'へ' => %w[へべ],
98
+ 'ほ' => %w[ほぼ],
99
+ 'ま' => %w[まま],
100
+ 'み' => %w[みみ],
101
+ 'む' => %w[むむ],
102
+ 'め' => %w[めめ],
103
+ 'も' => %w[もも],
104
+ 'や' => %w[やや],
105
+ 'ゆ' => %w[ゆゆ],
106
+ 'よ' => %w[よよ],
107
+ 'ら' => %w[らら],
108
+ 'り' => %w[りり],
109
+ 'る' => %w[るる],
110
+ 'れ' => %w[れれ],
111
+ 'ろ' => %w[ろろ],
112
+ 'わ' => %w[わわ],
113
+ 'を' => %w[お],
114
+ }
115
+
116
+ word.chars.each_with_index do |char, i|
117
+ next unless japanese_substitutions.key?(char)
118
+
119
+ japanese_substitutions[char].each do |sub|
120
+ substituted = word.dup
121
+ substituted[i] = sub
122
+ variations << substituted if @dictionary.include?(substituted)
123
+ end
124
+ end
125
+
126
+ # Suggest similar dictionary words
127
+ if word.length >= 2
128
+ @dictionary.each do |dict_word|
129
+ distance = levenshtein_distance(word, dict_word)
130
+ if distance <= 2 && distance > 0
131
+ variations << dict_word
132
+ end
133
+ break if variations.length >= max_suggestions * 2
134
+ end
135
+ end
136
+
137
+ variations.uniq.first(max_suggestions)
138
+ end
139
+
140
+ def levenshtein_distance(a, b)
141
+ return a.length if b.empty?
142
+ return b.length if a.empty?
143
+
144
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
145
+ (1..b.length).each { |j| matrix[0][j] = j }
146
+ (1..a.length).each do |i|
147
+ (1..b.length).each do |j|
148
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
149
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
150
+ end
151
+ end
152
+ matrix[a.length][b.length]
153
+ end
154
+ end
155
+
156
+ # Japanese tokenizer with morphological analysis.
157
+ class Tokenizer < Language::Tokenizer::JapaneseTokenizer
158
+ end
159
+
160
+ # Japanese POS tagger using morphological analysis.
161
+ #
162
+ # Japanese POS tagging is integrated with tokenization via Suika gem,
163
+ # which provides both segmentation and part-of-speech information.
164
+ #
165
+ # Suika output format: surface<TAB>POS,subcat1,subcat2,subcat3,conj_type,conj_form,lemma,reading,pronunciation
166
+ # Example: "すもも\t名詞,一般,*,*,*,*,すもも,スモモ,スモモ"
167
+ #
168
+ # POS tags use universal English categories for common types, and ROMAJI
169
+ # (Latin script) identifiers based on Japanese terminology only for
170
+ # language-specific categories without universal equivalents.
171
+ class POSTagger < Components::PosTagger
172
+ # Japanese POS tag mappings from Suika to standard identifiers.
173
+ #
174
+ # Strategy: Use universal English POS tags (NOUN, VERB, etc.) with
175
+ # English suffixes for subcategories. All identifiers are ASCII.
176
+ #
177
+ # Main categories (field 0) - universal:
178
+ # - 名詞 → NOUN
179
+ # - 動詞 → VERB
180
+ # - 助詞 → PARTICLE
181
+ # - 助動詞 → AUX
182
+ #
183
+ # Noun subcategories (field 1):
184
+ # - NOUN_COMMON: 一般 - common nouns
185
+ # - NOUN_PROPER: 固有名詞 - proper nouns
186
+ # - NOUN_PROPER_GEOGRAPHIC: 固有名詞,地域 - proper noun, geographic
187
+ # - NOUN_SUFFIX: 接尾 - suffixes
188
+ # - NOUN_DEPENDENT: 非自立 - dependent nouns (cannot stand alone)
189
+ # - NOUN_SA_CONNECTION: サ変接続 - sa-variant connection nouns
190
+ #
191
+ # Particle subcategories (field 1):
192
+ # - PARTICLE_GRAMMAR: 格助詞 - grammar/case particles (が, を, に, etc.)
193
+ # - PARTICLE_BINDING: 係助詞 - binding particles (は, も, etc.)
194
+ # - PARTICLE_ADNOMINAL: 連体化 - adnominal particles (の)
195
+ #
196
+ # Verb subcategories (field 1):
197
+ # - VERB_INDEPENDENT: 自立 - independent verbs
198
+ FLAG_TO_POS = {
199
+ # Main categories - universal English
200
+ '名詞' => 'NOUN',
201
+ '動詞' => 'VERB',
202
+ '助詞' => 'PARTICLE',
203
+ '助動詞' => 'AUX',
204
+
205
+ # Noun subcategories
206
+ '名詞,一般' => 'NOUN_COMMON',
207
+ '名詞,固有名詞' => 'NOUN_PROPER',
208
+ '名詞,固有名詞,地域' => 'NOUN_PROPER_GEOGRAPHIC',
209
+ '名詞,接尾' => 'NOUN_SUFFIX',
210
+ '名詞,非自立' => 'NOUN_DEPENDENT',
211
+ '名詞,サ変接続' => 'NOUN_SA_CONNECTION',
212
+
213
+ # Particle subcategories
214
+ '助詞,格助詞' => 'PARTICLE_GRAMMAR',
215
+ '助詞,係助詞' => 'PARTICLE_BINDING',
216
+ '助詞,連体化' => 'PARTICLE_ADNOMINAL',
217
+
218
+ # Verb subcategories
219
+ '動詞,自立' => 'VERB_INDEPENDENT',
220
+ }.freeze
221
+
222
+ def initialize(dictionary_path: nil, flag_mapping: FLAG_TO_POS)
223
+ @dictionary_path = dictionary_path
224
+ @flag_mapping = flag_mapping
225
+ @suika_tagger = nil
226
+ @lookup_cache = {}
227
+ end
228
+
229
+ def tag(tokens)
230
+ return [] if tokens.nil? || tokens.empty?
231
+
232
+ # Suika::Tagger is process-wide memoized in Language::Suika;
233
+ # raises SuikaUnavailable when the gem is missing.
234
+ @suika_tagger ||= Kotoshu::Language::Suika.tagger
235
+
236
+ tokens.map do |token|
237
+ word = token[:token]
238
+ if word.nil? || word.empty?
239
+ token.merge(pos_tag: nil, lemma: nil)
240
+ else
241
+ lookup_result = lookup_with_pos(word)
242
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
243
+ end
244
+ end
245
+ end
246
+
247
+ def flag_mapping
248
+ @flag_mapping
249
+ end
250
+
251
+ def flag_mapping=(mapping)
252
+ @flag_mapping = mapping
253
+ end
254
+
255
+ def clear_cache
256
+ @lookup_cache.clear
257
+ end
258
+
259
+ private
260
+
261
+ def lookup_with_pos(word)
262
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
263
+ return @lookup_cache[word] if @lookup_cache.key?(word)
264
+
265
+ # Use Suika to parse and get POS information
266
+ parsed = @suika_tagger.parse(word)
267
+
268
+ # Suika returns tab-separated values: surface\tfeatures
269
+ # Features contain POS information
270
+ pos_tag = extract_pos_from_suika(parsed)
271
+ lemma = extract_lemma_from_suika(parsed)
272
+
273
+ cache_result = { pos_tag: pos_tag, lemma: lemma }
274
+ @lookup_cache[word] = cache_result
275
+ cache_result
276
+ end
277
+
278
+ def extract_pos_from_suika(parsed)
279
+ return nil unless parsed && parsed.first
280
+
281
+ # Parse features from Suika output
282
+ # Format: surface<TAB>POS,sub1,sub2,sub3,conj_type,conj_form,lemma,reading,pronunciation
283
+ parts = parsed.first.split("\t")
284
+ return nil unless parts.length > 1
285
+
286
+ # Features are comma-separated
287
+ # Field 0: Surface form
288
+ # Field 1: Main POS category (e.g., 名詞, 動詞, 助詞)
289
+ # Field 2-6: POS subcategories and conjugation info
290
+ # Field 7: Lemma (dictionary form)
291
+ # Field 8: Reading (katakana)
292
+ # Field 9: Pronunciation (katakana)
293
+ features = parts[1].split(',')
294
+
295
+ # Build hierarchical POS paths from most specific to least specific
296
+ # e.g., ["名詞,固有名詞,地域", "名詞,固有名詞", "名詞"]
297
+ pos_paths = []
298
+ 6.times do |i|
299
+ path = features[0..i].join(',')
300
+ pos_paths << path
301
+ end
302
+ # Reverse to check most specific first
303
+ pos_paths.reverse!
304
+
305
+ # Try to match from most specific to least specific
306
+ pos_paths.each do |pos_path|
307
+ if FLAG_TO_POS.key?(pos_path)
308
+ return FLAG_TO_POS[pos_path]
309
+ end
310
+ end
311
+
312
+ nil
313
+ end
314
+
315
+ def extract_lemma_from_suika(parsed)
316
+ return nil unless parsed && parsed.first
317
+
318
+ parts = parsed.first.split("\t")
319
+ return nil unless parts.length > 1
320
+
321
+ # Extract lemma from Suika features
322
+ # Format is complex, so simplified version
323
+ parts[0] # Return surface form as lemma
324
+ end
325
+ end
326
+
327
+ # Japanese grammar rules module.
328
+ module GrammarRules
329
+ class Rule
330
+ attr_reader :id, :name, :description
331
+
332
+ def initialize(id, name, description)
333
+ @id = id
334
+ @name = name
335
+ @description = description
336
+ end
337
+
338
+ def check(tokens)
339
+ raise NotImplementedError, "#{self.class} must implement #check"
340
+ end
341
+ end
342
+
343
+ # Rule: Particle usage (wa vs ga)
344
+ class ParticleRule < Rule
345
+ def initialize
346
+ super('JA_PARTICLE_USAGE', 'Particle Usage', 'Correct usage of topic marker は vs subject marker が.')
347
+ end
348
+
349
+ def check(_tokens)
350
+ # Simplified implementation
351
+ []
352
+ end
353
+ end
354
+
355
+ # Rule: Script mixing
356
+ class ScriptMixingRule < Rule
357
+ def initialize
358
+ super('JA_SCRIPT_MIXING', 'Script Mixing', 'Japanese text uses multiple scripts (Hiragana, Katakana, Kanji).')
359
+ end
360
+
361
+ def check(tokens)
362
+ errors = []
363
+ tokens.each do |token|
364
+ word = token[:token]
365
+ next if word.nil? || word.empty?
366
+
367
+ # Check for script mixing inconsistencies
368
+ has_hiragana = word.match?(/[\u3040-\u309F]/)
369
+ has_katakana = word.match?(/[\u30A0-\u30FF]/)
370
+ has_kanji = word.match?(/[\u4E00-\u9FFF]/)
371
+
372
+ # Words typically shouldn't mix all three scripts
373
+ if has_hiragana && has_katakana && has_kanji
374
+ errors << {
375
+ rule_id: @id,
376
+ position: token[:position],
377
+ message: "Unusual script mixing in word '#{word}'",
378
+ suggestion: 'Review script usage',
379
+ context: word,
380
+ suggestions: ['Use consistent script']
381
+ }
382
+ end
383
+ end
384
+ errors
385
+ end
386
+ end
387
+
388
+ class RuleRegistry
389
+ class << self
390
+ def default_rules
391
+ [ParticleRule.new, ScriptMixingRule.new]
392
+ end
393
+
394
+ def get_rule(id)
395
+ default_rules.find { |rule| rule.id == id }
396
+ end
397
+ end
398
+ end
399
+ end
400
+
401
+ # Registration
402
+ register "ja"
403
+ register "ja-JP"
404
+
405
+ HUNSPELL_DICTIONARIES = {
406
+ 'ja-JP' => {
407
+ # Japanese dictionaries are in custom formats
408
+ # Suika uses its own dictionary format
409
+ }
410
+ }.freeze
411
+
412
+ VARIANT_NAMES = {
413
+ 'JP' => 'Japan'
414
+ }.freeze
415
+
416
+ def initialize(code: "ja", name: "Japanese", variant: nil)
417
+ variant ||= extract_region_code(code)
418
+ super
419
+ end
420
+
421
+ def description
422
+ return name unless variant
423
+
424
+ variant_name = VARIANT_NAMES[variant] || variant
425
+ "#{name} (#{variant_name})"
426
+ end
427
+
428
+ def tokenizer
429
+ @tokenizer ||= Tokenizer.new
430
+ end
431
+
432
+ def normalizer
433
+ @normalizer ||= Language::Normalizer::Base.new
434
+ end
435
+
436
+ def dictionary_class
437
+ Dictionary::UnixWords
438
+ end
439
+
440
+ def default_dictionary_paths
441
+ ["/usr/share/dict/words"]
442
+ end
443
+
444
+ def script_type
445
+ :cjk
446
+ end
447
+
448
+ def create_spell_checker
449
+ # Japanese uses custom dictionary, not Hunspell format
450
+ SpellChecker.new(
451
+ dic_path: default_dictionary_paths.first,
452
+ script: :cjk
453
+ )
454
+ end
455
+
456
+ def create_tokenizer
457
+ Tokenizer.new
458
+ end
459
+
460
+ def create_pos_tagger
461
+ POSTagger.new(
462
+ dictionary_path: default_dictionary_paths.first,
463
+ flag_mapping: POSTagger::FLAG_TO_POS
464
+ )
465
+ end
466
+
467
+ private
468
+
469
+ def extract_region_code(code)
470
+ return nil unless code.include?("-")
471
+
472
+ code.split("-", 2).last.upcase
473
+ end
474
+ end
475
+ end
476
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Korean language implementation (plan 108).
6
+ #
7
+ # Registers +ko+ and +ko-KR+. The staged ko dictionary (LibreOffice
8
+ # ko_KR, jamo-decomposed behind an ICONV table of all 11,172
9
+ # syllables) serves checking through the standard Hunspell engine;
10
+ # this module contributes the language wiring the basic tier
11
+ # lacks: the eojeol-aware Hangul tokenizer for word extraction and
12
+ # the Dubeolsik (KS X 5002) keyboard grid for jamo-proximity
13
+ # suggestion ranking.
14
+ #
15
+ # @example
16
+ # lang = Kotoshu::Languages::Korean.new
17
+ # lang.script_type # => :hangul
18
+ # lang.rtl? # => false
19
+ class Korean < Language::Base
20
+ register "ko"
21
+ register "ko-KR"
22
+
23
+ def initialize(code: "ko", name: "Korean", variant: nil)
24
+ super
25
+ end
26
+
27
+ def description
28
+ name
29
+ end
30
+
31
+ def tokenizer
32
+ @tokenizer ||= Language::Tokenizer::HangulTokenizer.new
33
+ end
34
+
35
+ def normalizer
36
+ @normalizer ||= Language::Normalizer::Base.new
37
+ end
38
+
39
+ def dictionary_class
40
+ Dictionary::Custom
41
+ end
42
+
43
+ def default_dictionary_paths
44
+ ["/usr/share/dict/korean"]
45
+ end
46
+
47
+ def script_type
48
+ :hangul
49
+ end
50
+
51
+ def rtl?
52
+ false
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Shared base for thin Latin-script language modules.
6
+ #
7
+ # The wave-1 modules (plan 84, Track B) are compositions, not
8
+ # engines: the shared Latin tokenizer, the base normalizer, and a
9
+ # registry entry. Language-specific care overrides one piece:
10
+ # Turkish swaps in the Turkish normalizer (dotless-i folding),
11
+ # Greek swaps in the Greek tokenizer and normalizer, Ukrainian the
12
+ # Cyrillic tokenizer.
13
+ #
14
+ # The existing ten modules (ar de en es fa fr he ja pt ru) are
15
+ # untouched and keep their own class bodies.
16
+ class LatinBase < Language::Base
17
+ # Latin tokenizer shared by every Latin-script module.
18
+ #
19
+ # @return [Language::Tokenizer::LatinTokenizer]
20
+ def tokenizer
21
+ @tokenizer ||= Language::Tokenizer::LatinTokenizer.new
22
+ end
23
+
24
+ # Base normalizer (whitespace, downcase).
25
+ #
26
+ # @return [Language::Normalizer::Base]
27
+ def normalizer
28
+ @normalizer ||= Language::Normalizer::Base.new
29
+ end
30
+
31
+ # Dictionary backend for this language.
32
+ #
33
+ # @return [Class]
34
+ def dictionary_class
35
+ Dictionary::UnixWords
36
+ end
37
+
38
+ # Script type.
39
+ #
40
+ # @return [Symbol] :latin
41
+ def script_type
42
+ :latin
43
+ end
44
+
45
+ # Create the tokenizer (memoized; tokenizers are stateless).
46
+ #
47
+ # @return [Language::Tokenizer::Base]
48
+ def create_tokenizer
49
+ tokenizer
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Lithuanian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. The Lithuanian letters ą č ę ė į š ų ū ž are
9
+ # dead-key / AltGr sequences over the US QWERTY grid
10
+ # (Keyboard::Layouts::Latin::Lithuanian), like the eval model.
11
+ class Lithuanian < LatinBase
12
+ register "lt"
13
+ register "lt-LT"
14
+
15
+ def initialize(code: "lt", name: "Lithuanian")
16
+ super
17
+ end
18
+
19
+ def default_dictionary_paths
20
+ ["/usr/share/dict/lithuanian"]
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Latvian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. The Latvian letters ā č ē ģ ī ķ ļ ņ š ū ž are
9
+ # dead-key / AltGr sequences over the US QWERTY grid
10
+ # (Keyboard::Layouts::Latin::Latvian), like the eval model.
11
+ class Latvian < LatinBase
12
+ register "lv"
13
+ register "lv-LV"
14
+
15
+ def initialize(code: "lv", name: "Latvian")
16
+ super
17
+ end
18
+
19
+ def default_dictionary_paths
20
+ ["/usr/share/dict/latvian"]
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Norwegian Bokmål (registry v1.3.0: models ship as `nb` from
6
+ # fastText's Bokmål-dominated cc.no vectors; the Language registry
7
+ # aliases `no` here). Thin composition like its Nordic siblings —
8
+ # shared Latin tokenizer, base normalizer, registry entry; å æ ø
9
+ # are real keys on the Nordic layout.
10
+ class NorwegianBokmal < LatinBase
11
+ register "nb"
12
+ register "nb-NO"
13
+ register "no"
14
+ register "no-NO"
15
+
16
+ def initialize(code: "nb", name: "Norwegian Bokmål")
17
+ super
18
+ end
19
+
20
+ def default_dictionary_paths
21
+ ["/usr/share/dict/norwegian"]
22
+ end
23
+ end
24
+ end
25
+ end