kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,476 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Spanish language implementation.
6
+ #
7
+ # Supports multiple dialects: es-ES, es-MX, es-AR, es-CO, es-PE, es-VE, es-CL, es-EC
8
+ #
9
+ # Full Hunspell integration with spell checking, POS tagging, and grammar rules
10
+ # specifically handling Spanish inverted punctuation and diacritics.
11
+ class Spanish < Language::Base
12
+ # Spanish spell checker with Hunspell integration.
13
+ class SpellChecker < Components::SpellChecker
14
+ attr_reader :aff_path, :dic_path, :script
15
+
16
+ # Spanish-specific character substitutions
17
+ SPANISH_SUBSTITUTIONS = {
18
+ 'á' => %w[a],
19
+ 'é' => %w[e],
20
+ 'í' => %w[i],
21
+ 'ó' => %w[o],
22
+ 'ú' => %w[u],
23
+ 'ü' => %w[u],
24
+ 'ñ' => %w[n],
25
+ '¿' => [],
26
+ '¡' => [],
27
+ }.freeze
28
+
29
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
30
+ @aff_path = aff_path
31
+ @dic_path = dic_path
32
+ @script = script
33
+ @encoding = encoding
34
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
35
+ end
36
+
37
+ def check(word)
38
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
39
+
40
+ first_form = @lookuper.good_forms(word).first
41
+ if first_form
42
+ { found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
43
+ else
44
+ { found: false, stem: nil, flags: [] }
45
+ end
46
+ end
47
+
48
+ def suggest(word, max_suggestions: 10)
49
+ return [] if word.nil? || word.empty?
50
+
51
+ first_form = @lookuper.good_forms(word).first
52
+ return [] if first_form
53
+
54
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
55
+ end
56
+
57
+ def correct?(word)
58
+ check(word)[:found]
59
+ end
60
+
61
+ def lookuper
62
+ @lookuper
63
+ end
64
+
65
+ private
66
+
67
+ def calculate_distance(a, b)
68
+ return a.length if b.empty?
69
+ return b.length if a.empty?
70
+
71
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
72
+ (1..b.length).each { |j| matrix[0][j] = j }
73
+ (1..a.length).each do |i|
74
+ (1..b.length).each do |j|
75
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
76
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
77
+ end
78
+ end
79
+ matrix[a.length][b.length]
80
+ end
81
+
82
+ def calculate_score(original, suggestion, rank)
83
+ distance = calculate_distance(original, suggestion)
84
+ max_len = [original.length, suggestion.length].max
85
+ distance_score = 1.0 - (distance.to_f / max_len)
86
+ rank_penalty = rank * 0.05
87
+ [distance_score - rank_penalty, 0.0].max
88
+ end
89
+
90
+ def generate_suggestions(word, _max_suggestions)
91
+ variations = []
92
+
93
+ # Missing accents and ñ
94
+ word.downcase.chars.each_with_index do |char, i|
95
+ SPANISH_SUBSTITUTIONS.each do |accented, variants|
96
+ variants.each do |variant|
97
+ if char == variant
98
+ accented_word = word.dup
99
+ accented_word[i] = accented
100
+ variations << accented_word if @lookuper.good_forms(accented_word).first
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+ # Common substitutions
107
+ word.chars.each_with_index do |char, i|
108
+ next unless SPANISH_SUBSTITUTIONS.key?(char.downcase)
109
+
110
+ SPANISH_SUBSTITUTIONS[char.downcase].each do |sub|
111
+ next if sub.empty?
112
+
113
+ substituted = word.dup
114
+ substituted[i] = sub
115
+ variations << substituted if @lookuper.good_forms(substituted).first
116
+ end
117
+ end
118
+
119
+ # Doubled and deleted letters
120
+ word.chars.each_with_index do |char, i|
121
+ next if i == 0
122
+
123
+ doubled = word.dup
124
+ doubled.insert(i, char)
125
+ variations << doubled if @lookuper.good_forms(doubled).first
126
+ end
127
+
128
+ (0...word.length).each do |i|
129
+ deleted = word.dup
130
+ deleted.slice!(i)
131
+ next if deleted.empty?
132
+
133
+ variations << deleted if @lookuper.good_forms(deleted).first
134
+ end
135
+
136
+ variations.uniq!
137
+ variations.map do |suggestion|
138
+ { word: suggestion, distance: calculate_distance(word, suggestion),
139
+ score: calculate_score(word, suggestion, 0) }
140
+ end.sort_by { |s| s[:distance] }
141
+ end
142
+ end
143
+
144
+ # Spanish tokenizer with ordinal and decimal handling.
145
+ class Tokenizer < Language::Tokenizer::SpanishTokenizer
146
+ end
147
+
148
+ # Spanish POS tagger.
149
+ class POSTagger < Components::PosTagger
150
+ FLAG_TO_POS = {
151
+ 'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
152
+ 'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
153
+ 'VBP' => 'VERB', 'VBZ' => 'VERB',
154
+ 'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
155
+ 'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
156
+ 'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
157
+ 'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
158
+ 'WP' => 'PRON', 'WP$' => 'PRON_POSS',
159
+ 'I' => 'PREP', 'IN' => 'PREP',
160
+ 'C' => 'CONJ', 'CC' => 'CONJ',
161
+ 'U' => 'PART', 'RP' => 'PART',
162
+ 'INTJ' => 'INTJ', 'UH' => 'INTJ',
163
+ 'CD' => 'NUM',
164
+ 'FW' => 'X',
165
+ 'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
166
+ '¿' => 'PUNCT', '¡' => 'PUNCT', '?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
167
+ }.freeze
168
+
169
+ attr_reader :aff_path, :dic_path, :script
170
+
171
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
172
+ @aff_path = aff_path
173
+ @dic_path = dic_path
174
+ @script = script
175
+ @encoding = encoding
176
+ @flag_mapping = flag_mapping
177
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
178
+ @lookup_cache = {}
179
+ end
180
+
181
+ def tag(tokens)
182
+ return [] if tokens.nil? || tokens.empty?
183
+
184
+ tokens.map do |token|
185
+ word = token[:token]
186
+ if word.nil? || word.empty?
187
+ token.merge(pos_tag: nil, lemma: nil)
188
+ else
189
+ lookup_result = lookup_with_pos(word)
190
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
191
+ end
192
+ end
193
+ end
194
+
195
+ def flag_mapping
196
+ @flag_mapping
197
+ end
198
+
199
+ def flag_mapping=(mapping)
200
+ @flag_mapping = mapping
201
+ end
202
+
203
+ def clear_cache
204
+ @lookup_cache.clear
205
+ end
206
+
207
+ def cache_size
208
+ @lookup_cache.size
209
+ end
210
+
211
+ private
212
+
213
+ def lookup_with_pos(word)
214
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
215
+ return @lookup_cache[word] if @lookup_cache.key?(word)
216
+
217
+ first_form = @lookuper.good_forms(word).first
218
+ pos_tag = derive_pos_tag(first_form)
219
+ cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
220
+ @lookup_cache[word] = cache_result
221
+ cache_result
222
+ end
223
+
224
+ def derive_pos_tag(result)
225
+ return nil unless result
226
+
227
+ flags = result.flags&.to_a || []
228
+ return guess_pos_from_affix(result) if flags.empty?
229
+
230
+ flags.each do |flag|
231
+ pos_tag = flag_to_pos(flag)
232
+ return pos_tag if pos_tag
233
+ end
234
+ guess_pos_from_affix(result)
235
+ end
236
+
237
+ def flag_to_pos(flag)
238
+ return @flag_mapping[flag] if @flag_mapping.key?(flag)
239
+
240
+ first_char = flag[0]
241
+ @flag_mapping[first_char]
242
+ end
243
+
244
+ def guess_pos_from_affix(result)
245
+ suffix = result.suffix
246
+ return guess_pos_from_suffix(suffix) if suffix
247
+
248
+ nil
249
+ end
250
+
251
+ def guess_pos_from_suffix(suffix)
252
+ # Spanish suffix patterns
253
+ return 'VERB' if suffix.match?(/^(ar|er|ir|ando|iendo|ado|ido|ó)$/)
254
+ return 'ADV' if suffix.match?(/^(mente)$/)
255
+ return 'NOUN' if suffix.match?(/^(ción|sión|miento|dad|eza|ismo|ista|or|nte|aje)$/)
256
+ return 'ADJ' if suffix.match?(/%(oso|oso|able|ible|ble|ico|ica|ante)$/)
257
+
258
+ nil
259
+ end
260
+ end
261
+
262
+ # Spanish grammar rules module.
263
+ module GrammarRules
264
+ class Rule
265
+ attr_reader :id, :name, :description
266
+
267
+ def initialize(id, name, description)
268
+ @id = id
269
+ @name = name
270
+ @description = description
271
+ end
272
+
273
+ def check(tokens)
274
+ raise NotImplementedError, "#{self.class} must implement #check"
275
+ end
276
+ end
277
+
278
+ # Rule: Inverted punctuation (¡, ¿)
279
+ class InvertedPunctuationRule < Rule
280
+ def initialize
281
+ super('ES_INVERTED_PUNCTUATION', 'Inverted Punctuation', 'Spanish requires inverted punctuation marks (¡, ¿) at the start of exclamations/questions.')
282
+ end
283
+
284
+ def check(tokens)
285
+ errors = []
286
+ tokens.each_with_index do |token, idx|
287
+ word = token[:token]
288
+ next if word.nil? || word.empty?
289
+
290
+ # Check for standard ? or ! without corresponding inverted marks
291
+ if ['?', '!'].include?(word)
292
+ # Look backwards to see if there's an inverted mark
293
+ found_inverted = false
294
+ (0...idx).reverse_each do |j|
295
+ check_token = tokens[j][:token]
296
+ if (word == '?' && check_token == '¿') || (word == '!' && check_token == '¡')
297
+ found_inverted = true
298
+ break
299
+ end
300
+ # Stop checking if we hit another sentence-ending punctuation
301
+ break if %w[. ? !].include?(check_token)
302
+ end
303
+
304
+ unless found_inverted
305
+ errors << {
306
+ rule_id: @id,
307
+ position: token[:position],
308
+ message: "Missing inverted punctuation mark: use '#{word == '?' ? '¿' : '¡'}' at the start",
309
+ suggestion: word == '?' ? '¿...?' : '¡...!',
310
+ context: word,
311
+ suggestions: [word == '?' ? '¿...?' : '¡...!']
312
+ }
313
+ end
314
+ end
315
+ end
316
+ errors
317
+ end
318
+ end
319
+
320
+ # Rule: Gender agreement
321
+ class GenderAgreementRule < Rule
322
+ def initialize
323
+ super('ES_GENDER_AGREEMENT', 'Gender Agreement', 'Nouns and adjectives must agree in gender.')
324
+ end
325
+
326
+ def check(_tokens)
327
+ # Simplified implementation
328
+ []
329
+ end
330
+ end
331
+
332
+ class RuleRegistry
333
+ class << self
334
+ def default_rules
335
+ [InvertedPunctuationRule.new, GenderAgreementRule.new]
336
+ end
337
+
338
+ def get_rule(id)
339
+ default_rules.find { |rule| rule.id == id }
340
+ end
341
+ end
342
+ end
343
+ end
344
+
345
+ # Registration
346
+ register "es"
347
+ register "es-ES"
348
+ register "es-MX"
349
+ register "es-AR"
350
+ register "es-CO"
351
+ register "es-PE"
352
+ register "es-VE"
353
+ register "es-CL"
354
+ register "es-EC"
355
+ register "es-GT"
356
+ register "es-CU"
357
+ register "es-BO"
358
+ register "es-DO"
359
+ register "es-HN"
360
+ register "es-PY"
361
+ register "es-SV"
362
+ register "es-NI"
363
+ register "es-CR"
364
+ register "es-PA"
365
+ register "es-UY"
366
+ register "es-PR"
367
+
368
+ HUNSPELL_DICTIONARIES = {
369
+ 'es-ES' => {
370
+ aff: 'spec/integrational/fixtures/es_ES.aff',
371
+ dic: 'spec/integrational/fixtures/es_ES.dic'
372
+ },
373
+ 'es-MX' => {
374
+ aff: 'spec/integrational/fixtures/es_MX.aff',
375
+ dic: 'spec/integrational/fixtures/es_MX.dic'
376
+ }
377
+ }.freeze
378
+
379
+ VARIANT_NAMES = {
380
+ 'ES' => 'European',
381
+ 'MX' => 'Mexican',
382
+ 'AR' => 'Argentinian',
383
+ 'CO' => 'Colombian',
384
+ 'PE' => 'Peruvian',
385
+ 'VE' => 'Venezuelan',
386
+ 'CL' => 'Chilean',
387
+ 'EC' => 'Ecuadorian',
388
+ 'GT' => 'Guatemalan',
389
+ 'CU' => 'Cuban',
390
+ 'BO' => 'Bolivian',
391
+ 'DO' => 'Dominican',
392
+ 'HN' => 'Honduran',
393
+ 'PY' => 'Paraguayan',
394
+ 'SV' => 'Salvadoran',
395
+ 'NI' => 'Nicaraguan',
396
+ 'CR' => 'Costa Rican',
397
+ 'PA' => 'Panamanian',
398
+ 'UY' => 'Uruguayan',
399
+ 'PR' => 'Puerto Rican'
400
+ }.freeze
401
+
402
+ def initialize(code: "es", name: "Spanish", variant: nil)
403
+ variant ||= extract_region_code(code)
404
+ super
405
+ @hunspell_paths = resolve_hunspell_paths(code)
406
+ end
407
+
408
+ def description
409
+ return name unless variant
410
+
411
+ variant_name = VARIANT_NAMES[variant] || variant
412
+ "#{name} (#{variant_name})"
413
+ end
414
+
415
+ def tokenizer
416
+ @tokenizer ||= Tokenizer.new
417
+ end
418
+
419
+ def normalizer
420
+ @normalizer ||= Language::Normalizer::Base.new
421
+ end
422
+
423
+ def dictionary_class
424
+ Dictionary::UnixWords
425
+ end
426
+
427
+ def default_dictionary_paths
428
+ case code
429
+ when "es-ES"
430
+ ["/usr/share/dict/spanish"]
431
+ when "es-MX"
432
+ ["/usr/share/dict/mexican"]
433
+ else
434
+ ["/usr/share/dict/words"]
435
+ end
436
+ end
437
+
438
+ def script_type
439
+ :latin
440
+ end
441
+
442
+ def create_spell_checker
443
+ SpellChecker.new(
444
+ aff_path: @hunspell_paths[:aff],
445
+ dic_path: @hunspell_paths[:dic],
446
+ script: :latin
447
+ )
448
+ end
449
+
450
+ def create_tokenizer
451
+ Tokenizer.new
452
+ end
453
+
454
+ def create_pos_tagger
455
+ POSTagger.new(
456
+ aff_path: @hunspell_paths[:aff],
457
+ dic_path: @hunspell_paths[:dic],
458
+ script: :latin,
459
+ flag_mapping: POSTagger::FLAG_TO_POS
460
+ )
461
+ end
462
+
463
+ private
464
+
465
+ def extract_region_code(code)
466
+ return nil unless code.include?("-")
467
+
468
+ code.split("-", 2).last.upcase
469
+ end
470
+
471
+ def resolve_hunspell_paths(code)
472
+ HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['es-ES']
473
+ end
474
+ end
475
+ end
476
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Estonian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. The Estonian letters ä ö õ ü š ž are dead-key /
9
+ # AltGr sequences over the US QWERTY grid
10
+ # (Keyboard::Layouts::Latin::Estonian), like the eval model.
11
+ class Estonian < LatinBase
12
+ register "et"
13
+ register "et-EE"
14
+
15
+ def initialize(code: "et", name: "Estonian")
16
+ super
17
+ end
18
+
19
+ def default_dictionary_paths
20
+ ["/usr/share/dict/estonian"]
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Persian (Farsi) language implementation.
6
+ #
7
+ # Registers the Persian language with {Language::Registry} under
8
+ # the +fa+ and +fa-IR+ codes. Uses the
9
+ # {Language::Normalizer::Persian} normalizer (Arabic normalization
10
+ # plus Arabic→Persian letter canonicalization).
11
+ #
12
+ # @example
13
+ # lang = Kotoshu::Languages::Persian.new
14
+ # lang.script_type # => :arabic
15
+ # lang.rtl? # => true
16
+ #
17
+ # Full-feature since plan 100 (batch 3): the spelling dictionary
18
+ # downloads from the dictionaries repo (AVAILABLE_LANGUAGES) and
19
+ # keyboard proximity uses the Persian standard grid
20
+ # (Keyboard::Layouts::PersianStandard).
21
+ class Persian < Language::Base
22
+ register "fa"
23
+ register "fa-IR"
24
+
25
+ def initialize(code: "fa", name: "Persian", variant: nil)
26
+ super
27
+ end
28
+
29
+ def description
30
+ name
31
+ end
32
+
33
+ def tokenizer
34
+ @tokenizer ||= Tokenizer.new
35
+ end
36
+
37
+ def normalizer
38
+ @normalizer ||= Language::Normalizer::Persian.new
39
+ end
40
+
41
+ def dictionary_class
42
+ Dictionary::Custom
43
+ end
44
+
45
+ def default_dictionary_paths
46
+ ["/usr/share/dict/persian"]
47
+ end
48
+
49
+ def script_type
50
+ :arabic
51
+ end
52
+
53
+ def rtl?
54
+ true
55
+ end
56
+
57
+ class Tokenizer < Language::Tokenizer::Base
58
+ PERSIAN_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·،؛؟]/
59
+
60
+ def tokenize(text)
61
+ return [] if text.nil? || text.strip.empty?
62
+
63
+ text.split(PERSIAN_SEPARATORS)
64
+ .map { |token| normalize(token) }
65
+ .reject { |token| skip_token?(token) }
66
+ end
67
+
68
+ # Spell-check word characters (plan 91 pattern, added by
69
+ # plan 100): Arabic-script letters plus ZWNJ (U+200C) —
70
+ # Persian compounds like می‌روم keep their nimb-fāsele inside
71
+ # the word, and the staged dictionary carries tens of
72
+ # thousands of ZWNJ forms.
73
+ #
74
+ # @return [Regexp] Regex matching a single word character
75
+ def spellcheck_word_regex
76
+ /[\p{Arabic}\u200C]/
77
+ end
78
+
79
+ protected
80
+
81
+ def normalize(token)
82
+ token.strip
83
+ end
84
+
85
+ def word_separators
86
+ PERSIAN_SEPARATORS
87
+ end
88
+ end
89
+ end
90
+ end
91
+ end