kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,505 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # French language implementation.
6
+ #
7
+ # Supports multiple dialects: fr-FR, fr-CA, fr-BE, fr-CH, fr-LU, fr-MC
8
+ #
9
+ # Full Hunspell integration with spell checking, POS tagging, and grammar rules.
10
+ class French < Language::Base
11
+ # French spell checker with Hunspell integration.
12
+ #
13
+ # Uses the Lookup algorithm with Hunspell-format dictionaries
14
+ # and French-specific character handling (accents, ligatures).
15
+ class SpellChecker < Components::SpellChecker
16
+ attr_reader :aff_path, :dic_path, :script
17
+
18
+ # French-specific character substitutions for suggestions
19
+ FRENCH_SUBSTITUTIONS = {
20
+ 'à' => %w[a],
21
+ 'â' => %w[a],
22
+ 'ä' => %w[a],
23
+ 'é' => %w[e],
24
+ 'è' => %w[e],
25
+ 'ê' => %w[e],
26
+ 'ë' => %w[e],
27
+ 'î' => %w[i],
28
+ 'ï' => %w[i],
29
+ 'ô' => %w[o],
30
+ 'ö' => %w[o],
31
+ 'ù' => %w[u],
32
+ 'û' => %w[u],
33
+ 'ü' => %w[u],
34
+ 'ç' => %w[c],
35
+ 'œ' => %w[oe],
36
+ 'æ' => %w[ae],
37
+ # Common French errors
38
+ 'c' => %w[ç], # garçon vs garcon
39
+ 'e' => %w[é è ê], # café vs caffe
40
+ 'a' => %w[à], # à vs a
41
+ }.freeze
42
+
43
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
44
+ @aff_path = aff_path
45
+ @dic_path = dic_path
46
+ @script = script
47
+ @encoding = encoding
48
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
49
+ end
50
+
51
+ def check(word)
52
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
53
+
54
+ first_form = @lookuper.good_forms(word).first
55
+ if first_form
56
+ { found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
57
+ else
58
+ { found: false, stem: nil, flags: [] }
59
+ end
60
+ end
61
+
62
+ def suggest(word, max_suggestions: 10)
63
+ return [] if word.nil? || word.empty?
64
+
65
+ first_form = @lookuper.good_forms(word).first
66
+ return [] if first_form
67
+
68
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
69
+ end
70
+
71
+ def correct?(word)
72
+ check(word)[:found]
73
+ end
74
+
75
+ def lookuper
76
+ @lookuper
77
+ end
78
+
79
+ private
80
+
81
+ def calculate_distance(a, b)
82
+ return a.length if b.empty?
83
+ return b.length if a.empty?
84
+
85
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
86
+ (1..b.length).each { |j| matrix[0][j] = j }
87
+ (1..a.length).each do |i|
88
+ (1..b.length).each do |j|
89
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
90
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
91
+ end
92
+ end
93
+ matrix[a.length][b.length]
94
+ end
95
+
96
+ def calculate_score(original, suggestion, rank)
97
+ distance = calculate_distance(original, suggestion)
98
+ max_len = [original.length, suggestion.length].max
99
+ distance_score = 1.0 - (distance.to_f / max_len)
100
+ rank_penalty = rank * 0.05
101
+ [distance_score - rank_penalty, 0.0].max
102
+ end
103
+
104
+ def generate_suggestions(word, _max_suggestions)
105
+ variations = []
106
+
107
+ # Missing accents
108
+ word.downcase.chars.each_with_index do |char, i|
109
+ FRENCH_SUBSTITUTIONS.each do |accented, unaccented_variants|
110
+ unaccented_variants.each do |variant|
111
+ if char == variant
112
+ unaccented_word = word.dup
113
+ unaccented_word[i] = accented
114
+ variations << unaccented_word if @lookuper.good_forms(unaccented_word).first
115
+ end
116
+ end
117
+ end
118
+ end
119
+
120
+ # Doubled letters
121
+ word.chars.each_with_index do |char, i|
122
+ next if i == 0
123
+
124
+ doubled = word.dup
125
+ doubled.insert(i, char)
126
+ variations << doubled if @lookuper.good_forms(doubled).first
127
+ end
128
+
129
+ # Deleted letters
130
+ (0...word.length).each do |i|
131
+ deleted = word.dup
132
+ deleted.slice!(i)
133
+ next if deleted.empty?
134
+
135
+ variations << deleted if @lookuper.good_forms(deleted).first
136
+ end
137
+
138
+ # Common substitutions
139
+ word.chars.each_with_index do |char, i|
140
+ next unless FRENCH_SUBSTITUTIONS.key?(char.downcase)
141
+
142
+ FRENCH_SUBSTITUTIONS[char.downcase].each do |sub|
143
+ substituted = word.dup
144
+ substituted[i] = sub
145
+ variations << substituted if @lookuper.good_forms(substituted).first
146
+ end
147
+ end
148
+
149
+ variations.uniq!
150
+ variations.map do |suggestion|
151
+ { word: suggestion, distance: calculate_distance(word, suggestion),
152
+ score: calculate_score(word, suggestion, 0) }
153
+ end.sort_by { |s| s[:distance] }
154
+ end
155
+ end
156
+
157
+ # French tokenizer with contraction handling.
158
+ class Tokenizer < Language::Tokenizer::FrenchTokenizer
159
+ end
160
+
161
+ # French POS tagger.
162
+ #
163
+ # Derives POS tags from Hunspell flags using French-specific mappings.
164
+ class POSTagger < Components::PosTagger
165
+ # French POS flag mappings based on Hunspell French dictionaries
166
+ FLAG_TO_POS = {
167
+ # Nouns
168
+ 'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
169
+ # Verbs
170
+ 'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
171
+ 'VBP' => 'VERB', 'VBZ' => 'VERB',
172
+ # Adjectives
173
+ 'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
174
+ # Adverbs
175
+ 'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
176
+ # Determiners
177
+ 'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
178
+ # Pronouns
179
+ 'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
180
+ 'WP' => 'PRON', 'WP$' => 'PRON_POSS',
181
+ # Prepositions
182
+ 'I' => 'PREP', 'IN' => 'PREP',
183
+ # Conjunctions
184
+ 'C' => 'CONJ', 'CC' => 'CONJ',
185
+ # Particles
186
+ 'U' => 'PART', 'RP' => 'PART',
187
+ # Interjections
188
+ 'INTJ' => 'INTJ', 'UH' => 'INTJ',
189
+ # Numbers
190
+ 'CD' => 'NUM',
191
+ # Foreign words
192
+ 'FW' => 'X',
193
+ # Punctuation
194
+ 'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
195
+ '?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
196
+ }.freeze
197
+
198
+ attr_reader :aff_path, :dic_path, :script
199
+
200
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
201
+ @aff_path = aff_path
202
+ @dic_path = dic_path
203
+ @script = script
204
+ @encoding = encoding
205
+ @flag_mapping = flag_mapping
206
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
207
+ @lookup_cache = {}
208
+ end
209
+
210
+ def tag(tokens)
211
+ return [] if tokens.nil? || tokens.empty?
212
+
213
+ tokens.map do |token|
214
+ word = token[:token]
215
+ if word.nil? || word.empty?
216
+ token.merge(pos_tag: nil, lemma: nil)
217
+ else
218
+ lookup_result = lookup_with_pos(word)
219
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
220
+ end
221
+ end
222
+ end
223
+
224
+ def flag_mapping
225
+ @flag_mapping
226
+ end
227
+
228
+ def flag_mapping=(mapping)
229
+ @flag_mapping = mapping
230
+ end
231
+
232
+ def clear_cache
233
+ @lookup_cache.clear
234
+ end
235
+
236
+ private
237
+
238
+ def lookup_with_pos(word)
239
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
240
+ return @lookup_cache[word] if @lookup_cache.key?(word)
241
+
242
+ first_form = @lookuper.good_forms(word).first
243
+ pos_tag = derive_pos_tag(first_form)
244
+ cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
245
+ @lookup_cache[word] = cache_result
246
+ cache_result
247
+ end
248
+
249
+ def derive_pos_tag(result)
250
+ return nil unless result
251
+
252
+ flags = result.flags&.to_a || []
253
+ return guess_pos_from_affix(result) if flags.empty?
254
+
255
+ flags.each do |flag|
256
+ pos_tag = flag_to_pos(flag)
257
+ return pos_tag if pos_tag
258
+ end
259
+ guess_pos_from_affix(result)
260
+ end
261
+
262
+ def flag_to_pos(flag)
263
+ return @flag_mapping[flag] if @flag_mapping.key?(flag)
264
+
265
+ first_char = flag[0]
266
+ @flag_mapping[first_char]
267
+ end
268
+
269
+ def guess_pos_from_affix(result)
270
+ suffix = result.suffix
271
+ return guess_pos_from_suffix(suffix) if suffix
272
+
273
+ nil
274
+ end
275
+
276
+ def guess_pos_from_suffix(suffix)
277
+ # French suffix patterns
278
+ return 'VERB' if suffix.match?(/^(er|ir|re|is|it|issent|issons|issez)$/)
279
+ return 'ADV' if suffix.end_with?('ment')
280
+ return 'NOUN' if suffix.match?(/^(tion|sion|ment|age|ure|ée|ée)$/)
281
+ return 'ADJ' if suffix.match?(/^(if|ive|eux|euse|able|ible)$/)
282
+
283
+ nil
284
+ end
285
+ end
286
+
287
+ # French grammar rules module.
288
+ module GrammarRules
289
+ # Base class for French grammar rules.
290
+ class Rule
291
+ attr_reader :id, :name, :description
292
+
293
+ def initialize(id, name, description)
294
+ @id = id
295
+ @name = name
296
+ @description = description
297
+ end
298
+
299
+ def check(tokens)
300
+ raise NotImplementedError, "#{self.class} must implement #check"
301
+ end
302
+
303
+ def applies?(_tokens, _index)
304
+ true
305
+ end
306
+ end
307
+
308
+ # Rule: Article agreement with gender/number.
309
+ class ArticleAgreementRule < Rule
310
+ MASCULINE_SINGULAR = %w[le un].freeze
311
+ FEMININE_SINGULAR = %w[la une].freeze
312
+ PLURAL = %w[les des].freeze
313
+
314
+ def initialize
315
+ super('FR_ARTICLE_AGREEMENT', 'Article Agreement', 'Articles must agree with noun gender and number.')
316
+ end
317
+
318
+ def check(tokens)
319
+ errors = []
320
+ tokens.each_cons(2) do |article_token, noun_token|
321
+ article = article_token[:token]&.downcase
322
+ next unless MASCULINE_SINGULAR.include?(article) ||
323
+ FEMININE_SINGULAR.include?(article) ||
324
+ PLURAL.include?(article)
325
+
326
+ # This is a simplified check - full implementation would need dictionary lookup
327
+ # for gender/number information
328
+ next unless article_token[:pos_tag] == 'DET'
329
+
330
+ noun = noun_token[:token]
331
+ # Check for common patterns
332
+ if noun&.end_with?('e') && MASCULINE_SINGULAR.include?(article)
333
+ # Possibly incorrect: masculine article with feminine-looking noun
334
+ errors << {
335
+ rule_id: @id,
336
+ position: article_token[:position],
337
+ message: "Article agreement: check if '#{noun}' is feminine",
338
+ suggestion: nil,
339
+ context: "#{article} #{noun}",
340
+ suggestions: ['la', 'une']
341
+ }
342
+ end
343
+ end
344
+ errors
345
+ end
346
+ end
347
+
348
+ # Rule: Double negation in French (correct usage).
349
+ class FrenchNegationRule < Rule
350
+ NEGATION_PARTICLES = %w[ne n'].freeze
351
+ SECOND_PARTICLES = %w[pas plus jamais rien personne].freeze
352
+
353
+ def initialize
354
+ super('FR_NEGATION', 'French Negation', 'French uses double negation (ne...pas).')
355
+ end
356
+
357
+ def check(tokens)
358
+ errors = []
359
+ tokens.each_with_index do |token, idx|
360
+ word = token[:token]&.downcase
361
+ next unless NEGATION_PARTICLES.include?(word)
362
+
363
+ # Check if second negation particle exists within reasonable distance
364
+ found_second = false
365
+ ((idx + 1)...[idx + 5, tokens.length].min).each do |j|
366
+ next_word = tokens[j][:token]&.downcase
367
+ if SECOND_PARTICLES.include?(next_word)
368
+ found_second = true
369
+ break
370
+ end
371
+ end
372
+
373
+ unless found_second
374
+ errors << {
375
+ rule_id: @id,
376
+ position: token[:position],
377
+ message: "Incomplete negation: French requires double negation (ne...pas)",
378
+ suggestion: 'Add pas or another negation particle',
379
+ context: word,
380
+ suggestions: ['ne...pas', 'ne...pas']
381
+ }
382
+ end
383
+ end
384
+ errors
385
+ end
386
+ end
387
+
388
+ # Rule registry for French.
389
+ class RuleRegistry
390
+ class << self
391
+ def default_rules
392
+ [ArticleAgreementRule.new, FrenchNegationRule.new]
393
+ end
394
+
395
+ def get_rule(id)
396
+ default_rules.find { |rule| rule.id == id }
397
+ end
398
+ end
399
+ end
400
+ end
401
+
402
+ # Registration
403
+ register "fr"
404
+ register "fr-FR"
405
+ register "fr-CA"
406
+ register "fr-BE"
407
+ register "fr-CH"
408
+ register "fr-LU"
409
+ register "fr-MC"
410
+
411
+ HUNSPELL_DICTIONARIES = {
412
+ 'fr-FR' => {
413
+ aff: 'spec/integrational/fixtures/fr_FR.aff',
414
+ dic: 'spec/integrational/fixtures/fr_FR.dic'
415
+ },
416
+ 'fr-CA' => {
417
+ aff: 'spec/integrational/fixtures/fr_CA.aff',
418
+ dic: 'spec/integrational/fixtures/fr_CA.dic'
419
+ }
420
+ }.freeze
421
+
422
+ VARIANT_NAMES = {
423
+ 'FR' => 'France',
424
+ 'CA' => 'Canadian',
425
+ 'BE' => 'Belgian',
426
+ 'CH' => 'Swiss',
427
+ 'LU' => 'Luxembourgish',
428
+ 'MC' => 'Monégasque'
429
+ }.freeze
430
+
431
+ def initialize(code: "fr", name: "French", variant: nil)
432
+ variant ||= extract_region_code(code)
433
+ super
434
+ @hunspell_paths = resolve_hunspell_paths(code)
435
+ end
436
+
437
+ def description
438
+ return name unless variant
439
+
440
+ variant_name = VARIANT_NAMES[variant] || variant
441
+ "#{name} (#{variant_name})"
442
+ end
443
+
444
+ def tokenizer
445
+ @tokenizer ||= Tokenizer.new
446
+ end
447
+
448
+ def normalizer
449
+ @normalizer ||= Language::Normalizer::Base.new
450
+ end
451
+
452
+ def dictionary_class
453
+ Dictionary::UnixWords
454
+ end
455
+
456
+ def default_dictionary_paths
457
+ case code
458
+ when "fr-FR"
459
+ ["/usr/share/dict/french"]
460
+ when "fr-CA"
461
+ ["/usr/share/dict/french-CA"]
462
+ else
463
+ ["/usr/share/dict/words"]
464
+ end
465
+ end
466
+
467
+ def script_type
468
+ :latin
469
+ end
470
+
471
+ def create_spell_checker
472
+ SpellChecker.new(
473
+ aff_path: @hunspell_paths[:aff],
474
+ dic_path: @hunspell_paths[:dic],
475
+ script: :latin
476
+ )
477
+ end
478
+
479
+ def create_tokenizer
480
+ Tokenizer.new
481
+ end
482
+
483
+ def create_pos_tagger
484
+ POSTagger.new(
485
+ aff_path: @hunspell_paths[:aff],
486
+ dic_path: @hunspell_paths[:dic],
487
+ script: :latin,
488
+ flag_mapping: POSTagger::FLAG_TO_POS
489
+ )
490
+ end
491
+
492
+ private
493
+
494
+ def extract_region_code(code)
495
+ return nil unless code.include?("-")
496
+
497
+ code.split("-", 2).last.upcase
498
+ end
499
+
500
+ def resolve_hunspell_paths(code)
501
+ HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['fr-FR']
502
+ end
503
+ end
504
+ end
505
+ end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Hebrew language implementation.
6
+ #
7
+ # Registers the Hebrew language with {Language::Registry} under the
8
+ # +he+ and +he-IL+ codes. Uses the {Language::Normalizer::Hebrew}
9
+ # normalizer (NFC + niqqud stripping + dagesh removal + maqaf
10
+ # normalization) so that bare dictionary forms match against
11
+ # user input that may include vowel points.
12
+ #
13
+ # @example
14
+ # lang = Kotoshu::Languages::Hebrew.new
15
+ # lang.script_type # => :hebrew
16
+ # lang.rtl? # => true
17
+ #
18
+ # Full-feature since plan 100 (batch 3): the spelling dictionary
19
+ # downloads from the dictionaries repo (AVAILABLE_LANGUAGES) and
20
+ # keyboard proximity uses the Hebrew SI-1452 grid
21
+ # (Keyboard::Layouts::HebrewSI1452).
22
+ class Hebrew < Language::Base
23
+ register "he"
24
+ register "he-IL"
25
+
26
+ def initialize(code: "he", name: "Hebrew", variant: nil)
27
+ super
28
+ end
29
+
30
+ def description
31
+ name
32
+ end
33
+
34
+ def tokenizer
35
+ @tokenizer ||= Tokenizer.new
36
+ end
37
+
38
+ def normalizer
39
+ @normalizer ||= Language::Normalizer::Hebrew.new
40
+ end
41
+
42
+ def dictionary_class
43
+ Dictionary::Custom
44
+ end
45
+
46
+ def default_dictionary_paths
47
+ ["/usr/share/dict/hebrew"]
48
+ end
49
+
50
+ def script_type
51
+ :hebrew
52
+ end
53
+
54
+ def rtl?
55
+ true
56
+ end
57
+
58
+ # Minimal Hebrew tokenizer.
59
+ #
60
+ # Splits on whitespace and standard punctuation — Hebrew text
61
+ # is space-delimited. Includes the Hebrew-specific geresh (׳)
62
+ # and gershayim (״) as word characters (they are used inside
63
+ # acronyms). Uses Base#skip_token? which correctly handles
64
+ # Unicode letters via \p{L}.
65
+ class Tokenizer < Language::Tokenizer::Base
66
+ HEBREW_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·]/
67
+
68
+ def tokenize(text)
69
+ return [] if text.nil? || text.strip.empty?
70
+
71
+ text.split(HEBREW_SEPARATORS)
72
+ .map { |token| normalize(token) }
73
+ .reject { |token| skip_token?(token) }
74
+ end
75
+
76
+ # Spell-check word characters (plan 91 pattern, added by
77
+ # plan 100): Hebrew letters plus the geresh/gershayim used
78
+ # inside acronyms. Niqqud are combining marks, not letters —
79
+ # the Hebrew normalizer strips them from input before lookup.
80
+ #
81
+ # @return [Regexp] Regex matching a single word character
82
+ def spellcheck_word_regex
83
+ /[\p{Hebrew}\u05F3\u05F4]/
84
+ end
85
+
86
+ protected
87
+
88
+ def normalize(token)
89
+ token.strip
90
+ end
91
+
92
+ def word_separators
93
+ HEBREW_SEPARATORS
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Croatian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. The five Croatian digraph letters š đ č ć ž are
9
+ # real keys on the South-Slavic QWERTZ physical layout
10
+ # (Keyboard::Layouts::Croatian), shared with Slovenian.
11
+ class Croatian < LatinBase
12
+ register "hr"
13
+ register "hr-HR"
14
+
15
+ def initialize(code: "hr", name: "Croatian")
16
+ super
17
+ end
18
+
19
+ def default_dictionary_paths
20
+ ["/usr/share/dict/croatian"]
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Hungarian language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Hungarian long vowels ő ű are dead-key sequences on the QWERTZ physical layout.
9
+ class Hungarian < LatinBase
10
+ register "hu"
11
+ register "hu-HU"
12
+
13
+ def initialize(code: "hu", name: "Hungarian")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/hungarian"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Indonesian language implementation (plan 100, batch 3).
6
+ #
7
+ # Thin composition like the wave-1 members: shared Latin
8
+ # tokenizer, base normalizer, registry entry. Indonesian is typed
9
+ # on the unmodified US QWERTY grid (Keyboard::Layouts::Latin::Indonesian)
10
+ # — the language has no diacritics, so the Latin base covers it
11
+ # without any per-language care.
12
+ class Indonesian < LatinBase
13
+ register "id"
14
+ register "id-ID"
15
+
16
+ def initialize(code: "id", name: "Indonesian")
17
+ super
18
+ end
19
+
20
+ def default_dictionary_paths
21
+ ["/usr/share/dict/indonesian"]
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Italian language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Italian accents (à è é ì ò ù) are dead-key sequences on the US physical layout.
9
+ class Italian < LatinBase
10
+ register "it"
11
+ register "it-IT"
12
+
13
+ def initialize(code: "it", name: "Italian")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/italian"]
19
+ end
20
+ end
21
+ end
22
+ end