kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Catalan language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Catalan diacritics (à é í ó ú ç l·l) are dead-key sequences on the Spanish physical layout.
9
+ class Catalan < LatinBase
10
+ register "ca"
11
+ register "ca-ES"
12
+
13
+ def initialize(code: "ca", name: "Catalan")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/catalan"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Czech language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Czech diacritics (háček and čárka letters) are dead-key sequences on the QWERTZ physical layout.
9
+ class Czech < LatinBase
10
+ register "cs"
11
+ register "cs-CZ"
12
+
13
+ def initialize(code: "cs", name: "Czech")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/czech"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Danish language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Danish å æ ø are real keys on the Nordic physical layout (Keyboard::Layouts::Latin::Danish).
9
+ class Danish < LatinBase
10
+ register "da"
11
+ register "da-DK"
12
+
13
+ def initialize(code: "da", name: "Danish")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/danish"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,554 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # German language implementation.
6
+ #
7
+ # Supports multiple dialects: de-DE, de-AT, de-CH, de-BE, de-IT, de-LI, de-LU
8
+ #
9
+ # Full Hunspell integration with spell checking, POS tagging, and grammar rules
10
+ # specifically handling German compound words and capitalization.
11
+ class German < Language::Base
12
+ # German spell checker with Hunspell integration.
13
+ #
14
+ # Uses the Lookup algorithm with Hunspell-format dictionaries
15
+ # and handles German-specific features (umlauts, ß, compound words).
16
+ class SpellChecker < Components::SpellChecker
17
+ attr_reader :aff_path, :dic_path, :script
18
+
19
+ # German-specific character substitutions for suggestions
20
+ GERMAN_SUBSTITUTIONS = {
21
+ # Umlauts
22
+ 'ä' => %w[a ae],
23
+ 'ö' => %w[o oe],
24
+ 'ü' => %w[u ue],
25
+ 'ß' => %w[ss sz],
26
+ # Common German errors
27
+ 'a' => %w[ä],
28
+ 'o' => %w[ö],
29
+ 'u' => %w[ü],
30
+ 's' => %w[ß],
31
+ }.freeze
32
+
33
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
34
+ @aff_path = aff_path
35
+ @dic_path = dic_path
36
+ @script = script
37
+ @encoding = encoding
38
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
39
+ end
40
+
41
+ def check(word)
42
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
43
+
44
+ # Try exact match first
45
+ first_form = @lookuper.good_forms(word).first
46
+ return { found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] } if first_form
47
+
48
+ # Try lowercase version (German nouns are capitalized)
49
+ unless word == word.downcase
50
+ lowercase_form = @lookuper.good_forms(word.downcase).first
51
+ if lowercase_form
52
+ return { found: true, stem: lowercase_form.stem || word.downcase,
53
+ flags: lowercase_form.flags&.to_a || [] }
54
+ end
55
+ end
56
+
57
+ { found: false, stem: nil, flags: [] }
58
+ end
59
+
60
+ def suggest(word, max_suggestions: 10)
61
+ return [] if word.nil? || word.empty?
62
+
63
+ first_form = @lookuper.good_forms(word).first
64
+ return [] if first_form
65
+
66
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
67
+ end
68
+
69
+ def correct?(word)
70
+ check(word)[:found]
71
+ end
72
+
73
+ def lookuper
74
+ @lookuper
75
+ end
76
+
77
+ private
78
+
79
+ def calculate_distance(a, b)
80
+ return a.length if b.empty?
81
+ return b.length if a.empty?
82
+
83
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
84
+ (1..b.length).each { |j| matrix[0][j] = j }
85
+ (1..a.length).each do |i|
86
+ (1..b.length).each do |j|
87
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
88
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
89
+ end
90
+ end
91
+ matrix[a.length][b.length]
92
+ end
93
+
94
+ def calculate_score(original, suggestion, rank)
95
+ distance = calculate_distance(original, suggestion)
96
+ max_len = [original.length, suggestion.length].max
97
+ distance_score = 1.0 - (distance.to_f / max_len)
98
+ rank_penalty = rank * 0.05
99
+ [distance_score - rank_penalty, 0.0].max
100
+ end
101
+
102
+ def generate_suggestions(word, _max_suggestions)
103
+ variations = []
104
+
105
+ # Missing umlauts
106
+ word.downcase.chars.each_with_index do |char, i|
107
+ GERMAN_SUBSTITUTIONS.each do |umlaut, variants|
108
+ variants.each do |variant|
109
+ if char == variant
110
+ umlaut_word = word.dup
111
+ umlaut_word[i] = umlaut
112
+ variations << umlaut_word if @lookuper.good_forms(umlaut_word).first
113
+ end
114
+ end
115
+ end
116
+ end
117
+
118
+ # ß vs ss
119
+ if word.include?('ss')
120
+ eszett_word = word.gsub('ss', 'ß')
121
+ variations << eszett_word if @lookuper.good_forms(eszett_word).first
122
+ elsif word.include?('ß')
123
+ double_s_word = word.gsub('ß', 'ss')
124
+ variations << double_s_word if @lookuper.good_forms(double_s_word).first
125
+ end
126
+
127
+ # Capitalization (German nouns are capitalized)
128
+ if word == word.downcase
129
+ capitalized_word = word.capitalize
130
+ variations << capitalized_word if @lookuper.good_forms(capitalized_word).first
131
+ end
132
+
133
+ # Doubled letters
134
+ word.chars.each_with_index do |char, i|
135
+ next if i == 0
136
+
137
+ doubled = word.dup
138
+ doubled.insert(i, char)
139
+ variations << doubled if @lookuper.good_forms(doubled).first
140
+ end
141
+
142
+ # Deleted letters
143
+ (0...word.length).each do |i|
144
+ deleted = word.dup
145
+ deleted.slice!(i)
146
+ next if deleted.empty?
147
+
148
+ variations << deleted if @lookuper.good_forms(deleted).first
149
+ end
150
+
151
+ # Compound word splitting (German has long compound words)
152
+ if word.length > 10
153
+ # Try splitting common compound patterns
154
+ common_prefixes = %w[Arbeits Baum Bau Bauern Berg Buch Dach Dollar Dorf Ein Frauen Feuer Finanz Flug Franz
155
+ Frei Haupt Haus Hoch Jahr Jung Kinder Klein Konsum Land Lehr Leben Leute Mann MarktMein Milli
156
+ Morgen Mutter Natur Papier Polizei Post Post Problem Recht Rhein Rot Sache Schule Schiff Schritt
157
+ Schiff See Sozial Stadt Stein Steuer Strom Tag Teil Tier Tor Tour Typ Uhr Umwelt Unter Volk
158
+ Wasser Weg Welt Wein Welt Zeit]
159
+ common_prefixes.each do |prefix|
160
+ if word.start_with?(prefix)
161
+ split_word = prefix + ' ' + word[prefix.length..]
162
+ # Check if both parts are valid
163
+ prefix_valid = @lookuper.good_forms(prefix).first
164
+ suffix_valid = @lookuper.good_forms(word[prefix.length..]).first
165
+ if prefix_valid && suffix_valid
166
+ variations << split_word
167
+ end
168
+ end
169
+ end
170
+ end
171
+
172
+ variations.uniq!
173
+ variations.map do |suggestion|
174
+ { word: suggestion, distance: calculate_distance(word, suggestion),
175
+ score: calculate_score(word, suggestion, 0) }
176
+ end.sort_by { |s| s[:distance] }
177
+ end
178
+ end
179
+
180
+ # German tokenizer with special character handling.
181
+ class Tokenizer < Language::Tokenizer::GermanTokenizer
182
+ end
183
+
184
+ # German POS tagger.
185
+ #
186
+ # Derives POS tags from Hunspell flags using German-specific mappings.
187
+ class POSTagger < Components::PosTagger
188
+ # German POS flag mappings based on Hunspell German dictionaries
189
+ FLAG_TO_POS = {
190
+ # Nouns (German nouns are capitalized)
191
+ 'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
192
+ 'Sub' => 'NOUN',
193
+ # Verbs
194
+ 'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
195
+ 'VBP' => 'VERB', 'VBZ' => 'VERB',
196
+ 'Vfin' => 'VERB', 'Vinf' => 'VERB', 'Vpp' => 'VERB',
197
+ # Adjectives
198
+ 'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
199
+ 'Adj' => 'ADJ',
200
+ # Adverbs
201
+ 'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
202
+ 'Adv' => 'ADV',
203
+ # Determiners
204
+ 'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
205
+ 'Art' => 'DET',
206
+ # Pronouns
207
+ 'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
208
+ 'WP' => 'PRON', 'WP$' => 'PRON_POSS',
209
+ 'Pro' => 'PRON',
210
+ # Prepositions
211
+ 'I' => 'PREP', 'IN' => 'PREP',
212
+ 'Prä' => 'PREP',
213
+ # Conjunctions
214
+ 'C' => 'CONJ', 'CC' => 'CONJ',
215
+ 'Kon' => 'CONJ',
216
+ # Particles
217
+ 'U' => 'PART', 'RP' => 'PART',
218
+ 'Pt' => 'PART',
219
+ # Interjections
220
+ 'INTJ' => 'INTJ', 'UH' => 'INTJ',
221
+ 'Int' => 'INTJ',
222
+ # Numbers
223
+ 'CD' => 'NUM',
224
+ 'Num' => 'NUM',
225
+ # Foreign words
226
+ 'FW' => 'X',
227
+ # Punctuation
228
+ 'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
229
+ '?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
230
+ }.freeze
231
+
232
+ attr_reader :aff_path, :dic_path, :script
233
+
234
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
235
+ @aff_path = aff_path
236
+ @dic_path = dic_path
237
+ @script = script
238
+ @encoding = encoding
239
+ @flag_mapping = flag_mapping
240
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
241
+ @lookup_cache = {}
242
+ end
243
+
244
+ def tag(tokens)
245
+ return [] if tokens.nil? || tokens.empty?
246
+
247
+ tokens.map do |token|
248
+ word = token[:token]
249
+ if word.nil? || word.empty?
250
+ token.merge(pos_tag: nil, lemma: nil)
251
+ else
252
+ lookup_result = lookup_with_pos(word)
253
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
254
+ end
255
+ end
256
+ end
257
+
258
+ def flag_mapping
259
+ @flag_mapping
260
+ end
261
+
262
+ def flag_mapping=(mapping)
263
+ @flag_mapping = mapping
264
+ end
265
+
266
+ def clear_cache
267
+ @lookup_cache.clear
268
+ end
269
+
270
+ private
271
+
272
+ def lookup_with_pos(word)
273
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
274
+ return @lookup_cache[word] if @lookup_cache.key?(word)
275
+
276
+ # German nouns are capitalized - try lowercase if capitalized doesn't work
277
+ first_form = @lookuper.good_forms(word).first
278
+ if !first_form && word == word.capitalize && word.length > 1
279
+ first_form = @lookuper.good_forms(word.downcase).first
280
+ end
281
+
282
+ pos_tag = derive_pos_tag(first_form)
283
+ cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
284
+ @lookup_cache[word] = cache_result
285
+ cache_result
286
+ end
287
+
288
+ def derive_pos_tag(result)
289
+ return nil unless result
290
+
291
+ flags = result.flags&.to_a || []
292
+ return guess_pos_from_affix(result) if flags.empty?
293
+
294
+ flags.each do |flag|
295
+ pos_tag = flag_to_pos(flag)
296
+ return pos_tag if pos_tag
297
+ end
298
+ guess_pos_from_affix(result)
299
+ end
300
+
301
+ def flag_to_pos(flag)
302
+ return @flag_mapping[flag] if @flag_mapping.key?(flag)
303
+
304
+ first_char = flag[0]
305
+ @flag_mapping[first_char]
306
+ end
307
+
308
+ def guess_pos_from_affix(result)
309
+ suffix = result.suffix
310
+ return guess_pos_from_suffix(suffix) if suffix
311
+
312
+ nil
313
+ end
314
+
315
+ def guess_pos_from_suffix(suffix)
316
+ # German suffix patterns
317
+ return 'VERB' if suffix.match?(/^(en|eln|ern|ten|tet|t|is|ieren)$/)
318
+ return 'ADV' if suffix.match?(/^(lich|weise|lings|maß|mäßig)$/)
319
+ return 'NOUN' if suffix.match?(/^(ung|heit|keit|schaft|tion|ismus|tum|ling|ner|eur)$/)
320
+ return 'ADJ' if suffix.match?(/^(isch|ig|lich|bar|sam|haft|los|mäßig)$/)
321
+
322
+ nil
323
+ end
324
+ end
325
+
326
+ # German grammar rules module.
327
+ module GrammarRules
328
+ # Base class for German grammar rules.
329
+ class Rule
330
+ attr_reader :id, :name, :description
331
+
332
+ def initialize(id, name, description)
333
+ @id = id
334
+ @name = name
335
+ @description = description
336
+ end
337
+
338
+ def check(tokens)
339
+ raise NotImplementedError, "#{self.class} must implement #check"
340
+ end
341
+
342
+ def applies?(_tokens, _index)
343
+ true
344
+ end
345
+ end
346
+
347
+ # Rule: German noun capitalization.
348
+ class NounCapitalizationRule < Rule
349
+ # Common German noun suffixes
350
+ NOUN_SUFFIXES = %w[ung heit keit schaft tion ismus tum ling ner eur
351
+ able ibil ig igkeit lich sam los losung].freeze
352
+
353
+ def initialize
354
+ super('DE_NOUN_CAPITALIZATION', 'Noun Capitalization', 'German nouns must be capitalized.')
355
+ end
356
+
357
+ def check(tokens)
358
+ errors = []
359
+ tokens.each_with_index do |token, idx|
360
+ word = token[:token]
361
+ next if word.nil? || word.empty?
362
+ next if word == word.capitalize # Already capitalized
363
+ next if word.length < 3 # Too short
364
+ next unless word.match?(/^[a-zäöüß]+$/i) # Only letters
365
+
366
+ # Check if it looks like a noun (has noun suffix or is in noun position)
367
+ if word.end_with?(*NOUN_SUFFIXES)
368
+ errors << {
369
+ rule_id: @id,
370
+ position: token[:position],
371
+ message: "German nouns must be capitalized: '#{word}'",
372
+ suggestion: word.capitalize,
373
+ context: word,
374
+ suggestions: [word.capitalize]
375
+ }
376
+ end
377
+
378
+ # Check position: after determiners often indicates a noun
379
+ if idx > 0
380
+ prev_token = tokens[idx - 1][:token]&.downcase
381
+ if %w[der die das ein eine einem einen einer
382
+ eines].include?(prev_token) && word == word.downcase && word.length > 2
383
+ errors << {
384
+ rule_id: @id,
385
+ position: token[:position],
386
+ message: "German nouns must be capitalized after articles: '#{word}'",
387
+ suggestion: word.capitalize,
388
+ context: "#{prev_token} #{word}",
389
+ suggestions: [word.capitalize]
390
+ }
391
+ end
392
+ end
393
+ end
394
+ errors
395
+ end
396
+ end
397
+
398
+ # Rule: Compound word spacing (German compounds are written together).
399
+ class CompoundSpacingRule < Rule
400
+ def initialize
401
+ super('DE_COMPOUND_SPACING', 'Compound Spacing', 'German compound words should not have spaces.')
402
+ end
403
+
404
+ def check(tokens)
405
+ errors = []
406
+ tokens.each_with_index do |token, idx|
407
+ next unless idx < tokens.length - 1
408
+
409
+ word1 = token[:token]
410
+ word2 = tokens[idx + 1][:token]
411
+ next if word1.nil? || word2.nil?
412
+
413
+ # Check if both are lowercase (might be parts of a compound)
414
+ if word1.match?(/^[a-zäöüß]+$/) && word2.match?(/^[a-zäöüß]+$/)
415
+ # Suggest they might be a compound word
416
+ compound = word1 + word2
417
+ errors << {
418
+ rule_id: @id,
419
+ position: token[:position],
420
+ message: "Possible compound word: '#{word1} #{word2}' should be '#{compound}'",
421
+ suggestion: compound,
422
+ context: "#{word1} #{word2}",
423
+ suggestions: [compound]
424
+ }
425
+ end
426
+ end
427
+ errors
428
+ end
429
+ end
430
+
431
+ # Rule registry for German.
432
+ class RuleRegistry
433
+ class << self
434
+ def default_rules
435
+ [NounCapitalizationRule.new, CompoundSpacingRule.new]
436
+ end
437
+
438
+ def get_rule(id)
439
+ default_rules.find { |rule| rule.id == id }
440
+ end
441
+ end
442
+ end
443
+ end
444
+
445
+ # Registration
446
+ register "de"
447
+ register "de-DE"
448
+ register "de-AT"
449
+ register "de-CH"
450
+ register "de-BE"
451
+ register "de-IT"
452
+ register "de-LI"
453
+ register "de-LU"
454
+
455
+ HUNSPELL_DICTIONARIES = {
456
+ 'de-DE' => {
457
+ aff: 'spec/integrational/fixtures/de_DE.aff',
458
+ dic: 'spec/integrational/fixtures/de_DE.dic'
459
+ },
460
+ 'de-AT' => {
461
+ aff: 'spec/integrational/fixtures/de_AT.aff',
462
+ dic: 'spec/integrational/fixtures/de_AT.dic'
463
+ },
464
+ 'de-CH' => {
465
+ aff: 'spec/integrational/fixtures/de_CH.aff',
466
+ dic: 'spec/integrational/fixtures/de_CH.dic'
467
+ }
468
+ }.freeze
469
+
470
+ VARIANT_NAMES = {
471
+ 'DE' => 'German',
472
+ 'AT' => 'Austrian',
473
+ 'CH' => 'Swiss',
474
+ 'BE' => 'Belgian',
475
+ 'IT' => 'South Tyrolean',
476
+ 'LI' => 'Liechtenstein',
477
+ 'LU' => 'Luxembourgish'
478
+ }.freeze
479
+
480
+ def initialize(code: "de", name: "German", variant: nil)
481
+ variant ||= extract_region_code(code)
482
+ super
483
+ @hunspell_paths = resolve_hunspell_paths(code)
484
+ end
485
+
486
+ def description
487
+ return name unless variant
488
+
489
+ variant_name = VARIANT_NAMES[variant] || variant
490
+ "#{name} (#{variant_name})"
491
+ end
492
+
493
+ def tokenizer
494
+ @tokenizer ||= Tokenizer.new
495
+ end
496
+
497
+ def normalizer
498
+ @normalizer ||= Language::Normalizer::Base.new
499
+ end
500
+
501
+ def dictionary_class
502
+ Dictionary::UnixWords
503
+ end
504
+
505
+ def default_dictionary_paths
506
+ case code
507
+ when "de-DE", "de-AT", "de-BE"
508
+ ["/usr/share/dict/german"]
509
+ when "de-CH"
510
+ ["/usr/share/dict/swiss-german"]
511
+ else
512
+ ["/usr/share/dict/words"]
513
+ end
514
+ end
515
+
516
+ def script_type
517
+ :latin
518
+ end
519
+
520
+ def create_spell_checker
521
+ SpellChecker.new(
522
+ aff_path: @hunspell_paths[:aff],
523
+ dic_path: @hunspell_paths[:dic],
524
+ script: :latin
525
+ )
526
+ end
527
+
528
+ def create_tokenizer
529
+ Tokenizer.new
530
+ end
531
+
532
+ def create_pos_tagger
533
+ POSTagger.new(
534
+ aff_path: @hunspell_paths[:aff],
535
+ dic_path: @hunspell_paths[:dic],
536
+ script: :latin,
537
+ flag_mapping: POSTagger::FLAG_TO_POS
538
+ )
539
+ end
540
+
541
+ private
542
+
543
+ def extract_region_code(code)
544
+ return nil unless code.include?("-")
545
+
546
+ code.split("-", 2).last.upcase
547
+ end
548
+
549
+ def resolve_hunspell_paths(code)
550
+ HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['de-DE']
551
+ end
552
+ end
553
+ end
554
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Greek language implementation (plan 84, wave 1).
6
+ #
7
+ # Greek-aware composition: the Greek tokenizer keeps Greek-script
8
+ # words (the Latin tokenizer's letter checks would drop them), and
9
+ # the Greek normalizer handles the two script-specific folds:
10
+ #
11
+ # - final sigma: ΕΛΛΑΣ folds to ελλας (ς), not ελλασ (σ), matching
12
+ # dictionary forms;
13
+ # - optional accent stripping (πάτησις -> πατησις) for
14
+ # accent-insensitive comparison, off by default because Greek
15
+ # Hunspell dictionaries carry accents.
16
+ #
17
+ # Keyboard proximity uses the Greek phonetic grid
18
+ # (Keyboard::Layouts::GreekPhonetic).
19
+ class Greek < LatinBase
20
+ register "el"
21
+ register "el-GR"
22
+
23
+ def initialize(code: "el", name: "Greek")
24
+ super
25
+ end
26
+
27
+ # Greek-script tokenizer.
28
+ #
29
+ # @return [Language::Tokenizer::GreekTokenizer]
30
+ def tokenizer
31
+ @tokenizer ||= Language::Tokenizer::GreekTokenizer.new
32
+ end
33
+
34
+ # Greek-aware normalizer (final sigma, accent stripping).
35
+ #
36
+ # @return [Language::Normalizer::Greek]
37
+ def normalizer
38
+ @normalizer ||= Language::Normalizer::Greek.new
39
+ end
40
+
41
+ def script_type
42
+ :greek
43
+ end
44
+
45
+ def default_dictionary_paths
46
+ ["/usr/share/dict/greek"]
47
+ end
48
+ end
49
+ end
50
+ end