kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,201 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ # Registry for keyboard layouts
6
+ #
7
+ # The registry provides a centralized way to access keyboard layouts
8
+ # and automatically selects the appropriate layout for a given language.
9
+ #
10
+ # @example Getting layout for a language
11
+ # layout = Keyboard::Registry.layout_for('de')
12
+ # layout.name # => "QWERTZ"
13
+ #
14
+ # @example Getting layout by name
15
+ # layout = Keyboard::Registry.layout_by_name('Dvorak')
16
+ # layout.name # => "Dvorak"
17
+ #
18
+ # @example Listing all available layouts
19
+ # Keyboard::Registry.available_layouts.each do |layout|
20
+ # puts "#{layout.name}: #{layout.language_codes.join(', ')}"
21
+ # end
22
+ #
23
+ class Registry
24
+ class << self
25
+ # Register a keyboard layout
26
+ #
27
+ # @param layout_class [Class<Layout>] the layout class to register
28
+ # @return [Layout] the instantiated layout
29
+ def register(layout_class)
30
+ layouts[layout_class.name] = layout_class.new
31
+ end
32
+
33
+ # Get layout for a specific language code
34
+ #
35
+ # Searches for a layout that supports the given language code.
36
+ # Falls back to the script default for module-less languages
37
+ # (plan 107): JCUKEN for Cyrillic, Arabic 101 for Arabic
38
+ # script, Hebrew SI-1452 for Hebrew, QWERTY for Latin and
39
+ # unknown codes.
40
+ #
41
+ # @param language_code [String] the language code (e.g., 'en', 'de', 'fr', 'ru')
42
+ # @return [Layout] the keyboard layout for the language
43
+ def layout_for(language_code)
44
+ # Try exact match first
45
+ layout = layouts.values.find { |l| l.supports_language?(language_code) }
46
+
47
+ # Try base language if variant (e.g., 'en-GB' -> 'en')
48
+ unless layout
49
+ base_lang = language_code.to_s.split('-').first
50
+ layout = layouts.values.find { |l| l.supports_language?(base_lang) }
51
+ end
52
+
53
+ layout || script_default_layout(language_code) || default_layout
54
+ end
55
+
56
+ # Get layout by name
57
+ #
58
+ # @param name [String, Symbol] the layout name (e.g., 'QWERTY', 'Dvorak')
59
+ # @return [Layout] the layout, or QWERTY as fallback if not found
60
+ def layout_by_name(name)
61
+ name_str = name.to_s
62
+ result = layouts.values.find do |layout|
63
+ layout.name == name_str ||
64
+ layout.class.name.end_with?("::#{name_str}")
65
+ end
66
+
67
+ # Return QWERTY as fallback (not default_layout to avoid recursion)
68
+ result || layouts['Kotoshu::Keyboard::Layouts::QWERTY']
69
+ end
70
+
71
+ # Get all available layouts
72
+ #
73
+ # @return [Array<Layout>] list of all registered layouts
74
+ def available_layouts
75
+ layouts.values
76
+ end
77
+
78
+ # Get all supported language codes
79
+ #
80
+ # @return [Array<String>] list of all language codes across all layouts
81
+ def supported_languages
82
+ layouts.values.flat_map(&:language_codes).uniq.sort
83
+ end
84
+
85
+ # Set the default layout
86
+ #
87
+ # @param layout_name [String, Symbol] the name of the layout to use as default
88
+ def register_default(layout_name)
89
+ @default_layout_name = layout_name
90
+ end
91
+
92
+ # Check if a language is supported
93
+ #
94
+ # @param language_code [String] the language code to check
95
+ # @return [Boolean] true if the language is supported by any layout
96
+ def supports_language?(language_code)
97
+ layouts.values.any? { |l| l.supports_language?(language_code) }
98
+ end
99
+
100
+ # Clear all registered layouts (mainly for testing)
101
+ #
102
+ # @return [void]
103
+ def clear!
104
+ @layouts = nil
105
+ @default_layout_name = nil
106
+ end
107
+
108
+ private
109
+
110
+ # Get or initialize the layouts hash
111
+ #
112
+ # @return [Hash] hash of layout class names to instances
113
+ def layouts
114
+ @layouts ||= {}
115
+ end
116
+
117
+ # Generic keyboard fallback for module-less languages
118
+ # (plan 107): the script's standard grid when no registered
119
+ # layout claims the code. Latin and unknown scripts fall
120
+ # through to +default_layout+ (QWERTY).
121
+ #
122
+ # @param language_code [String] the language code
123
+ # @return [Layout, nil] the script default, or nil
124
+ def script_default_layout(language_code)
125
+ case Language::Script.script_for(language_code.to_s)
126
+ when :cyrillic then layouts['Kotoshu::Keyboard::Layouts::JCUKEN']
127
+ when :arabic then layouts['Kotoshu::Keyboard::Layouts::Arabic101']
128
+ when :hebrew then layouts['Kotoshu::Keyboard::Layouts::HebrewSI1452']
129
+ end
130
+ end
131
+
132
+ # Get the default layout
133
+ #
134
+ # @return [Layout] the default layout (QWERTY if none specified)
135
+ def default_layout
136
+ if @default_layout_name
137
+ name_str = @default_layout_name.to_s
138
+ layout = layouts.values.find do |l|
139
+ l.name == name_str || l.class.name.end_with?("::#{name_str}")
140
+ end
141
+ return layout if layout
142
+ end
143
+
144
+ # Return QWERTY as the ultimate fallback
145
+ layouts['Kotoshu::Keyboard::Layouts::QWERTY'] || layouts.values.first
146
+ end
147
+ end
148
+
149
+ # Auto-register all layout classes on load
150
+ register(Layouts::QWERTY)
151
+ register(Layouts::QWERTZ)
152
+ register(Layouts::AZERTY)
153
+ # Serbian Latin registers before Serbian Cyrillic: the Cyrillic
154
+ # layout claims sr, and base-language matching would fold
155
+ # sr-Latn onto it; the Latin variant needs its own grid (plan
156
+ # 110). Ukrainian, Bulgarian and Serbian register before JCUKEN:
157
+ # the base JCUKEN layout blanket-claims uk bg, but typists of
158
+ # those languages get their dedicated grids (plan 84 uk, plan
159
+ # 100 bg/sr). Lookup is first-registered-wins, so order here is
160
+ # load-bearing for uk bg sr sr-Latn.
161
+ register(Layouts::SerbianLatin)
162
+ register(Layouts::Ukrainian)
163
+ register(Layouts::BulgarianBds)
164
+ register(Layouts::SerbianCyrillic)
165
+ register(Layouts::JCUKEN)
166
+ register(Layouts::Dvorak)
167
+ # Wave-1 national layouts (plan 84)
168
+ register(Layouts::TurkishQ)
169
+ register(Layouts::GreekPhonetic)
170
+ # Parameterized Latin family members (plan 84)
171
+ register(Layouts::Latin::Italian)
172
+ register(Layouts::Latin::Dutch)
173
+ register(Layouts::Latin::Polish)
174
+ register(Layouts::Latin::Czech)
175
+ register(Layouts::Latin::Hungarian)
176
+ register(Layouts::Latin::Romanian)
177
+ register(Layouts::Latin::Catalan)
178
+ register(Layouts::Latin::Vietnamese)
179
+ register(Layouts::Latin::Danish)
180
+ register(Layouts::Latin::Norwegian)
181
+ register(Layouts::Latin::Swedish)
182
+ # Batch-3 national layouts (plan 100): ar fa he type on their
183
+ # national-script grids; hr and sl share the South-Slavic QWERTZ
184
+ # grid; the id sk et lt lv Latin members follow.
185
+ register(Layouts::Arabic101)
186
+ register(Layouts::PersianStandard)
187
+ register(Layouts::HebrewSI1452)
188
+ register(Layouts::Croatian)
189
+ register(Layouts::Slovenian)
190
+ register(Layouts::Latin::Indonesian)
191
+ register(Layouts::Latin::Slovak)
192
+ register(Layouts::Latin::Estonian)
193
+ register(Layouts::Latin::Lithuanian)
194
+ register(Layouts::Latin::Latvian)
195
+ # ko and ne national layouts (plan 108): Dubeolsik (KS X 5002)
196
+ # and Devanagari InScript.
197
+ register(Layouts::Dubeolsik)
198
+ register(Layouts::DevanagariInScript)
199
+ end
200
+ end
201
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Keyboard layout system for Kotoshu
5
+ #
6
+ # This module provides access to keyboard layouts for typo detection
7
+ # and suggestion ranking in multi-language spell checking.
8
+ #
9
+ # @example Getting a keyboard layout for a language
10
+ # layout = Kotoshu::Keyboard.layout_for('de')
11
+ # layout.distance('z', 'y') # => 1 (adjacent on QWERTZ)
12
+ #
13
+ # @example Getting a layout by name
14
+ # dvorak = Kotoshu::Keyboard.layout_by_name('Dvorak')
15
+ # dvorak.distance('a', 'e') # => 2 (home row on Dvorak)
16
+ #
17
+ module Keyboard
18
+ autoload :Layout, "kotoshu/keyboard/layout"
19
+ autoload :Registry, "kotoshu/keyboard/registry"
20
+
21
+ module Layouts
22
+ autoload :QWERTY, "kotoshu/keyboard/layouts/qwerty"
23
+ autoload :QWERTZ, "kotoshu/keyboard/layouts/qwertz"
24
+ autoload :AZERTY, "kotoshu/keyboard/layouts/azerty"
25
+ autoload :JCUKEN, "kotoshu/keyboard/layouts/jcuken"
26
+ autoload :Dvorak, "kotoshu/keyboard/layouts/dvorak"
27
+ # Wave-1 national layouts (plan 84): grids mirrored from the
28
+ # models repo eval harness.
29
+ autoload :TurkishQ, "kotoshu/keyboard/layouts/turkish_q"
30
+ autoload :Ukrainian, "kotoshu/keyboard/layouts/ukrainian"
31
+ autoload :GreekPhonetic, "kotoshu/keyboard/layouts/greek"
32
+ # Parameterized Latin family (plan 84): one file, per-language
33
+ # declarations over the qwerty/qwertz base grids.
34
+ autoload :Latin, "kotoshu/keyboard/layouts/latin"
35
+ # Batch-3 national layouts (plan 100): grids mirrored from the
36
+ # models repo eval harness (plan 83 batch-2 grids).
37
+ autoload :Arabic101, "kotoshu/keyboard/layouts/arabic"
38
+ autoload :PersianStandard, "kotoshu/keyboard/layouts/persian"
39
+ autoload :HebrewSI1452, "kotoshu/keyboard/layouts/hebrew"
40
+ autoload :BulgarianBds, "kotoshu/keyboard/layouts/bulgarian"
41
+ autoload :SerbianCyrillic, "kotoshu/keyboard/layouts/serbian"
42
+ autoload :Croatian, "kotoshu/keyboard/layouts/hrsl"
43
+ autoload :Slovenian, "kotoshu/keyboard/layouts/hrsl"
44
+ # Serbian Latin (plan 110): the South-Slavic QWERTZ grid the hr
45
+ # and sl layouts share, claiming sr-Latn so it does not fold
46
+ # onto the Serbian Cyrillic grid.
47
+ autoload :SerbianLatin, "kotoshu/keyboard/layouts/hrsl"
48
+ # ko and ne layouts (plan 108): Dubeolsik cites the KS X 5002
49
+ # layout standard (the eval harness models ko slips as jamo
50
+ # confusion pairs, not a key grid); Devanagari InScript is
51
+ # mirrored from the eval harness grid.
52
+ autoload :Dubeolsik, "kotoshu/keyboard/layouts/dubeolsik"
53
+ autoload :DevanagariInScript, "kotoshu/keyboard/layouts/devanagari_inscript"
54
+ end
55
+
56
+ class << self
57
+ # Get keyboard layout for a language code
58
+ #
59
+ # @param language_code [String] the language code (e.g., 'en', 'de', 'fr', 'ru')
60
+ # @return [Layout] the keyboard layout for the language
61
+ def layout_for(language_code)
62
+ Registry.layout_for(language_code)
63
+ end
64
+
65
+ # Get keyboard layout by name
66
+ #
67
+ # @param name [String, Symbol] the layout name (e.g., 'QWERTY', 'Dvorak')
68
+ # @return [Layout, nil] the layout, or nil if not found
69
+ def layout_by_name(name)
70
+ Registry.layout_by_name(name)
71
+ end
72
+
73
+ # Get all available layouts
74
+ #
75
+ # @return [Array<Layout>] list of all registered layouts
76
+ def available_layouts
77
+ Registry.available_layouts
78
+ end
79
+
80
+ # Get all supported language codes
81
+ #
82
+ # @return [Array<String>] list of all language codes across all layouts
83
+ def supported_languages
84
+ Registry.supported_languages
85
+ end
86
+
87
+ # Check if a language is supported
88
+ #
89
+ # @param language_code [String] the language code to check
90
+ # @return [Boolean] true if the language is supported by any layout
91
+ def supports_language?(language_code)
92
+ Registry.supports_language?(language_code)
93
+ end
94
+ end
95
+ end
96
+ end
Binary file
@@ -0,0 +1,242 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # Language detection based on character sets and patterns.
6
+ #
7
+ # Uses Unicode character ranges to identify probable language.
8
+ # Provides confidence scoring for multiple matches.
9
+ #
10
+ # @example Detect language
11
+ # Language::Detector.detect("Hello world") # => "en"
12
+ class Detector
13
+ # Character set ranges for language detection
14
+ CHARACTER_SETS = {
15
+ cyrillic: /\p{Cyrillic}/,
16
+ hiragana: /[\u3040-\u309F]/,
17
+ katakana: /[\u30A0-\u30FF]/,
18
+ cjk: /[\u4E00-\u9FFF]/,
19
+ hangul: /[\uAC00-\uD7AF]/,
20
+ latin: /[a-zA-Zà-ÿ]/
21
+ }.freeze
22
+
23
+ # Language-specific patterns
24
+ LANGUAGE_PATTERNS = {
25
+ # Russian: Cyrillic
26
+ russian: {
27
+ pattern: /\p{Cyrillic}[а-яА-ЯёЁ]/,
28
+ min_ratio: 0.3,
29
+ scripts: [:cyrillic]
30
+ },
31
+
32
+ # Japanese: Mixed script (Hiragana + Katakana + Kanji)
33
+ japanese: {
34
+ pattern: /[\u3040-\u309F]|[\u30A0-\u30FF]|[\u4E00-\u9FFF]/,
35
+ min_ratio: 0.2,
36
+ scripts: %i[hiragana katakana cjk],
37
+ must_have: [:hiragana] # Only require hiragana, not both
38
+ },
39
+
40
+ # Portuguese: Latin with specific accents
41
+ portuguese: {
42
+ pattern: /[ãõáàâéêíóôúç]/i,
43
+ min_ratio: 0.05,
44
+ scripts: [:latin]
45
+ },
46
+
47
+ # French: Latin with specific accents (NOT German umlauts)
48
+ french: {
49
+ pattern: /[éèêëàâùûüîïôç]/i, # Removed ä, ö (not French)
50
+ min_ratio: 0.02, # Lower threshold
51
+ scripts: [:latin],
52
+ priority: 1 # Higher priority than English
53
+ },
54
+
55
+ # Spanish: Latin with inverted punctuation
56
+ spanish: {
57
+ pattern: /[áéíóúüñ¿¡]/i,
58
+ min_ratio: 0.02, # Lower threshold
59
+ scripts: [:latin],
60
+ priority: 1
61
+ },
62
+
63
+ # German: Latin with umlauts and eszett
64
+ german: {
65
+ pattern: /[äöüßÄÖÜ]/, # Explicitly include uppercase
66
+ min_ratio: 0.02, # Lower threshold
67
+ scripts: [:latin],
68
+ priority: 1
69
+ },
70
+
71
+ # English: Latin with minimal accents
72
+ english: {
73
+ pattern: /[a-zA-Z]/,
74
+ min_ratio: 0.3,
75
+ scripts: [:latin],
76
+ max_accent_ratio: 0.02
77
+ }
78
+ }.freeze
79
+
80
+ # Language code mapping
81
+ CODE_MAPPING = {
82
+ russian: "ru",
83
+ japanese: "ja",
84
+ portuguese: "pt",
85
+ french: "fr",
86
+ spanish: "es",
87
+ german: "de",
88
+ english: "en"
89
+ }.freeze
90
+
91
+ class << self
92
+ # Detect language from text.
93
+ #
94
+ # Returns the most probable language code based on character analysis.
95
+ #
96
+ # @param text [String] Text to analyze
97
+ # @return [String, nil] Detected language code or nil if uncertain
98
+ def detect(text)
99
+ return nil if text.nil? || text.strip.empty?
100
+
101
+ scores = analyze_languages(text)
102
+ return nil if scores.empty?
103
+
104
+ # Sort by score, then by priority (higher priority first)
105
+ result = scores.max_by do |code, score|
106
+ config = LANGUAGE_PATTERNS.find { |k, _v| CODE_MAPPING[k] == code }
107
+ priority = config&.last&.dig(:priority) || 0
108
+ [score, priority]
109
+ end
110
+
111
+ result&.first
112
+ end
113
+
114
+ # Detect with confidence score.
115
+ #
116
+ # @param text [String] Text to analyze
117
+ # @return [Array<String, Float>] Language code and confidence (0-1)
118
+ def detect_with_confidence(text)
119
+ return [nil, 0.0] if text.nil? || text.strip.empty?
120
+
121
+ scores = analyze_languages(text)
122
+ return [nil, 0.0] if scores.empty?
123
+
124
+ top_language, top_score = scores.max_by { |_, score| score }
125
+ confidence = normalize_confidence(top_score, scores.values)
126
+
127
+ [top_language, confidence]
128
+ end
129
+
130
+ # Get multiple language candidates.
131
+ #
132
+ # @param text [String] Text to analyze
133
+ # @param limit [Integer] Maximum candidates to return
134
+ # @return [Array<Array<String, Float>>] Array of [code, confidence] pairs
135
+ def detect_candidates(text, limit: 3)
136
+ return [] if text.nil? || text.strip.empty?
137
+
138
+ scores = analyze_languages(text)
139
+ return [] if scores.empty?
140
+
141
+ total_score = scores.values.sum.to_f
142
+ scores
143
+ .sort_by { |_, score| -score }
144
+ .first(limit)
145
+ .map { |code, score| [code, score / total_score] }
146
+ end
147
+
148
+ private
149
+
150
+ # Analyze text and score each language.
151
+ #
152
+ # @param text [String] Text to analyze
153
+ # @return [Hash] Hash mapping language codes to scores
154
+ def analyze_languages(text)
155
+ text_length = text.length.to_f
156
+ return {} if text_length.zero?
157
+
158
+ scores = {}
159
+
160
+ LANGUAGE_PATTERNS.each do |language, config|
161
+ score = score_language(text, language, config, text_length)
162
+ scores[CODE_MAPPING[language]] = score if score > 0
163
+ end
164
+
165
+ scores
166
+ end
167
+
168
+ # Score a specific language against text.
169
+ #
170
+ # @param text [String] Text to analyze
171
+ # @param language [Symbol] Language key
172
+ # @param config [Hash] Language configuration
173
+ # @param text_length [Float] Length of text
174
+ # @return [Float] Score (0-1)
175
+ def score_language(text, language, config, text_length)
176
+ # Check required scripts
177
+ if config[:must_have] && !config[:must_have].all? do |script|
178
+ text.match?(CHARACTER_SETS[script])
179
+ end
180
+ return 0
181
+ end
182
+
183
+ # Check forbidden scripts
184
+ if config[:must_not_have] && config[:must_not_have].any? do |script|
185
+ text.match?(CHARACTER_SETS[script])
186
+ end
187
+ return 0
188
+ end
189
+
190
+ # Count matching characters
191
+ matches = text.scan(config[:pattern]).length
192
+ ratio = matches / text_length
193
+
194
+ # Check minimum ratio
195
+ return 0 if ratio < config[:min_ratio]
196
+
197
+ # Check maximum accent ratio (for English)
198
+ if config[:max_accent_ratio]
199
+ accent_chars = text.scan(/[à-ÿ]/).length
200
+ accent_ratio = accent_chars / text_length
201
+ return 0 if accent_ratio > config[:max_accent_ratio]
202
+ end
203
+
204
+ # Bonus for having required scripts
205
+ score = ratio
206
+ if config[:scripts]
207
+ script_bonus = config[:scripts].count do |script|
208
+ text.match?(CHARACTER_SETS[script])
209
+ end
210
+ score *= (1 + (script_bonus * 0.1))
211
+ end
212
+
213
+ # Extra bonus for non-Latin specific characters (accents, umlauts, etc.)
214
+ # This helps distinguish languages with special characters from plain English
215
+ if language != :english && matches > 0
216
+ # Calculate what portion of the text is the special characters
217
+ special_char_ratio = matches / text_length
218
+ # Give bonus proportional to special character presence
219
+ score *= (1 + special_char_ratio)
220
+ end
221
+
222
+ [score, 1.0].min
223
+ end
224
+
225
+ # Normalize confidence score.
226
+ #
227
+ # @param top_score [Float] Highest score
228
+ # @param all_scores [Array<Float>] All scores
229
+ # @return [Float] Normalized confidence (0-1)
230
+ def normalize_confidence(top_score, all_scores)
231
+ return 0.0 if top_score.zero?
232
+
233
+ second_best = all_scores.sort.reverse[1] || 0
234
+ return 1.0 if second_best.zero?
235
+
236
+ ratio = top_score / (top_score + second_best)
237
+ ((ratio * 0.8) + 0.2).clamp(0.0, 1.0) # Minimum confidence 0.2
238
+ end
239
+ end
240
+ end
241
+ end
242
+ end