kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,376 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # Language identification using FastText LID model.
6
+ #
7
+ # Identifies the language of text using FastText's pretrained
8
+ # language identification model (lid.176.ftz).
9
+ #
10
+ # @example Detect language
11
+ # lid = LanguageIdentifier.new
12
+ # result = lid.detect("Hello world")
13
+ # result.language # => "en"
14
+ # result.confidence # => 0.95
15
+ #
16
+ # @example Detect from file
17
+ # results = lid.detect_from_file("document.txt", top_k: 3)
18
+ # results.map(&:language) # => ["en", "de", "fr"]
19
+ class LanguageIdentifier
20
+ # FastText LID model URL
21
+ MODEL_URL = 'https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz'
22
+
23
+ # Language code mapping (FastText LID → ISO 639-1)
24
+ LANGUAGE_MAPPING = {
25
+ # FastText uses format like "__label__en" for English
26
+ 'en' => 'en',
27
+ 'de' => 'de',
28
+ 'es' => 'es',
29
+ 'fr' => 'fr',
30
+ 'pt' => 'pt',
31
+ 'ru' => 'ru',
32
+ 'it' => 'it',
33
+ 'nl' => 'nl',
34
+ 'pl' => 'pl',
35
+ 'sv' => 'sv',
36
+ 'da' => 'da',
37
+ 'no' => 'no',
38
+ 'fi' => 'fi',
39
+ 'cs' => 'cs',
40
+ 'el' => 'el',
41
+ 'hu' => 'hu',
42
+ 'ro' => 'ro',
43
+ 'bg' => 'bg',
44
+ 'sk' => 'sk',
45
+ 'sl' => 'sl',
46
+ 'hr' => 'hr',
47
+ 'sr' => 'sr',
48
+ 'et' => 'et',
49
+ 'lv' => 'lv',
50
+ 'lt' => 'lt',
51
+ 'mt' => 'mt',
52
+ 'ga' => 'ga',
53
+ 'cy' => 'cy',
54
+ 'tr' => 'tr',
55
+ 'ar' => 'ar',
56
+ 'he' => 'he',
57
+ 'fa' => 'fa',
58
+ 'ur' => 'ur',
59
+ 'hi' => 'hi',
60
+ 'bn' => 'bn',
61
+ 'ta' => 'ta',
62
+ 'te' => 'te',
63
+ 'ml' => 'ml',
64
+ 'kn' => 'kn',
65
+ 'th' => 'th',
66
+ 'vi' => 'vi',
67
+ 'id' => 'id',
68
+ 'ms' => 'ms',
69
+ 'sw' => 'sw',
70
+ 'zh' => 'zh',
71
+ 'ja' => 'ja',
72
+ 'ko' => 'ko'
73
+ }.freeze
74
+
75
+ # Value object for detection result.
76
+ #
77
+ # @attr_reader [String] language ISO 639-1 language code
78
+ # @attr_reader [Float] confidence Confidence score (0.0 to 1.0)
79
+ # @attr_reader [String] label Raw FastText label
80
+ DetectionResult = Struct.new(:language, :confidence, :label, keyword_init: true) do
81
+ def to_s
82
+ "#{language} (#{(confidence * 100).round(1)}%)"
83
+ end
84
+ end
85
+
86
+ attr_reader :model_path, :loaded
87
+
88
+ # Create a new language identifier.
89
+ #
90
+ # @param model_path [String] Path to lid.176.ftz model
91
+ # @param auto_download [Boolean] Download model if not found
92
+ def initialize(model_path: nil, auto_download: true)
93
+ @model_path = model_path || default_model_path
94
+ @auto_download = auto_download
95
+ @loaded = false
96
+ end
97
+
98
+ # Detect language of text.
99
+ #
100
+ # @param text [String] Text to analyze
101
+ # @param top_k [Integer] Number of top results to return
102
+ # @return [Array<DetectionResult>] Detection results sorted by confidence
103
+ def detect(text, top_k: 1)
104
+ ensure_model_loaded
105
+
106
+ # Preprocess text
107
+ text = preprocess_text(text)
108
+
109
+ # Run detection
110
+ run_detection(text, top_k)
111
+ end
112
+
113
+ # Detect language from file.
114
+ #
115
+ # @param filepath [String] Path to file
116
+ # @param top_k [Integer] Number of top results
117
+ # @return [Array<DetectionResult>] Detection results
118
+ def detect_from_file(filepath, top_k: 1)
119
+ text = File.read(filepath, encoding: 'UTF-8')
120
+ detect(text, top_k: top_k)
121
+ end
122
+
123
+ # Get the most likely language.
124
+ #
125
+ # @param text [String] Text to analyze
126
+ # @return [DetectionResult, nil] Top detection result
127
+ def detect_primary(text)
128
+ detect(text, top_k: 1).first
129
+ end
130
+
131
+ # Check if model is downloaded.
132
+ #
133
+ # @return [Boolean] True if model file exists
134
+ def model_downloaded?
135
+ File.exist?(@model_path)
136
+ end
137
+
138
+ # Download the FastText LID model.
139
+ #
140
+ # @return [String] Path to downloaded model
141
+ def download_model
142
+ require 'net/http'
143
+ require 'uri'
144
+ require 'fileutils'
145
+
146
+ # Create directory
147
+ FileUtils.mkdir_p(File.dirname(@model_path))
148
+
149
+ puts "Downloading language identification model..."
150
+ puts " From: #{MODEL_URL}"
151
+ puts " To: #{@model_path}"
152
+
153
+ uri = URI.parse(MODEL_URL)
154
+ http = Net::HTTP.new(uri.host, uri.port)
155
+ http.use_ssl = true
156
+
157
+ request = Net::HTTP::Get.new(uri.request_uri)
158
+
159
+ http.request(request) do |response|
160
+ case response
161
+ when Net::HTTPSuccess
162
+ File.open(@model_path, 'wb') do |file|
163
+ response.read_body do |chunk|
164
+ file.write(chunk)
165
+ end
166
+ end
167
+ puts " ✓ Download complete"
168
+ when Net::HTTPRedirection
169
+ # Follow redirect
170
+ follow_redirect(response['location'])
171
+ else
172
+ raise "Failed to download model: #{response.code} #{response.message}"
173
+ end
174
+ end
175
+
176
+ @model_path
177
+ end
178
+
179
+ # Get supported languages.
180
+ #
181
+ # @return [Array<String>] List of supported ISO 639-1 codes
182
+ def self.supported_languages
183
+ LANGUAGE_MAPPING.keys
184
+ end
185
+
186
+ private
187
+
188
+ # Get default model path.
189
+ #
190
+ # @return [String] Default path for lid.176.ftz
191
+ def default_model_path
192
+ File.join(Kotoshu::Paths.cache_path, 'models', 'lid.176.ftz')
193
+ end
194
+
195
+ # Ensure model is loaded.
196
+ def ensure_model_loaded
197
+ # Download if needed
198
+ if @auto_download && @auto_download && !model_downloaded?
199
+ download_model
200
+ end
201
+
202
+ raise "Model not found: #{@model_path}" unless model_downloaded?
203
+
204
+ # Load model (lazy)
205
+ return if @loaded
206
+
207
+ load_model
208
+ end
209
+
210
+ # Load the FastText model.
211
+ def load_model
212
+ # Try to use fasttext CLI
213
+ if fasttext_available?
214
+ @loaded = true
215
+ return
216
+ end
217
+
218
+ # Try to use Python fasttext library
219
+ if python_fasttext_available?
220
+ @loaded = true
221
+ return
222
+ end
223
+
224
+ raise "FastText not available. Install fasttext CLI or Python library"
225
+ end
226
+
227
+ # Check if fasttext CLI is available.
228
+ #
229
+ # @return [Boolean] True if fasttext command exists
230
+ def fasttext_available?
231
+ system('which', 'fasttext', out: File::NULL, err: File::NULL)
232
+ end
233
+
234
+ # Check if Python fasttext library is available.
235
+ #
236
+ # @return [Boolean] True if fasttext Python package is installed
237
+ def python_fasttext_available?
238
+ system('python3', '-c', 'import fasttext', out: File::NULL, err: File::NULL)
239
+ end
240
+
241
+ # Preprocess text for detection.
242
+ #
243
+ # @param text [String] Raw text
244
+ # @return [String] Preprocessed text
245
+ def preprocess_text(text)
246
+ # Remove leading/trailing whitespace
247
+ text = text.strip
248
+
249
+ # Take first N characters (FastText LID works best with 100-1000 chars)
250
+ # Taking first 500 characters as default
251
+ text = text[0..500] if text.length > 500
252
+
253
+ # Normalize whitespace
254
+ text.gsub(/\s+/, ' ')
255
+ end
256
+
257
+ # Run language detection.
258
+ #
259
+ # @param text [String] Preprocessed text
260
+ # @param top_k [Integer] Number of results
261
+ # @return [Array<DetectionResult>] Detection results
262
+ def run_detection(text, top_k)
263
+ # Create temp file with text
264
+ require 'tempfile'
265
+ Tempfile.create('lid_input_', encoding: 'UTF-8') do |f|
266
+ f.write(text)
267
+ f.flush
268
+
269
+ # Run fasttext command
270
+ if fasttext_available?
271
+ return run_fasttext_cli(f.path, top_k)
272
+ end
273
+
274
+ # Run Python fasttext
275
+ if python_fasttext_available?
276
+ return run_python_fasttext(f.path, top_k)
277
+ end
278
+ end
279
+ end
280
+
281
+ # Run detection using fasttext CLI.
282
+ #
283
+ # @param input_file [String] Path to input file
284
+ # @param top_k [Integer] Number of results
285
+ # @return [Array<DetectionResult>] Detection results
286
+ def run_fasttext_cli(input_file, top_k)
287
+ require 'open3'
288
+
289
+ cmd = [
290
+ 'fasttext',
291
+ 'predict',
292
+ @model_path,
293
+ input_file,
294
+ top_k.to_s
295
+ ]
296
+
297
+ output, = Open3.capture3(*cmd)
298
+
299
+ parse_fasttext_output(output)
300
+ end
301
+
302
+ # Run detection using Python fasttext.
303
+ #
304
+ # @param input_file [String] Path to input file
305
+ # @param top_k [Integer] Number of results
306
+ # @return [Array<DetectionResult>] Detection results
307
+ def run_python_fasttext(input_file, top_k)
308
+ require 'open3'
309
+
310
+ script = <<~PYTHON
311
+ import fasttext
312
+ model = fasttext.load_model('#{@model_path}')
313
+ with open('#{input_file}', 'r') as f:
314
+ text = f.read().strip()
315
+ labels, probs = model.predict(text, k=#{top_k})
316
+ for label, prob in zip(labels, probs):
317
+ print(f"{label} {prob}")
318
+ PYTHON
319
+
320
+ output, = Open3.capture3('python3', '-c', script)
321
+
322
+ parse_fasttext_output(output)
323
+ end
324
+
325
+ # Parse FastText output.
326
+ #
327
+ # @param output [String] Raw output from fasttext
328
+ # @return [Array<DetectionResult>] Parsed results
329
+ def parse_fasttext_output(output)
330
+ output.split("\n").map do |line|
331
+ next if line.empty?
332
+
333
+ # Parse: __label__en 0.95
334
+ parts = line.strip.split
335
+ next unless parts.size == 2
336
+
337
+ label = parts[0].sub('__label__', '')
338
+ confidence = parts[1].to_f
339
+
340
+ # Map to ISO 639-1
341
+ language = LANGUAGE_MAPPING[label] || label
342
+
343
+ DetectionResult.new(
344
+ language: language,
345
+ confidence: confidence,
346
+ label: label
347
+ )
348
+ end.compact.sort_by { |r| -r.confidence }
349
+ end
350
+
351
+ # Follow HTTP redirect.
352
+ #
353
+ # @param url [String] Redirect URL
354
+ def follow_redirect(url)
355
+ uri = URI.parse(url)
356
+ http = Net::HTTP.new(uri.host, uri.port)
357
+ http.use_ssl = true if uri.scheme == 'https'
358
+
359
+ request = Net::HTTP::Get.new(uri.request_uri)
360
+
361
+ http.request(request) do |response|
362
+ case response
363
+ when Net::HTTPSuccess
364
+ File.open(@model_path, 'wb') do |file|
365
+ response.read_body do |chunk|
366
+ file.write(chunk)
367
+ end
368
+ end
369
+ when Net::HTTPRedirection
370
+ follow_redirect(response['location'])
371
+ end
372
+ end
373
+ end
374
+ end
375
+ end
376
+ end
@@ -0,0 +1,271 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # Abstract base class for language implementations.
6
+ #
7
+ # Uses Template Method pattern to define the interface that all
8
+ # language implementations must follow.
9
+ #
10
+ # Each language implementation should:
11
+ # 1. Inherit from this class
12
+ # 2. Implement the required template methods
13
+ # 3. Register itself with Language::Registry
14
+ #
15
+ # @example Implement a language
16
+ # class English < Kotoshu::Language::Base
17
+ # register "en"
18
+ #
19
+ # def initialize
20
+ # super(code: "en", name: "English")
21
+ # end
22
+ #
23
+ # def tokenizer
24
+ # @tokenizer ||= Tokenizer::LatinTokenizer.new
25
+ # end
26
+ #
27
+ # def normalizer
28
+ # @normalizer ||= Normalizer::Base.new
29
+ # end
30
+ #
31
+ # def dictionary_class
32
+ # Dictionary::UnixWords
33
+ # end
34
+ # end
35
+ class Base
36
+ attr_reader :code, :name, :variant, :region
37
+
38
+ # Initialize language.
39
+ #
40
+ # @param code [String] Language code (e.g., "en", "en-US", "de-DE")
41
+ # @param name [String] Human-readable name
42
+ # @param variant [String, nil] Variant name (e.g., "American", "British")
43
+ def initialize(code:, name:, variant: nil)
44
+ @code = code
45
+ @name = name
46
+ @variant = variant
47
+ @region = extract_region(code)
48
+ end
49
+
50
+ # Get tokenizer for this language.
51
+ #
52
+ # Subclasses must implement.
53
+ #
54
+ # @return [Tokenizer::Base] Language-specific tokenizer
55
+ # @raise [NotImplementedError] If not implemented
56
+ def tokenizer
57
+ raise NotImplementedError, "#{self.class} must implement #tokenizer"
58
+ end
59
+
60
+ # Get normalizer for this language.
61
+ #
62
+ # Subclasses must implement.
63
+ #
64
+ # @return [Normalizer::Base] Language-specific normalizer
65
+ # @raise [NotImplementedError] If not implemented
66
+ def normalizer
67
+ raise NotImplementedError, "#{self.class} must implement #normalizer"
68
+ end
69
+
70
+ # Get dictionary class for this language.
71
+ #
72
+ # Subclasses must implement.
73
+ #
74
+ # @return [Class] Dictionary backend class
75
+ # @raise [NotImplementedError] If not implemented
76
+ def dictionary_class
77
+ raise NotImplementedError, "#{self.class} must implement #dictionary_class"
78
+ end
79
+
80
+ # Get default dictionary paths for this language.
81
+ #
82
+ # Subclasses can override to provide language-specific paths.
83
+ #
84
+ # @return [Array<String>] List of dictionary paths to search
85
+ def default_dictionary_paths
86
+ []
87
+ end
88
+
89
+ # Get character encoding for this language.
90
+ #
91
+ # Default is UTF-8 for all languages.
92
+ #
93
+ # @return [String] Character encoding name
94
+ def encoding
95
+ "UTF-8"
96
+ end
97
+
98
+ # Check if language uses right-to-left script.
99
+ #
100
+ # Default is false. Override for Arabic, Hebrew, etc.
101
+ #
102
+ # @return [Boolean] True if RTL
103
+ def rtl?
104
+ false
105
+ end
106
+
107
+ # Get script type for this language.
108
+ #
109
+ # Possible values: :latin, :cyrillic, :arabic, :cjk, :mixed
110
+ #
111
+ # @return [Symbol] Script type
112
+ def script_type
113
+ :latin
114
+ end
115
+
116
+ # Tokenize text using language-specific tokenizer.
117
+ #
118
+ # @param text [String] Text to tokenize
119
+ # @return [Array<String>] Array of tokens
120
+ def tokenize(text)
121
+ tokenizer.tokenize(text)
122
+ end
123
+
124
+ # Normalize text using language-specific normalizer.
125
+ #
126
+ # @param text [String] Text to normalize
127
+ # @param options [Hash] Normalization options
128
+ # @return [String] Normalized text
129
+ def normalize(text, options = {})
130
+ normalizer.normalize(text, options)
131
+ end
132
+
133
+ # Check if a word is valid in this language.
134
+ #
135
+ # Uses dictionary lookup.
136
+ #
137
+ # @param word [String] Word to check
138
+ # @param dictionary [Dictionary::Base] Dictionary to use
139
+ # @return [Boolean] True if word is valid
140
+ def valid_word?(word, dictionary:)
141
+ normalized = normalize_word(word)
142
+ dictionary.lookup(normalized)
143
+ end
144
+
145
+ # Normalize a word for checking.
146
+ #
147
+ # @param word [String] Word to normalize
148
+ # @return [String] Normalized word
149
+ def normalize_word(word)
150
+ normalizer.normalize_word(word)
151
+ end
152
+
153
+ # Get language info hash.
154
+ #
155
+ # @return [Hash] Language information
156
+ def info
157
+ {
158
+ code: code,
159
+ name: name,
160
+ variant: variant,
161
+ region: region,
162
+ encoding: encoding,
163
+ rtl?: rtl?,
164
+ script_type: script_type,
165
+ dictionary_class: dictionary_class.name
166
+ }
167
+ end
168
+
169
+ # Check if this language matches given code.
170
+ #
171
+ # Supports base language matching (e.g., "en" matches "en-US").
172
+ #
173
+ # @param other_code [String] Code to compare
174
+ # @return [Boolean] True if matches
175
+ def matches_code?(other_code)
176
+ return false if other_code.nil?
177
+
178
+ code == other_code ||
179
+ code.split("-").first == other_code.split("-").first
180
+ end
181
+
182
+ # Get full language name with variant.
183
+ #
184
+ # @return [String] Full name
185
+ def full_name
186
+ return name unless variant
187
+
188
+ "#{name} (#{variant})"
189
+ end
190
+
191
+ # Check if this is a base language (no region).
192
+ #
193
+ # @return [Boolean] True if base language
194
+ def base_language?
195
+ !code.include?("-")
196
+ end
197
+
198
+ # Get base language code.
199
+ #
200
+ # @return [String] Base language code (e.g., "en" from "en-US")
201
+ def base_code
202
+ code.split("-").first
203
+ end
204
+
205
+ # Get region code.
206
+ #
207
+ # @return [String, nil] Region code or nil
208
+ def region_code
209
+ return nil unless code.include?("-")
210
+
211
+ code.split("-", 2).last
212
+ end
213
+
214
+ # Check if another language is compatible.
215
+ #
216
+ # Languages are compatible if they share the same base code.
217
+ #
218
+ # @param other [Base] Other language
219
+ # @return [Boolean] True if compatible
220
+ def compatible_with?(other)
221
+ return false unless other.is_a?(Base)
222
+
223
+ base_code == other.base_code
224
+ end
225
+
226
+ class << self
227
+ # Register this language with the registry.
228
+ #
229
+ # Records +code+ in {registered_codes} so the registry can be
230
+ # rebuilt after a {Kotoshu::Language::Registry.clear} without
231
+ # re-loading the per-language file (which Ruby autoload only
232
+ # runs once). See {Language::Registry.restore_autoload!}.
233
+ #
234
+ # @param code [String] Language code
235
+ # @return [void]
236
+ def register(code)
237
+ registered_codes << code
238
+ Kotoshu::Language::Registry.register(code, self)
239
+ end
240
+
241
+ # Per-language class method reader for the codes registered via
242
+ # {register}. Used by {Language::Registry.restore_autoload!} to
243
+ # rebuild the registry after a clear.
244
+ #
245
+ # @return [Array<String>]
246
+ def registered_codes
247
+ @registered_codes ||= []
248
+ end
249
+
250
+ # Get or create singleton instance.
251
+ #
252
+ # @return [Base] Language instance
253
+ def instance
254
+ @instance ||= new
255
+ end
256
+ end
257
+
258
+ private
259
+
260
+ # Extract region from language code.
261
+ #
262
+ # @param code [String] Language code
263
+ # @return [String, nil] Region or nil
264
+ def extract_region(code)
265
+ return nil unless code.include?("-")
266
+
267
+ code.split("-", 2).last.upcase
268
+ end
269
+ end
270
+ end
271
+ end