kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,160 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Portuguese text.
7
+ #
8
+ # Ported from LanguageTool's PortugueseWordTokenizer.
9
+ #
10
+ # Handles:
11
+ # - Decimal comma between digits (3,14)
12
+ # - Dotted numbers (1.000.000)
13
+ # - Dates (01.01.2024, 2024-01-01)
14
+ # - Colons in time (12:25)
15
+ # - Hyphens with do-not-split list
16
+ # - Spaced decimals (2 000 000)
17
+ class PortugueseTokenizer < Base
18
+ # Portuguese word separators - most punctuation and whitespace
19
+ # Note: We protect special patterns before splitting
20
+ WORD_SEPARATORS = /[\s"()\[\]{}<>@€£\\$%‰‱ºªᵃᵒˢ|`~#^·]/
21
+
22
+ # Placeholder characters (using non-printing characters)
23
+ DECIMAL_COMMA_SUBST = "\uE001"
24
+ NON_BREAKING_SPACE_SUBST = "\uE002"
25
+ NON_BREAKING_DOT_SUBST = "\uE003"
26
+ NON_BREAKING_COLON_SUBST = "\uE004"
27
+
28
+ # Decimal comma between digits: 3,14
29
+ DECIMAL_COMMA_PATTERN = /(\d),(\d)/
30
+
31
+ # Dotted numbers: 1.000.000
32
+ DOTTED_NUMBERS_PATTERN = /(\d)\.(\d)/
33
+
34
+ # Colon in numbers (time): 12:25
35
+ COLON_NUMBERS_PATTERN = /(\d):(\d)/
36
+
37
+ # Date patterns: 01.01.2024, 2024-01-01
38
+ DATE_PATTERN = /(\d{2})\.(\d{2})\.(\d{4})|(\d{4})\.(\d{2})\.(\d{2})|(\d{4})-(\d{2})-(\d{2})/
39
+
40
+ # Spaced decimals: 2 000 000
41
+ SPACED_DECIMAL_PATTERN = /(?<=^|[\s(])\d{1,3}( \d{3})+(?:[,#{DECIMAL_COMMA_SUBST}#{NON_BREAKING_DOT_SUBST}]\d+)?(?=\D|$)/
42
+
43
+ # Do-not-split list (from LanguageTool)
44
+ DO_NOT_SPLIT = %w[
45
+ mers-cov mcgraw-hill sars-cov-2 sars-cov
46
+ ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
47
+ ].freeze
48
+
49
+ def tokenize(text)
50
+ return [] if text.nil? || text.strip.empty?
51
+
52
+ # Handle decimal commas
53
+ if text.include?(",")
54
+ text = text.gsub(DECIMAL_COMMA_PATTERN, "\\1#{DECIMAL_COMMA_SUBST}\\2")
55
+ end
56
+
57
+ # Handle dots in numbers and dates
58
+ if text.include?(".")
59
+ # Handle dates first (before dotted numbers to avoid conflicts)
60
+ text = text.gsub(DATE_PATTERN) do |match|
61
+ # match[0] is the full match, match[1-9] are the capture groups
62
+ if match[1] && match[2] && match[3] # DD.MM.YYYY
63
+ "#{match[1]}#{NON_BREAKING_DOT_SUBST}#{match[2]}#{NON_BREAKING_DOT_SUBST}#{match[3]}"
64
+ elsif match[4] && match[5] && match[6] # YYYY.MM.DD
65
+ "#{match[4]}#{NON_BREAKING_DOT_SUBST}#{match[5]}#{NON_BREAKING_DOT_SUBST}#{match[6]}"
66
+ elsif match[7] && match[8] && match[9] # YYYY-MM-DD (keep as-is)
67
+ match[0]
68
+ else
69
+ match[0]
70
+ end
71
+ end
72
+ text = text.gsub(DOTTED_NUMBERS_PATTERN, "\\1#{NON_BREAKING_DOT_SUBST}\\2")
73
+ end
74
+
75
+ # Handle spaced decimals: 2 000 000
76
+ text = handle_spaced_decimals(text)
77
+
78
+ # Handle colons in time: 12:25
79
+ if text.include?(":")
80
+ text = text.gsub(COLON_NUMBERS_PATTERN, "\\1#{NON_BREAKING_COLON_SUBST}\\2")
81
+ end
82
+
83
+ # Split on word boundaries
84
+ raw_tokens = text.split(WORD_SEPARATORS)
85
+
86
+ # Process each token
87
+ tokens = []
88
+ raw_tokens.each do |token|
89
+ next if token.empty?
90
+
91
+ # Restore placeholders
92
+ token = restore_placeholders(token)
93
+
94
+ # Handle hyphenated words
95
+ parts = words_to_add(token)
96
+ tokens.concat(parts)
97
+ end
98
+
99
+ # Filter and normalize
100
+ tokens
101
+ .map { |token| normalize(token) }
102
+ .reject { |token| skip_token?(token) }
103
+ end
104
+
105
+ protected
106
+
107
+ # Restore placeholders to original characters.
108
+ #
109
+ # @param token [String] Token with placeholders
110
+ # @return [String] Token with restored characters
111
+ def restore_placeholders(token)
112
+ token
113
+ .gsub(DECIMAL_COMMA_SUBST, ",")
114
+ .gsub(NON_BREAKING_COLON_SUBST, ":")
115
+ .gsub(NON_BREAKING_SPACE_SUBST, " ")
116
+ .gsub(NON_BREAKING_DOT_SUBST, ".")
117
+ end
118
+
119
+ # Split a word into tokens, handling hyphens.
120
+ #
121
+ # @param word [String] Word to split
122
+ # @return [Array<String>] Array of tokens
123
+ def words_to_add(word)
124
+ return [word] unless word.include?("-")
125
+
126
+ # Check do-not-split list
127
+ return [word] if DO_NOT_SPLIT.include?(word.downcase)
128
+
129
+ # For now, split on hyphens if not in do-not-split list
130
+ # Future: integrate with tagger for better handling
131
+ word.split("-", -1).flat_map do |part|
132
+ part.empty? ? ["-"] : [part]
133
+ end
134
+ end
135
+
136
+ def word_separators
137
+ WORD_SEPARATORS
138
+ end
139
+
140
+ private
141
+
142
+ # Handle spaced decimals: 2 000 000.
143
+ #
144
+ # @param text [String] Input text
145
+ # @return [String] Text with non-breaking spaces
146
+ def handle_spaced_decimals(text)
147
+ result = text
148
+ text.scan(SPACED_DECIMAL_PATTERN) do
149
+ match = Regexp.last_match
150
+ split_number = match[0]
151
+ split_number_adjusted = split_number.gsub(" ", NON_BREAKING_SPACE_SUBST)
152
+ split_number_adjusted = split_number_adjusted.gsub("\u00A0", NON_BREAKING_SPACE_SUBST)
153
+ result = result.sub(split_number, split_number_adjusted)
154
+ end
155
+ result
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Russian text.
7
+ #
8
+ # Ported from LanguageTool's RussianWordTokenizer.
9
+ #
10
+ # Handles:
11
+ # - Apostrophe as word character
12
+ # - Dot as word character (for abbreviations)
13
+ # - Special abbreviations: б/у (second-hand), б/н (new)
14
+ # - Spaced dots: .. , .
15
+ class RussianTokenizer < Base
16
+ # Russian-specific word separators (exclude apostrophe and dot)
17
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,;:!?\\\/|`~@#$%^&*+\-·]/
18
+
19
+ # Special abbreviations that should not be split
20
+ # Using non-printing characters as placeholders
21
+ ABBREVIATION_PLACEHOLDERS = {
22
+ "б/у" => "\u0001\u0001SOCR_BU\u0001\u0001",
23
+ "б/н" => "\u0001\u0001SOCR_BN\u0001\u0001"
24
+ }.freeze
25
+
26
+ # Reverse placeholders for restoration
27
+ PLACEHOLDER_RESTORE = {
28
+ "\u0001\u0001SOCR_BU\u0001\u0001" => "б/у",
29
+ "\u0001\u0001SOCR_BN\u0001\u0001" => "б/н",
30
+ "\u0001\u0001SP_DDOT_SP\u0001\u0001" => " .. ",
31
+ "\u0001\u0001SP_DOT_SP\u0001\u0001" => " . ",
32
+ "\u0001\u0001SP_DOT\u0001\u0001" => "."
33
+ }.freeze
34
+
35
+ def tokenize(text)
36
+ return [] if text.nil? || text.strip.empty?
37
+
38
+ # Replace abbreviations with placeholders
39
+ text = replace_abbreviations(text)
40
+
41
+ # Split on word boundaries
42
+ raw_tokens = text.split(WORD_SEPARATORS)
43
+
44
+ # Restore abbreviations and filter
45
+ raw_tokens
46
+ .map { |token| restore_abbreviations(token) }
47
+ .map { |token| normalize(token) }
48
+ .reject { |token| skip_token?(token) }
49
+ end
50
+
51
+ protected
52
+
53
+ def word_separators
54
+ WORD_SEPARATORS
55
+ end
56
+
57
+ private
58
+
59
+ # Replace special abbreviations with placeholders.
60
+ #
61
+ # @param text [String] Input text
62
+ # @return [String] Text with placeholders
63
+ def replace_abbreviations(text)
64
+ result = text
65
+ ABBREVIATION_PLACEHOLDERS.each do |abbr, placeholder|
66
+ result = result.gsub(abbr, placeholder)
67
+ end
68
+
69
+ # Handle spaced dots
70
+ result = result.gsub(" .. ", "\u0001\u0001SP_DDOT_SP\u0001\u0001")
71
+ result = result.gsub(" . ", "\u0001\u0001SP_DOT_SP\u0001\u0001")
72
+ result = result.gsub(" .", " \u0001\u0001SP_DOT\u0001\u0001")
73
+
74
+ # Restore spaced dots first, then single dot pattern
75
+ result = result.gsub("\u0001\u0001SP_DDOT_SP\u0001\u0001", " .. ")
76
+ result.gsub("\u0001\u0001SP_DOT_SP\u0001\u0001", " . ")
77
+ end
78
+
79
+ # Restore abbreviations from placeholders.
80
+ #
81
+ # @param text [String] Text with placeholders
82
+ # @return [String] Text with restored abbreviations
83
+ def restore_abbreviations(text)
84
+ result = text
85
+ PLACEHOLDER_RESTORE.each do |placeholder, original|
86
+ result = result.gsub(placeholder, original)
87
+ end
88
+ result
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Regex-parameterized tokenizer for scripts without a dedicated
7
+ # tokenizer class (plan 107 basic tier).
8
+ #
9
+ # Languages without a gem module still need the plan-91 contract
10
+ # — #spellcheck_word_regex deciding which letters form words —
11
+ # for their script. Latin, Cyrillic and Greek reuse their real
12
+ # tokenizer classes; every other script (Arabic, Hebrew,
13
+ # Armenian, Georgian, Devanagari, Hangul) is served by this
14
+ # generic, word-run tokenizer parameterized with the script's
15
+ # character class. Modules remain the upgrade path: when one
16
+ # lands it brings tokenizer care (contractions, jamo, conjuncts)
17
+ # and this fallback stops being consulted for that language.
18
+ class ScriptTokenizer < Base
19
+ # @param word_regex [Regexp] characters forming a word, as a
20
+ # single-character regex (the plan-91 set)
21
+ def initialize(word_regex:)
22
+ super()
23
+ @word_regex = word_regex
24
+ end
25
+
26
+ # Tokenize text into words by scanning runs of word characters.
27
+ #
28
+ # @param text [String] Text to tokenize
29
+ # @return [Array<String>] Array of tokens
30
+ def tokenize(text)
31
+ return [] if text.nil? || text.empty?
32
+
33
+ text.scan(/#{@word_regex}+/).reject { |token| skip_token?(token) }
34
+ end
35
+
36
+ # Get word boundary regex.
37
+ #
38
+ # @return [Regexp] Word boundary regex
39
+ def word_boundary_regex
40
+ @word_regex
41
+ end
42
+
43
+ # Spell-check word characters (plan 91): the script set this
44
+ # tokenizer was built with.
45
+ #
46
+ # @return [Regexp] Regex matching a single word character
47
+ def spellcheck_word_regex
48
+ @word_regex
49
+ end
50
+
51
+ protected
52
+
53
+ # Get word characters.
54
+ #
55
+ # @return [String] Character class source
56
+ def word_chars
57
+ @word_regex.source
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Spanish text.
7
+ #
8
+ # Ported from LanguageTool's SpanishWordTokenizer.
9
+ #
10
+ # Handles:
11
+ # - Decimal point between digits (3.14)
12
+ # - Decimal comma between digits (3,14)
13
+ # - Ordinals (1.º, 2.ª, 1.er, 1.os, 1.as)
14
+ # - Hyphens (with do-not-split list since no tagger)
15
+ # - Soft hyphens
16
+ # - Inverted punctuation (¡, ¿)
17
+ class SpanishTokenizer < Base
18
+ # Spanish word separators - most punctuation and whitespace
19
+ # Note: We need to handle decimals specially, so we protect them first
20
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
21
+
22
+ # Decimal point between digits: 3.14
23
+ DECIMAL_POINT = /(\d)\.(\d)/
24
+
25
+ # Decimal comma between digits: 3,14
26
+ DECIMAL_COMMA = /(\d),(\d)/
27
+
28
+ # Ordinal patterns: 1.º, 2.ª, 1.er, 1.os, 1.as
29
+ ORDINAL = /\b(\d+)\.(º|ª|o|a|er|os|as)\b/
30
+
31
+ # Placeholders for special patterns
32
+ DECIMAL_POINT_PLACEHOLDER = "\uE101"
33
+ DECIMAL_COMMA_PLACEHOLDER = "\uE102"
34
+ ORDINAL_PLACEHOLDER = "\uE103"
35
+
36
+ # Soft hyphen
37
+ SOFT_HYPHEN = "\u00AD"
38
+
39
+ # Do-not-split list (from LanguageTool)
40
+ DO_NOT_SPLIT = %w[
41
+ mers-cov mcgraw-hill sars-cov-2 sars-cov
42
+ ph-metre ph-metres
43
+ ].freeze
44
+
45
+ def tokenize(text)
46
+ return [] if text.nil? || text.strip.empty?
47
+
48
+ # Replace hyphen variants
49
+ text = text.gsub("\u2010", "\u002d") # hyphen to hyphen-minus
50
+ text = text.gsub("\u2011", "\u002d") # non-breaking hyphen to hyphen-minus
51
+
52
+ # Protect decimal points
53
+ text = text.gsub(DECIMAL_POINT, "\\1#{DECIMAL_POINT_PLACEHOLDER}\\2")
54
+
55
+ # Protect decimal commas
56
+ text = text.gsub(DECIMAL_COMMA, "\\1#{DECIMAL_COMMA_PLACEHOLDER}\\2")
57
+
58
+ # Protect ordinals
59
+ text = text.gsub(ORDINAL, "\\1#{ORDINAL_PLACEHOLDER}\\2")
60
+
61
+ # Split on word boundaries
62
+ raw_tokens = text.split(WORD_SEPARATORS)
63
+
64
+ # Process each token
65
+ tokens = []
66
+ raw_tokens.each do |token|
67
+ next if token.empty?
68
+
69
+ # Restore placeholders
70
+ token = restore_placeholders(token)
71
+
72
+ # Handle hyphenated words
73
+ parts = words_to_add(token)
74
+ tokens.concat(parts)
75
+ end
76
+
77
+ # Filter and normalize
78
+ tokens
79
+ .map { |token| normalize(token) }
80
+ .reject { |token| skip_token?(token) }
81
+ end
82
+
83
+ protected
84
+
85
+ # Restore placeholders to original characters.
86
+ #
87
+ # @param token [String] Token with placeholders
88
+ # @return [String] Token with restored characters
89
+ def restore_placeholders(token)
90
+ token
91
+ .gsub(DECIMAL_POINT_PLACEHOLDER, ".")
92
+ .gsub(DECIMAL_COMMA_PLACEHOLDER, ",")
93
+ .gsub(ORDINAL_PLACEHOLDER, ".")
94
+ end
95
+
96
+ # Split a word into tokens, handling hyphens.
97
+ #
98
+ # @param word [String] Word to split
99
+ # @return [Array<String>] Array of tokens
100
+ def words_to_add(word)
101
+ return [word] unless word.include?("-")
102
+
103
+ # Check do-not-split list
104
+ return [word] if DO_NOT_SPLIT.include?(word.downcase)
105
+
106
+ # Remove soft hyphens and check
107
+ normalized = word.gsub(SOFT_HYPHEN, "").gsub("'", "'")
108
+
109
+ # For now, split on hyphens if not in do-not-split list
110
+ # Future: integrate with tagger for better handling
111
+ normalized.split("-", -1).flat_map do |part|
112
+ part.empty? ? ["-"] : [part]
113
+ end
114
+ end
115
+
116
+ def word_separators
117
+ WORD_SEPARATORS
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Language module for multi-language support.
5
+ #
6
+ # Provides language detection, tokenization, and normalization
7
+ # for different languages with proper OOP design.
8
+ #
9
+ # @example Detect language
10
+ # Kotoshu::Language.detect("Hello world") # => "en"
11
+ #
12
+ # @example Get language class
13
+ # lang_class = Kotoshu::Language.get("en-US")
14
+ #
15
+ # @example List supported languages
16
+ # Kotoshu::Language.supported_codes # => ["de-DE", "en-US", ...]
17
+ module Language
18
+ autoload :Registry, "kotoshu/language/registry"
19
+ autoload :Detector, "kotoshu/language/detector"
20
+ # Detection value + the native LID reader behind it (plan 106) share
21
+ # lid_detector.rb.
22
+ autoload :Detection, "kotoshu/language/lid_detector"
23
+ autoload :LanguageIdentifier, "kotoshu/language/identifier"
24
+ autoload :LidDetector, "kotoshu/language/lid_detector"
25
+ autoload :Segmenter, "kotoshu/language/segmenter"
26
+ autoload :Segment, "kotoshu/language/segmenter"
27
+ autoload :Suika, "kotoshu/language/suika"
28
+ # Script classification for module-less languages (plan 107).
29
+ autoload :Script, "kotoshu/language/script"
30
+
31
+ module Tokenizer
32
+ autoload :Base, "kotoshu/language/tokenizer/base"
33
+ autoload :LatinTokenizer, "kotoshu/language/tokenizer/latin_tokenizer"
34
+ autoload :FrenchTokenizer, "kotoshu/language/tokenizer/french_tokenizer"
35
+ autoload :GermanTokenizer, "kotoshu/language/tokenizer/german_tokenizer"
36
+ autoload :SpanishTokenizer, "kotoshu/language/tokenizer/spanish_tokenizer"
37
+ autoload :PortugueseTokenizer, "kotoshu/language/tokenizer/portuguese_tokenizer"
38
+ autoload :RussianTokenizer, "kotoshu/language/tokenizer/russian_tokenizer"
39
+ autoload :JapaneseTokenizer, "kotoshu/language/tokenizer/japanese_tokenizer"
40
+ autoload :GreekTokenizer, "kotoshu/language/tokenizer/greek_tokenizer"
41
+ autoload :CyrillicTokenizer, "kotoshu/language/tokenizer/cyrillic_tokenizer"
42
+ autoload :ScriptTokenizer, "kotoshu/language/tokenizer/script_tokenizer"
43
+ autoload :HangulTokenizer, "kotoshu/language/tokenizer/hangul_tokenizer"
44
+ autoload :DevanagariTokenizer, "kotoshu/language/tokenizer/devanagari_tokenizer"
45
+ end
46
+
47
+ module Normalizer
48
+ autoload :Base, "kotoshu/language/normalizer/base"
49
+ autoload :Arabic, "kotoshu/language/normalizer/arabic"
50
+ autoload :Hebrew, "kotoshu/language/normalizer/hebrew"
51
+ autoload :Persian, "kotoshu/language/normalizer/persian"
52
+ autoload :Greek, "kotoshu/language/normalizer/greek"
53
+ autoload :Turkish, "kotoshu/language/normalizer/turkish"
54
+ end
55
+
56
+ # Base class for per-language implementations (Kotoshu::Languages::*).
57
+ # File path is historical (languages/base.rb under language/).
58
+ autoload :Base, "kotoshu/language/languages/base"
59
+
60
+ class << self
61
+ # Detect language from text.
62
+ #
63
+ # Delegates to Detector.
64
+ #
65
+ # @param text [String] Text to analyze
66
+ # @return [String, nil] Detected language code
67
+ def detect(text)
68
+ Detector.detect(text)
69
+ end
70
+
71
+ # Detect with confidence score.
72
+ #
73
+ # @param text [String] Text to analyze
74
+ # @return [Array<String, Float>] Language code and confidence
75
+ def detect_with_confidence(text)
76
+ Detector.detect_with_confidence(text)
77
+ end
78
+
79
+ # Get language class by code.
80
+ #
81
+ # Delegates to Registry.
82
+ #
83
+ # @param code [String] Language code
84
+ # @return [Class, nil] Language class or nil
85
+ def get(code)
86
+ Registry.get(code)
87
+ end
88
+
89
+ # Check if language is registered.
90
+ #
91
+ # @param code [String] Language code
92
+ # @return [Boolean] True if registered
93
+ def registered?(code)
94
+ Registry.registered?(code)
95
+ end
96
+
97
+ # Get all supported language codes.
98
+ #
99
+ # @return [Array<String>] List of codes
100
+ def supported_codes
101
+ Registry.supported_codes
102
+ end
103
+
104
+ # Get language info.
105
+ #
106
+ # @param code [String] Language code
107
+ # @return [Hash, nil] Language info or nil
108
+ def info(code)
109
+ Registry.info(code)
110
+ end
111
+
112
+ # Register a language.
113
+ #
114
+ # @param code [String] Language code
115
+ # @param klass [Class] Language class
116
+ # @return [void]
117
+ def register(code, klass)
118
+ Registry.register(code, klass)
119
+ end
120
+ end
121
+ end
122
+ end
123
+
124
+ # Register the default detector with the registry.
125
+ # Both Registry and Detector are autoloaded above; referencing them
126
+ # here triggers their loads.
127
+ Kotoshu::Language::Registry.register_detector(Kotoshu::Language::Detector)
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Arabic language implementation.
6
+ #
7
+ # Registers the Arabic language with {Language::Registry} under the
8
+ # +ar+ and +ar-SA+ codes. Uses the {Language::Normalizer::Arabic}
9
+ # normalizer (NFC + presentation-form canonicalization + tashkeel
10
+ # stripping) so that the same word typed in different input
11
+ # methods produces the same normalized form for dictionary lookup.
12
+ #
13
+ # @example
14
+ # lang = Kotoshu::Languages::Arabic.new
15
+ # lang.script_type # => :arabic
16
+ # lang.rtl? # => true
17
+ #
18
+ # Full-feature since plan 100 (batch 3): the spelling dictionary
19
+ # downloads from the dictionaries repo (AVAILABLE_LANGUAGES) and
20
+ # keyboard proximity uses the Arabic 101 grid
21
+ # (Keyboard::Layouts::Arabic101).
22
+ class Arabic < Language::Base
23
+ register "ar"
24
+ register "ar-SA"
25
+
26
+ def initialize(code: "ar", name: "Arabic", variant: nil)
27
+ super
28
+ end
29
+
30
+ def description
31
+ name
32
+ end
33
+
34
+ def tokenizer
35
+ @tokenizer ||= Tokenizer.new
36
+ end
37
+
38
+ def normalizer
39
+ @normalizer ||= Language::Normalizer::Arabic.new
40
+ end
41
+
42
+ def dictionary_class
43
+ Dictionary::Custom
44
+ end
45
+
46
+ def default_dictionary_paths
47
+ ["/usr/share/dict/arabic"]
48
+ end
49
+
50
+ def script_type
51
+ :arabic
52
+ end
53
+
54
+ # Arabic is written right-to-left.
55
+ def rtl?
56
+ true
57
+ end
58
+
59
+ # Minimal Arabic tokenizer.
60
+ #
61
+ # Splits on whitespace and standard punctuation — Arabic text
62
+ # is space-delimited. Uses Base#skip_token? which correctly
63
+ # handles Unicode letters via \p{L}, so Arabic characters are
64
+ # kept while pure numbers and single non-letter chars are
65
+ # filtered.
66
+ class Tokenizer < Language::Tokenizer::Base
67
+ ARABIC_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·،؛؟]/
68
+
69
+ def tokenize(text)
70
+ return [] if text.nil? || text.strip.empty?
71
+
72
+ text.split(ARABIC_SEPARATORS)
73
+ .map { |token| normalize(token) }
74
+ .reject { |token| skip_token?(token) }
75
+ end
76
+
77
+ # Spell-check word characters (plan 91 pattern, added by
78
+ # plan 100): Arabic-script letters. Arabic needs no ZWNJ (the
79
+ # staged dictionary carries none) and no apostrophe.
80
+ #
81
+ # @return [Regexp] Regex matching a single word character
82
+ def spellcheck_word_regex
83
+ /\p{Arabic}/
84
+ end
85
+
86
+ protected
87
+
88
+ def normalize(token)
89
+ token.strip
90
+ end
91
+
92
+ def word_separators
93
+ ARABIC_SEPARATORS
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Bulgarian language implementation (plan 100, batch 3).
6
+ #
7
+ # Cyrillic composition like Ukrainian: the Cyrillic tokenizer
8
+ # keeps Bulgarian-script words, the base normalizer suffices —
9
+ # Bulgarian case folding has no special pairs. The Bulgarian-only
10
+ # letter ъ (and ь) are real keys on the Bulgarian BDS grid
11
+ # (Keyboard::Layouts::BulgarianBds), not the Russian JCUKEN the
12
+ # pre-existing Cyrillic layout models.
13
+ class Bulgarian < LatinBase
14
+ register "bg"
15
+ register "bg-BG"
16
+
17
+ def initialize(code: "bg", name: "Bulgarian")
18
+ super
19
+ end
20
+
21
+ # Cyrillic-script tokenizer.
22
+ #
23
+ # @return [Language::Tokenizer::CyrillicTokenizer]
24
+ def tokenizer
25
+ @tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
26
+ end
27
+
28
+ def script_type
29
+ :cyrillic
30
+ end
31
+
32
+ def default_dictionary_paths
33
+ ["/usr/share/dict/bulgarian"]
34
+ end
35
+ end
36
+ end
37
+ end