kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,5 @@
1
+ # Russian confusion sets
2
+ # See the English 'confusion_sets.txt' for a description of file format
3
+
4
+ не; ни; 1000000; # p=1.000, r=0.550, 116+191, 3grams, 2016-10-13
5
+ шасси; шоссе; 10000000; # p=1.000, r=0.125, 48+48, 3grams, 2016-10-12
@@ -0,0 +1,128 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+
5
+ module Kotoshu
6
+ module Language
7
+ # Script classification for languages without a module (plan 107).
8
+ #
9
+ # Language modules are the full-feature tier: they bring keyboard
10
+ # layouts, tokenizer care and verified specimens. Every language
11
+ # staged in the dictionaries manifest is still usable without
12
+ # one — this module maps a language code to its writing script
13
+ # and supplies the tokenizer the spell-check word extraction
14
+ # (plan 91) rides on:
15
+ #
16
+ # - Latin, Cyrillic and Greek reuse their real tokenizer classes;
17
+ # - other scripts get a generic {Tokenizer::ScriptTokenizer}
18
+ # parameterized with the script's Unicode set;
19
+ # - unknown codes return nil and the ASCII fallback applies.
20
+ #
21
+ # The map covers the languages of the dictionaries manifest plus
22
+ # their near neighbors, so a code is misclassified only when the
23
+ # manifest grows without this file — which the
24
+ # kotoshu:staged_languages:sync rake task surfaces.
25
+ module Script
26
+ # Language code -> script symbol for every manifest language.
27
+ # Latin is the default and therefore not listed.
28
+ SCRIPT_BY_LANGUAGE = {
29
+ # Cyrillic (module languages bg ru sr uk plus staged mk mn).
30
+ "bg" => :cyrillic, "ru" => :cyrillic, "sr" => :cyrillic,
31
+ "uk" => :cyrillic, "be" => :cyrillic,
32
+ "mk" => :cyrillic, "mn" => :cyrillic,
33
+ # Greek (el; the manifest also stages el-polyton).
34
+ "el" => :greek, "el-polyton" => :greek,
35
+ # Arabic script (ar fa; fa also needs ZWNJ, which its module
36
+ # tokenizer handles — this generic set is the module-less
37
+ # fallback only).
38
+ "ar" => :arabic, "fa" => :arabic, "ur" => :arabic, "ps" => :arabic,
39
+ # Hebrew script.
40
+ "he" => :hebrew, "yi" => :hebrew,
41
+ # Armenian (staged hy hyw).
42
+ "hy" => :armenian, "hyw" => :armenian,
43
+ # Georgian (staged ka).
44
+ "ka" => :georgian,
45
+ # Devanagari (staged ne; hi mr sa ride the same script).
46
+ "ne" => :devanagari, "hi" => :devanagari, "mr" => :devanagari,
47
+ "sa" => :devanagari,
48
+ # Hangul (staged ko).
49
+ "ko" => :hangul
50
+ }.freeze
51
+
52
+ # Word-character regex per script, for the generic tokenizer.
53
+ SCRIPT_WORD_REGEXES = {
54
+ arabic: /\p{Arabic}/,
55
+ hebrew: /\p{Hebrew}/,
56
+ armenian: /\p{Armenian}/,
57
+ georgian: /\p{Georgian}/
58
+ }.freeze
59
+
60
+ # Tokenizer classes reused from the dedicated script tokenizers
61
+ # (plan 108 added Hangul and Devanagari alongside the plan 84
62
+ # Cyrillic and Greek classes).
63
+ SCRIPT_TOKENIZER_CLASSES = {
64
+ cyrillic: Tokenizer::CyrillicTokenizer,
65
+ greek: Tokenizer::GreekTokenizer,
66
+ hangul: Tokenizer::HangulTokenizer,
67
+ devanagari: Tokenizer::DevanagariTokenizer
68
+ }.freeze
69
+
70
+ class << self
71
+ # The writing script of a language code (:latin default).
72
+ #
73
+ # @param code [String] Language code (e.g. "mk", "ko")
74
+ # @return [Symbol] Script name (:latin, :cyrillic, ...)
75
+ def script_for(code)
76
+ return :latin if code.nil? || code.empty?
77
+
78
+ SCRIPT_BY_LANGUAGE.fetch(normalized_base(code), :latin)
79
+ end
80
+
81
+ # The script-appropriate tokenizer for a language without a
82
+ # module, or nil when the language is unknown (the ASCII
83
+ # fallback then applies, exactly as before plan 107).
84
+ #
85
+ # Latin-script languages return the shared LatinTokenizer —
86
+ # the same tokenizer every LatinBase module uses — so the
87
+ # basic tier and the full-feature tier agree on word
88
+ # extraction until a module adds language-specific care.
89
+ #
90
+ # @param code [String] Language code
91
+ # @return [Tokenizer::Base, nil] Tokenizer or nil
92
+ def tokenizer_for(code)
93
+ return nil if code.nil? || code.empty?
94
+
95
+ script = script_for(code)
96
+ if script == :latin
97
+ latin_script_languages.include?(normalized_base(code)) ? Tokenizer::LatinTokenizer.new : nil
98
+ elsif SCRIPT_TOKENIZER_CLASSES.key?(script)
99
+ SCRIPT_TOKENIZER_CLASSES.fetch(script).new
100
+ elsif SCRIPT_WORD_REGEXES.key?(script)
101
+ Tokenizer::ScriptTokenizer.new(word_regex: SCRIPT_WORD_REGEXES.fetch(script))
102
+ end
103
+ end
104
+
105
+ private
106
+
107
+ # Downcase and strip the region ("sv-FI" -> "sv", "en-GB" ->
108
+ # "en"), mirroring ResourceManager's language normalization.
109
+ def normalized_base(code)
110
+ code.to_s.split("-").first.split("_").first.downcase
111
+ end
112
+
113
+ # Latin-script codes this module knows: the manifest's Latin
114
+ # languages. Unknown Latin-ish codes stay nil so new manifest
115
+ # entries do not silently widen word extraction beyond the
116
+ # vendored list (the sync rake task refreshes it).
117
+ def latin_script_languages
118
+ @latin_script_languages ||= begin
119
+ classified = SCRIPT_BY_LANGUAGE.keys.map { |k| normalized_base(k) }.to_set
120
+ Cache::StagedLanguages::CODES.map { |c| normalized_base(c) }.uniq
121
+ .reject { |c| classified.include?(c) }
122
+ .to_set
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # Value object: a contiguous run of text nodes that all share a
6
+ # detected language. Produced by {Segmenter#segment}.
7
+ #
8
+ # Carries the detected +language_code+, the +text_nodes+ belonging
9
+ # to the segment (in source order), and the +confidence+ the
10
+ # detector returned for the segment's text. Multi-language
11
+ # checkers consume segments one at a time, resolving resources
12
+ # per-language.
13
+ Segment = Struct.new(:language_code, :text_nodes, :confidence, keyword_init: true) do
14
+ # Concatenated flattened text of every node in this segment.
15
+ #
16
+ # @return [String]
17
+ def flattened_text
18
+ text_nodes.map(&:text).join
19
+ end
20
+
21
+ # Number of text nodes in this segment.
22
+ #
23
+ # @return [Integer]
24
+ def length
25
+ text_nodes.length
26
+ end
27
+
28
+ # True when no text nodes are in the segment.
29
+ #
30
+ # @return [Boolean]
31
+ def empty?
32
+ text_nodes.nil? || text_nodes.empty?
33
+ end
34
+ end
35
+
36
+ # Splits a {Documents::Document} into language-tagged segments.
37
+ #
38
+ # Walks +document.text_nodes+, detects the dominant language for
39
+ # each node via the configured detector (default: {Detector}, the
40
+ # pure-Ruby character-set-based detector — no fasttext needed),
41
+ # and groups consecutive same-language nodes into {Segment}s.
42
+ #
43
+ # The detector is pluggable via the +detector:+ constructor
44
+ # argument. Any object that responds to +detect_with_confidence+
45
+ # and returns +[code, confidence]+ will do — the FastText-backed
46
+ # {LanguageIdentifier} is a drop-in for higher accuracy at the
47
+ # cost of a model download.
48
+ class Segmenter
49
+ DEFAULT_DETECTOR = Detector
50
+
51
+ # Minimum text length (chars) for a node to participate in
52
+ # detection. Shorter nodes ("a", "the", whitespace) don't carry
53
+ # enough signal — they inherit the surrounding language.
54
+ MIN_NODE_LENGTH_FOR_DETECTION = 3
55
+
56
+ # @param detector [Object, nil] responds to +detect_with_confidence+.
57
+ # Defaults to {DEFAULT_DETECTOR}.
58
+ # @param min_confidence [Float] detection results below this
59
+ # threshold are treated as unknown (segment language becomes
60
+ # the fallback language, default "en").
61
+ # @param fallback_language [String] language code to use when
62
+ # detection is inconclusive.
63
+ def initialize(detector: nil, min_confidence: 0.3, fallback_language: "en")
64
+ @detector = detector || DEFAULT_DETECTOR
65
+ @min_confidence = min_confidence
66
+ @fallback_language = fallback_language
67
+ end
68
+
69
+ # Segment a document.
70
+ #
71
+ # @param document [Kotoshu::Documents::Document]
72
+ # @return [Array<Segment>] language-tagged segments, in source order
73
+ def segment(document)
74
+ unless document.is_a?(Kotoshu::Documents::Document)
75
+ raise ArgumentError,
76
+ "document must be a Kotoshu::Documents::Document"
77
+ end
78
+
79
+ segments = []
80
+ current = nil
81
+
82
+ document.text_nodes.each do |node|
83
+ code, confidence = detect_for_node(node)
84
+ if current && current.language_code == code
85
+ current.text_nodes << node
86
+ else
87
+ segments << current if current && !current.empty?
88
+ current = Segment.new(language_code: code, text_nodes: [node],
89
+ confidence: confidence)
90
+ end
91
+ end
92
+ segments << current if current && !current.empty?
93
+ segments
94
+ end
95
+
96
+ private
97
+
98
+ # Detect language for a single node. Short nodes (< MIN length)
99
+ # inherit nil and the caller's group-merge logic picks up the
100
+ # previous segment's language (because nil != previous code).
101
+ # To avoid fragmenting segments on short connector words, we
102
+ # return the previous detection — but Ruby doesn't easily carry
103
+ # state into a `private` helper without ivars, so the caller
104
+ # handles short-node inheritance via the group-merge rule.
105
+ #
106
+ # For now: short nodes return the fallback language so they
107
+ # merge with whatever segment is open (assuming the previous
108
+ # detection was also the fallback). This is a heuristic; a
109
+ # richer segmenter would carry last-detected-code as state.
110
+ def detect_for_node(node)
111
+ text = node.text
112
+ return [@fallback_language, 0.0] if text.nil? || text.strip.empty?
113
+ return [@fallback_language, 0.0] if text.strip.length < MIN_NODE_LENGTH_FOR_DETECTION
114
+
115
+ code, confidence = @detector.detect_with_confidence(text)
116
+ return [@fallback_language, 0.0] if code.nil? || confidence < @min_confidence
117
+
118
+ [code, confidence]
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # Soft-load suika. The gem is intentionally NOT a hard runtime
6
+ # dependency — its native extension (dartsclone) fails to build on
7
+ # slim/minimal environments and would block install for users who
8
+ # only want non-Japanese spell-checking. Japanese tokenization and
9
+ # POS tagging light up automatically when the gem is present.
10
+ #
11
+ # KOTOSHU_NO_SUIKA=1 forces Japanese analysis off even when the gem
12
+ # is installed (useful for benchmarks / CI determinism).
13
+ module Suika
14
+ LOADED = begin
15
+ if ENV["KOTOSHU_NO_SUIKA"] == "1"
16
+ false
17
+ else
18
+ require "suika"
19
+ true
20
+ end
21
+ rescue LoadError
22
+ false
23
+ end
24
+
25
+ @tagger = nil
26
+
27
+ class << self
28
+ # Return a process-wide memoized Suika::Tagger, or raise
29
+ # {SuikaUnavailable} when the gem is missing.
30
+ def tagger
31
+ raise Kotoshu::SuikaUnavailable unless LOADED
32
+
33
+ @tagger ||= ::Suika::Tagger.new
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,187 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Abstract base class for tokenizers.
7
+ #
8
+ # Uses Strategy pattern to allow different tokenization approaches
9
+ # for different languages.
10
+ #
11
+ # Subclasses must implement the tokenize method.
12
+ #
13
+ # @example Implement a tokenizer
14
+ # class MyTokenizer < Tokenizer::Base
15
+ # def tokenize(text)
16
+ # text.split(/ /)
17
+ # end
18
+ # end
19
+ class Base
20
+ # Tokenize text into words.
21
+ #
22
+ # @param text [String] Text to tokenize
23
+ # @return [Array<String>] Array of tokens
24
+ # @raise [NotImplementedError] Must be implemented by subclass
25
+ def tokenize(text)
26
+ raise NotImplementedError, "#{self.class} must implement #tokenize"
27
+ end
28
+
29
+ # Tokenize text with positions.
30
+ #
31
+ # Returns tokens along with their position information.
32
+ #
33
+ # @param text [String] Text to tokenize
34
+ # @return [Array<Hash>] Array of {token:, start:, end:, line:, column:}
35
+ def tokenize_with_positions(text)
36
+ return [] if text.nil?
37
+ return [] if text.empty?
38
+
39
+ tokens = []
40
+ line = 1
41
+ column = 1
42
+ position = 0
43
+
44
+ while position < text.length
45
+ # Skip whitespace
46
+ while position < text.length && text[position].match?(/\s/)
47
+ if text[position] == "\n"
48
+ line += 1
49
+ column = 1
50
+ else
51
+ column += 1
52
+ end
53
+ position += 1
54
+ end
55
+
56
+ break if position >= text.length
57
+
58
+ # Find token
59
+ start_pos = position
60
+ start_line = line
61
+ start_column = column
62
+
63
+ token_text = extract_next_token(text, position)
64
+
65
+ if token_text
66
+ tokens << {
67
+ token: token_text,
68
+ start: start_pos,
69
+ end: start_pos + token_text.length,
70
+ line: start_line,
71
+ column: start_column
72
+ }
73
+
74
+ token_text.each_char do |char|
75
+ column += 1
76
+ position += 1
77
+ if char == "\n"
78
+ line += 1
79
+ column = 1
80
+ end
81
+ end
82
+ else
83
+ position += 1
84
+ column += 1
85
+ end
86
+ end
87
+
88
+ tokens
89
+ end
90
+
91
+ # Check if a character is a word character.
92
+ #
93
+ # @param char [String] Single character
94
+ # @return [Boolean] True if word character
95
+ def word_char?(char)
96
+ match?(word_boundary_regex, char)
97
+ end
98
+
99
+ # Get word boundary regex for this tokenizer.
100
+ #
101
+ # Subclasses should override this to define word boundaries.
102
+ #
103
+ # @return [Regexp] Word boundary regex
104
+ def word_boundary_regex
105
+ raise NotImplementedError, "#{self.class} must implement #word_boundary_regex"
106
+ end
107
+
108
+ # Regex matching one word-forming character for spell-check text
109
+ # extraction (plan 91).
110
+ #
111
+ # The Spellchecker walks text character by character with this
112
+ # regex, so it must cover exactly the letters this tokenizer
113
+ # recognizes: script subclasses override it with their script's
114
+ # letters, and wrong-script characters stay non-word so foreign
115
+ # words are never checked. The Base default is the historical
116
+ # ASCII set, so tokenizers without an override keep their
117
+ # extraction behavior; digits are never included so number
118
+ # handling stays as it was.
119
+ #
120
+ # @return [Regexp] Regex matching a single word character
121
+ def spellcheck_word_regex
122
+ /[a-zA-Z']/
123
+ end
124
+
125
+ # Normalize a token.
126
+ #
127
+ # Subclasses can override this for language-specific normalization.
128
+ #
129
+ # @param token [String] Token to normalize
130
+ # @return [String] Normalized token
131
+ def normalize(token)
132
+ token
133
+ end
134
+
135
+ # Check if a token should be skipped.
136
+ #
137
+ # Subclasses can override this for language-specific filtering.
138
+ #
139
+ # @param token [String] Token to check
140
+ # @return [Boolean] True if token should be skipped
141
+ def skip_token?(token)
142
+ return true if token.empty?
143
+ return true if token.match?(/^\d+$/) # Pure numbers
144
+ return true if token.length < 2 && token.match?(/^[^\p{L}]$/)
145
+
146
+ false
147
+ end
148
+
149
+ protected
150
+
151
+ # Extract the next token from text at position.
152
+ #
153
+ # @param text [String] Full text
154
+ # @param position [Integer] Current position
155
+ # @return [String, nil] Next token or nil
156
+ def extract_next_token(text, position)
157
+ remaining = text[position..]
158
+ match = remaining.match(/^#{word_pattern}/)
159
+ match ? match[0] : nil
160
+ end
161
+
162
+ # Get pattern for matching tokens.
163
+ #
164
+ # @return [String] Regex pattern string
165
+ def word_pattern
166
+ "[#{word_chars}]+"
167
+ end
168
+
169
+ # Get word characters for this tokenizer.
170
+ #
171
+ # @return [String] Character class of word characters
172
+ def word_chars
173
+ raise NotImplementedError, "#{self.class} must implement #word_chars"
174
+ end
175
+
176
+ # Check if string matches regex.
177
+ #
178
+ # @param regex [Regexp] Regex to match
179
+ # @param string [String] String to check
180
+ # @return [Boolean] True if matches
181
+ def match?(regex, string)
182
+ regex.match?(string)
183
+ end
184
+ end
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Cyrillic-script text without a language-specific
7
+ # subclass (RussianTokenizer carries Russian abbreviation
8
+ # handling; Ukrainian, Belarusian, Bulgarian, Serbian and friends
9
+ # need only the script-aware split).
10
+ #
11
+ # Splits on the same punctuation/whitespace separators as the
12
+ # Latin tokenizer, but keeps tokens that contain Cyrillic
13
+ # letters — the Latin tokenizer's letter checks would drop a
14
+ # pure Cyrillic word. The apostrophe stays a word character so
15
+ # Ukrainian names like Мар'яна tokenize as one word.
16
+ class CyrillicTokenizer < Base
17
+ # Separators between Cyrillic words. The apostrophe is NOT a
18
+ # separator (Ukrainian uses it inside words); the dot is, like
19
+ # in the Latin tokenizer.
20
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
21
+
22
+ # Tokenize text into Cyrillic words.
23
+ #
24
+ # @param text [String] Text to tokenize
25
+ # @return [Array<String>] Array of tokens
26
+ def tokenize(text)
27
+ return [] if text.nil? || text.strip.empty?
28
+
29
+ text.split(WORD_SEPARATORS)
30
+ .map(&:strip)
31
+ .reject { |token| skip_token?(token) }
32
+ end
33
+
34
+ # Get word boundary regex.
35
+ #
36
+ # @return [Regexp] Word boundary regex
37
+ def word_boundary_regex
38
+ /\p{Cyrillic}/
39
+ end
40
+
41
+ # Spell-check word characters (plan 91): the Cyrillic letters
42
+ # and apostrophe this tokenizer already declares in
43
+ # #word_chars (Ukrainian names like Мар'яна keep the
44
+ # apostrophe in-word).
45
+ #
46
+ # @return [Regexp] Regex matching a single word character
47
+ def spellcheck_word_regex
48
+ /[#{word_chars}]/o
49
+ end
50
+
51
+ # Check if token should be skipped.
52
+ #
53
+ # Skips tokens that contain no Cyrillic letters; Base's rules
54
+ # (empty, pure numbers) also apply.
55
+ #
56
+ # @param token [String] Token to check
57
+ # @return [Boolean] True if should skip
58
+ def skip_token?(token)
59
+ return true if super
60
+
61
+ !token.match?(/\p{Cyrillic}/)
62
+ end
63
+
64
+ protected
65
+
66
+ # Get word characters.
67
+ #
68
+ # @return [String] Character class of Cyrillic letters and the
69
+ # apostrophe
70
+ def word_chars
71
+ "\\p{Cyrillic}'"
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Devanagari-script languages (plan 108: ne).
7
+ #
8
+ # A Devanagari word is a run of block characters in which the
9
+ # combining marks stay attached to their base: inherent-vowel
10
+ # killers and matras (U+093E-094C), the virama (U+094D) that
11
+ # forms conjunct consonants, anusvara/visarga/candrabindu and
12
+ # the vowel signs — all share the Devanagari script property, so
13
+ # one script run IS one grapheme-cluster sequence and the
14
+ # plan-91 per-character regex machinery keeps each cluster
15
+ # intact (क + ि renders as कि and extracts as one token).
16
+ # ZWJ/ZWNJ (U+200C/U+200D) join the word set because conjunct
17
+ # spellings insert them mid-word; the danda marks (U+0964-0965)
18
+ # are sentence punctuation and the Devanagari digits
19
+ # (U+0966-096F) are digits, so both stay separators — plan 91
20
+ # keeps digits out of words in every script.
21
+ class DevanagariTokenizer < Base
22
+ # Devanagari block minus danda/digits/abbreviation sign, plus
23
+ # ZWJ/ZWNJ (regex source text, like CyrillicTokenizer's
24
+ # word_chars).
25
+ DEVANAGARI_WORD_CHARS = "\\u0900-\\u0963\\u0971-\\u097F\\u200C\\u200D"
26
+
27
+ # One word (a run of word characters).
28
+ WORD_RUN = /[#{DEVANAGARI_WORD_CHARS}]+/
29
+
30
+ # One word character.
31
+ WORD_CHAR = /[#{DEVANAGARI_WORD_CHARS}]/
32
+
33
+ # Tokenize text into words.
34
+ #
35
+ # @param text [String] Text to tokenize
36
+ # @return [Array<String>] Array of tokens
37
+ def tokenize(text)
38
+ return [] if text.nil? || text.empty?
39
+
40
+ text.scan(WORD_RUN).reject { |token| skip_token?(token) }
41
+ end
42
+
43
+ # Get word boundary regex.
44
+ #
45
+ # @return [Regexp] Word boundary regex
46
+ def word_boundary_regex
47
+ WORD_CHAR
48
+ end
49
+
50
+ # Spell-check word characters (plan 91): the Devanagari set —
51
+ # base consonants with their matras and virama conjuncts, no
52
+ # digits and no Latin.
53
+ #
54
+ # @return [Regexp] Regex matching a single word character
55
+ def spellcheck_word_regex
56
+ WORD_CHAR
57
+ end
58
+
59
+ protected
60
+
61
+ # Get word characters.
62
+ #
63
+ # @return [String] Character class source
64
+ def word_chars
65
+ DEVANAGARI_WORD_CHARS
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end