kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,172 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for French text.
7
+ #
8
+ # Ported from LanguageTool's FrenchWordTokenizer.
9
+ #
10
+ # Handles:
11
+ # - Apostrophes (l', d', qu', c'est, j'ai, etc.)
12
+ # - Hyphens (c'est-à-dire, rendez-vous, etc.)
13
+ # - Decimal points/commas
14
+ # - Multiple contraction patterns (7 total)
15
+ class FrenchTokenizer < Base
16
+ # French word separators - most punctuation and whitespace
17
+ # Note: apostrophe (') is NOT a separator in French (used for contractions)
18
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
19
+
20
+ # Do-not-split list (from LanguageTool)
21
+ DO_NOT_SPLIT = %w[
22
+ mers-cov mcgraw-hill sars-cov-2 sars-cov
23
+ ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
24
+ c'est-à-dire add-on add-ons rendez-vous garde-à-vous
25
+ chez-eux chez-moi chez-nous chez-soi chez-toi chez-vous
26
+ m'as-tu-vu
27
+ ].freeze
28
+
29
+ # Contraction patterns (from LanguageTool)
30
+ # French contractions are complex: l', d', qu', c'est, j'ai, n'a, etc.
31
+ CONTRACTION_PATTERNS = [
32
+ # c' followed by word: c'est, c'était, etc.
33
+ /^(c')$/i,
34
+ # j' (je): j'ai, j'aime, etc.
35
+ /^(j')$/i,
36
+ # n' (ne): n'a, n'est, etc.
37
+ /^(n')$/i,
38
+ # m' (me): m'a, m'appelle, etc.
39
+ /^(m')$/i,
40
+ # t' (te): t'a, t'asseoir, etc.
41
+ /^(t')$/i,
42
+ # s' (se): s'a, s'appelle, etc.
43
+ /^(s')$/i,
44
+ # l' (le/la): l'a, l'homme, l'eau, etc.
45
+ /^(l')$/i,
46
+ # d' (de): d'un, d'une, d'abord, etc.
47
+ /^(d')$/i,
48
+ # qu' (que): qu'un, qu'une, qu'est, etc.
49
+ /^(qu')$/i,
50
+ # jusqu'à, jusqu'aux, etc.
51
+ /^(jusqu')$/i,
52
+ # puisque, puisqu'il, etc.
53
+ /^(puisqu')$/i,
54
+ # quoique, quoiqu'il, etc.
55
+ /^(quoiqu')$/i,
56
+ # lorsque, lorsqu'il, etc.
57
+ /^(lorsqu')$/i,
58
+ ].freeze
59
+
60
+ def tokenize(text)
61
+ return [] if text.nil? || text.strip.empty?
62
+
63
+ # Replace hyphen variants
64
+ text = text.gsub("\u2010", "\u002d")
65
+ text = text.gsub("\u2011", "\u002d")
66
+
67
+ # Normalize apostrophes
68
+ text = normalize_apostrophes(text)
69
+
70
+ # Split on word boundaries
71
+ raw_tokens = text.split(WORD_SEPARATORS)
72
+
73
+ # Process each token
74
+ tokens = []
75
+ raw_tokens.each do |token|
76
+ next if token.empty?
77
+
78
+ # Try to split contractions and hyphenated words
79
+ parts = split_french_word(token)
80
+ tokens.concat(parts)
81
+ end
82
+
83
+ # Filter and normalize
84
+ tokens
85
+ .map { |token| normalize(token) }
86
+ .reject { |token| skip_token?(token) }
87
+ end
88
+
89
+ protected
90
+
91
+ # Normalize apostrophes to straight quotes.
92
+ #
93
+ # @param text [String] Input text
94
+ # @return [String] Text with normalized apostrophes
95
+ def normalize_apostrophes(text)
96
+ text
97
+ .gsub("'", "'")
98
+ .gsub("'", "'")
99
+ .gsub("'", "'")
100
+ end
101
+
102
+ # Split French word, handling contractions and hyphens.
103
+ #
104
+ # @param word [String] Word to split
105
+ # @return [Array<String>] Array of tokens
106
+ def split_french_word(word)
107
+ # Check do-not-split list
108
+ return [word] if DO_NOT_SPLIT.include?(word.downcase)
109
+
110
+ # Handle hyphens first (but not for do-not-split words)
111
+ if word.include?("-")
112
+ # Check if it's a contraction pattern like "jusqu'à-ce"
113
+ if word.match?(/^(jusqu'|[cç]'|j'|n'|m'|t'|s'|l'|d'|qu'|lorsqu'|puisqu'|quoiqu')/)
114
+ # Split on hyphen for contractions
115
+ parts = []
116
+ word.split("-", -1).each do |part|
117
+ next if part.empty?
118
+
119
+ parts.concat(split_contractions(part))
120
+ end
121
+ return parts
122
+ else
123
+ # Regular hyphenated word - split it
124
+ return word.split("-", -1).reject(&:empty?)
125
+ end
126
+ end
127
+
128
+ # Handle contractions
129
+ if word.include?("'")
130
+ return split_contractions(word)
131
+ end
132
+
133
+ # No special handling needed
134
+ [word]
135
+ end
136
+
137
+ # Split contractions into component parts.
138
+ #
139
+ # @param word [String] Word that might be a contraction
140
+ # @return [Array<String>] Array of tokens
141
+ def split_contractions(word)
142
+ # Try each contraction pattern
143
+ CONTRACTION_PATTERNS.each do |pattern|
144
+ match = word.match(pattern)
145
+ if match
146
+ # Return the contraction and the rest of the word
147
+ contraction = match[1]
148
+ rest = word.sub(/^#{Regexp.escape(contraction)}/, "")
149
+ return [contraction, rest] unless rest.empty?
150
+
151
+ return [contraction]
152
+ end
153
+ end
154
+
155
+ # Handle special case: word starts with apostrophe
156
+ if word.match?(/^[cç]'|^[a-z]'/i)
157
+ # Split at the apostrophe
158
+ parts = word.split("'", 2)
159
+ return parts if parts.length == 2
160
+ end
161
+
162
+ # No pattern matched, return the word as-is
163
+ [word]
164
+ end
165
+
166
+ def word_separators
167
+ WORD_SEPARATORS
168
+ end
169
+ end
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for German text.
7
+ #
8
+ # Ported from LanguageTool's GermanWordTokenizer.
9
+ #
10
+ # Handles:
11
+ # - Underscore as word character (not a separator)
12
+ # - Single low quote (‚) as word character (not a separator)
13
+ # - Umlauts (ä, ö, ü, ß)
14
+ #
15
+ # The LanguageTool implementation adds two characters to the word characters:
16
+ # underscore (_) and single low quote (‚ - U+201A).
17
+ class GermanTokenizer < Base
18
+ # German-specific word separators (exclude underscore and single low quote)
19
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·]/
20
+
21
+ def tokenize(text)
22
+ return [] if text.nil? || text.strip.empty?
23
+
24
+ # Split on word boundaries
25
+ raw_tokens = text.split(WORD_SEPARATORS)
26
+
27
+ # Filter and normalize
28
+ raw_tokens
29
+ .map { |token| normalize(token) }
30
+ .reject { |token| skip_token?(token) }
31
+ end
32
+
33
+ protected
34
+
35
+ def word_separators
36
+ WORD_SEPARATORS
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Greek-script text.
7
+ #
8
+ # Splits on the same punctuation/whitespace separators as the
9
+ # Latin tokenizer, but keeps tokens that contain Greek letters
10
+ # (the Latin tokenizer's letter checks would drop a pure Greek
11
+ # word). Greek punctuation keeps its separators; the Greek
12
+ # question mark is the semicolon character and is treated as
13
+ # punctuation, matching the physical Greek keyboard where it is
14
+ # the shift of the question-mark key.
15
+ class GreekTokenizer < Base
16
+ # Separators between Greek words. Mirrors LatinTokenizer's
17
+ # separator set; Greek text needs no contraction handling.
18
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
19
+
20
+ # Tokenize text into Greek words.
21
+ #
22
+ # @param text [String] Text to tokenize
23
+ # @return [Array<String>] Array of tokens
24
+ def tokenize(text)
25
+ return [] if text.nil? || text.strip.empty?
26
+
27
+ text.split(WORD_SEPARATORS)
28
+ .map(&:strip)
29
+ .reject { |token| skip_token?(token) }
30
+ end
31
+
32
+ # Get word boundary regex.
33
+ #
34
+ # @return [Regexp] Word boundary regex
35
+ def word_boundary_regex
36
+ /\p{Greek}/
37
+ end
38
+
39
+ # Spell-check word characters (plan 91): Greek letters plus the
40
+ # apostrophe, which is not a Greek separator so elisions such
41
+ # as απ’ keep it in-word.
42
+ #
43
+ # @return [Regexp] Regex matching a single word character
44
+ def spellcheck_word_regex
45
+ /[\p{Greek}']/
46
+ end
47
+
48
+ # Check if token should be skipped.
49
+ #
50
+ # Skips tokens that contain no Greek letters; Base's rules
51
+ # (empty, pure numbers) also apply.
52
+ #
53
+ # @param token [String] Token to check
54
+ # @return [Boolean] True if should skip
55
+ def skip_token?(token)
56
+ return true if super
57
+
58
+ !token.match?(/\p{Greek}/)
59
+ end
60
+
61
+ protected
62
+
63
+ # Get word characters.
64
+ #
65
+ # @return [String] Character class of Greek letters
66
+ def word_chars
67
+ "\\p{Greek}"
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Korean (plan 108).
7
+ #
8
+ # Korean text is space-delimited: an eojeol is a run of Hangul
9
+ # characters between spaces or punctuation, and every character
10
+ # inside the run belongs to the word — precomposed syllables
11
+ # (U+AC00-D7A3) as typed, plus any bare jamo (combining jamo
12
+ # U+1100-11FF, compatibility jamo U+3130-318F) that escaped IME
13
+ # composition and attaches to the preceding block. The staged ko
14
+ # dictionary stores words jamo-decomposed behind an ICONV table,
15
+ # so both surface forms must stay word characters for the
16
+ # spell-check extraction (plan 91) to hand the engine whole
17
+ # eojeol.
18
+ #
19
+ # Word segmentation (morpheme splitting inside an eojeol) is out
20
+ # of scope — like the pre-plan-108 basic tier, checking is
21
+ # eojeol-granular, matching how the staged dictionary is keyed.
22
+ class HangulTokenizer < Base
23
+ # Precomposed syllables plus the two jamo blocks (regex source
24
+ # text, like CyrillicTokenizer's word_chars).
25
+ HANGUL_WORD_CHARS = "\\uAC00-\\uD7A3\\u1100-\\u11FF\\u3130-\\u318F"
26
+
27
+ # One eojeol (a run of word characters).
28
+ WORD_RUN = /[#{HANGUL_WORD_CHARS}]+/
29
+
30
+ # One word character.
31
+ WORD_CHAR = /[#{HANGUL_WORD_CHARS}]/
32
+
33
+ # Tokenize text into eojeol.
34
+ #
35
+ # @param text [String] Text to tokenize
36
+ # @return [Array<String>] Array of eojeol tokens
37
+ def tokenize(text)
38
+ return [] if text.nil? || text.empty?
39
+
40
+ text.scan(WORD_RUN).reject { |token| skip_token?(token) }
41
+ end
42
+
43
+ # Get word boundary regex.
44
+ #
45
+ # @return [Regexp] Word boundary regex
46
+ def word_boundary_regex
47
+ WORD_CHAR
48
+ end
49
+
50
+ # Spell-check word characters (plan 91): the Hangul set —
51
+ # syllable blocks and jamo, no apostrophe (Korean has no
52
+ # in-word apostrophe) and no digits.
53
+ #
54
+ # @return [Regexp] Regex matching a single word character
55
+ def spellcheck_word_regex
56
+ WORD_CHAR
57
+ end
58
+
59
+ protected
60
+
61
+ # Get word characters.
62
+ #
63
+ # @return [String] Character class source
64
+ def word_chars
65
+ HANGUL_WORD_CHARS
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Japanese text.
7
+ #
8
+ # Uses Suika gem for morphological analysis. Suika is a soft runtime
9
+ # dependency — see {Kotoshu::Language::Suika} for load status and
10
+ # {Kotoshu::SuikaUnavailable} for the error raised when Japanese
11
+ # tokenization is requested without it.
12
+ #
13
+ # @see https://github.com/yoshoku/suika
14
+ class JapaneseTokenizer < Base
15
+ # Japanese word separators - keep it simple since Suika handles tokenization
16
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
17
+
18
+ def tokenize(text)
19
+ return [] if text.nil? || text.strip.empty?
20
+
21
+ # Suika::Tagger is process-wide memoized in Language::Suika;
22
+ # raises SuikaUnavailable when the gem is missing.
23
+ tagger = Kotoshu::Language::Suika.tagger
24
+
25
+ # Suika.parse returns an array of "surface\tfeatures" strings
26
+ tokens = []
27
+ parsed = tagger.parse(text)
28
+
29
+ parsed.each do |token|
30
+ # Suika returns: "すもも 名詞,一般,*,*,*,*,すもも,スモモ,スモモ"
31
+ # The surface form is tab-separated from the POS features
32
+ surface = token.split("\t").first
33
+ tokens << surface if surface && !surface.strip.empty?
34
+ end
35
+
36
+ tokens
37
+ end
38
+
39
+ protected
40
+
41
+ # Detect if text contains Japanese script.
42
+ #
43
+ # @param text [String] Text to check
44
+ # @return [Boolean] True if Japanese
45
+ def japanese?(text)
46
+ text.match?(/[぀-ゟ゠-ヿ]/) # Hiragana or Katakana
47
+ end
48
+
49
+ def word_separators
50
+ WORD_SEPARATORS
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Tokenizer
6
+ # Tokenizer for Latin-script languages.
7
+ #
8
+ # Base tokenizer for English, French, German, Spanish, Portuguese,
9
+ # and other European languages using Latin script.
10
+ #
11
+ # Handles:
12
+ # - Standard word boundaries (whitespace, punctuation)
13
+ # - Apostrophes within words (contractions, elisions)
14
+ # - Hyphenated words
15
+ # - Numbers with units
16
+ #
17
+ # Subclasses can override for language-specific handling.
18
+ class LatinTokenizer < Base
19
+ # Latin word characters including accented characters
20
+ WORD_CHARS = "a-zA-Zà-ÿ0-9'"
21
+
22
+ # Punctuation that separates words
23
+ WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
24
+
25
+ # Contractions that should stay together
26
+ CONTRACTIONS = %w[
27
+ I'm I'd I've I'll you're you'd you've you'll he's he'd he'll
28
+ she's she'd she'll it's it'd we're we'd we've we'll they're
29
+ they'd they've they'll that's that'd that'll who's who'd who'll
30
+ what's what'd what'll where's where'd when's when'd why's why'd
31
+ how's how'd can't won't don't shouldn't couldn't wouldn't didn't
32
+ isn't aren't wasn't weren't hasn't haven't hadn't doesn't do
33
+ doesn't didn't mightn't mustn't shan't shouldn't wouldn't
34
+ ].freeze
35
+
36
+ # Tokenize text into words.
37
+ #
38
+ # @param text [String] Text to tokenize
39
+ # @return [Array<String>] Array of tokens
40
+ def tokenize(text)
41
+ return [] if text.nil? || text.strip.empty?
42
+
43
+ # Split on word boundaries
44
+ raw_tokens = text.split(WORD_SEPARATORS)
45
+
46
+ # Filter and normalize
47
+ raw_tokens
48
+ .map { |token| normalize(token) }
49
+ .reject { |token| skip_token?(token) }
50
+ end
51
+
52
+ # Get word boundary regex.
53
+ #
54
+ # @return [Regexp] Word boundary regex
55
+ def word_boundary_regex
56
+ /[#{WORD_CHARS}]/o
57
+ end
58
+
59
+ # Spell-check word characters (plan 91): Latin-script letters
60
+ # plus the apostrophe, mirroring the Greek/Cyrillic overrides
61
+ # (\p{Latin} covers the accented letters WORD_CHARS spells out
62
+ # by range, including capitals such as Å and Ä). Digits are
63
+ # excluded so number handling is unchanged from the ASCII-era
64
+ # extraction.
65
+ #
66
+ # @return [Regexp] Regex matching a single word character
67
+ def spellcheck_word_regex
68
+ /[\p{Latin}']/
69
+ end
70
+
71
+ # Normalize token.
72
+ #
73
+ # Subclasses can override for language-specific normalization.
74
+ #
75
+ # @param token [String] Token to normalize
76
+ # @return [String] Normalized token
77
+ def normalize(token)
78
+ token.strip
79
+ end
80
+
81
+ # Check if token should be skipped.
82
+ #
83
+ # @param token [String] Token to check
84
+ # @return [Boolean] True if should skip
85
+ def skip_token?(token)
86
+ return true if super
87
+
88
+ # Skip pure numbers
89
+ return true if token.match?(/^\d+$/)
90
+
91
+ # Skip single characters (unless a word)
92
+ return true if token.length == 1 && token.match?(/[^a-zA-Zà-ÿ]/)
93
+
94
+ # Skip empty tokens
95
+ return true if token.empty?
96
+
97
+ # Skip tokens with no letters
98
+ return true unless token.match?(/[a-zA-Zà-ÿ]/)
99
+
100
+ false
101
+ end
102
+
103
+ protected
104
+
105
+ # Get word characters.
106
+ #
107
+ # @return [String] Character class
108
+ def word_chars
109
+ WORD_CHARS
110
+ end
111
+
112
+ # Handle contractions to keep them together.
113
+ #
114
+ # @param text [String] Input text
115
+ # @return [String] Text with protected contractions
116
+ def handle_contractions(text)
117
+ result = text.dup
118
+
119
+ # Protect common contractions
120
+ CONTRACTIONS.each do |contraction|
121
+ # Use word boundaries to avoid partial matches
122
+ result = result.gsub(/\b#{Regexp.escape(contraction)}\b/, contraction.gsub("'", "\uFEFF"))
123
+ end
124
+
125
+ result
126
+ end
127
+
128
+ # Extract next token with position.
129
+ #
130
+ # Override to handle apostrophes within words.
131
+ #
132
+ # @param text [String] Full text
133
+ # @param position [Integer] Current position
134
+ # @return [String, nil] Next token or nil
135
+ def extract_next_token(text, position)
136
+ remaining = text[position..]
137
+
138
+ # Check for contraction first
139
+ CONTRACTIONS.each do |contraction|
140
+ if remaining.start_with?(contraction) &&
141
+ remaining[contraction.length]&.match?(/\s|[^a-zA-Zà-ÿ]/)
142
+ return contraction
143
+ end
144
+ end
145
+
146
+ # Extract word with potential apostrophe
147
+ match = remaining.match(/^([#{WORD_CHARS}]+(?:'[#{WORD_CHARS}]+)?)/o)
148
+ match ? match[1] : nil
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end