kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,188 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # One language detection: the top label code (ISO 639) and its
6
+ # probability, the shape the plan-106 facade returns.
7
+ #
8
+ # A Struct (keyword_init) per the plain-data rule — no behavior
9
+ # beyond readers, #to_s (the code, so string interpolation reads
10
+ # naturally), and equality.
11
+ Detection = Struct.new(:code, :score, keyword_init: true) do
12
+ # The code, so `"detected #{detection}"` reads naturally.
13
+ def to_s
14
+ code.to_s
15
+ end
16
+ end
17
+
18
+ # Document language detection through the pure-Rust LID reader
19
+ # (plan 106).
20
+ #
21
+ # The gem's {LanguageIdentifier} needs the Python fastText
22
+ # bindings (broken under numpy 2), which is why /v1/detect and the
23
+ # CLI shipped a 7-language heuristic. The native extension now
24
+ # wraps kotoshu-rs' `lid` reader over the
25
+ # `kotoshu://models/lid/lid-176` registry artifact pair (the
26
+ # ONNX-converted lid.176 model + vocab sidecar, 176 languages,
27
+ # bit-faithful to the bindings — parity frozen by the kotoshu-rs
28
+ # suite over the shared corpus fixture).
29
+ #
30
+ # Selection mirrors {NativeBackend} with one difference: the
31
+ # native engine serves detect whenever the extension is built and
32
+ # the artifact pair is cached — an explicitly ruby backend
33
+ # (KOTOSHU_BACKEND=ruby) opts out, but the unset default does not,
34
+ # because the {Detector} heuristic is strictly weaker (7 languages
35
+ # vs 176). Pure-Ruby installs keep working unchanged — no new
36
+ # gemspec dependency, no onnxruntime.
37
+ #
38
+ # Two-stage like every resource: {setup} downloads the pair once
39
+ # (registry-resolved, sha-verified); {detect} resolves from the
40
+ # cache only and never downloads implicitly.
41
+ #
42
+ # @example Setup once, then detect
43
+ # Kotoshu.setup_lid
44
+ # Kotoshu.detect_language("Der Ausschuss prüft den Vorschlag")
45
+ # # => #<struct Kotoshu::Language::Detection code="de", score=0.99>
46
+ class LidDetector
47
+ class << self
48
+ # Detect the language of text.
49
+ #
50
+ # Native LID when available (see {LidDetector} for the
51
+ # selection rules), the {Detector} heuristic otherwise.
52
+ #
53
+ # @param text [String] Text to analyze
54
+ # @param config [Configuration] configuration to consult
55
+ # (defaults to the global instance)
56
+ # @return [Detection] code (nil when the heuristic is
57
+ # uncertain) and score in [0, 1]
58
+ def detect(text, config: Kotoshu.configuration)
59
+ # nil is not text; an empty/blank string IS (the native
60
+ # reader scores its EOS token row — "en", ~0.12 — exactly
61
+ # like the fastText bindings, per the frozen corpus).
62
+ return Detection.new(code: nil, score: 0.0) if text.nil?
63
+
64
+ model = native_model(config: config)
65
+ return heuristic_detection(text) unless model
66
+
67
+ row = model.detect(text)
68
+ Detection.new(code: row.fetch("code"), score: row.fetch("score"))
69
+ end
70
+
71
+ # Whether {detect} would run the native engine for this
72
+ # configuration — the extension is built, the backend is not
73
+ # "ruby", and the artifact pair is cached.
74
+ #
75
+ # @param config [Configuration]
76
+ # @return [Boolean]
77
+ def available?(config: Kotoshu.configuration)
78
+ !native_model(config: config).nil?
79
+ end
80
+
81
+ # Download the LID artifact pair into the model cache (stage
82
+ # one of the two-stage model; registry-resolved, sha-verified).
83
+ #
84
+ # @param config [Configuration]
85
+ # @param force [Boolean] re-fetch even if already cached
86
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
87
+ # @raise [Kotoshu::Error] no registry entry, or offline with
88
+ # no cached registry (KOTOSHU_OFFLINE=1)
89
+ # @raise [Kotoshu::IntegrityError] downloaded bytes fail the
90
+ # registry checksum
91
+ def setup(config: Kotoshu.configuration, force: false)
92
+ pair = model_cache(config: config).load_cached_lid
93
+ return pair if pair && !force
94
+
95
+ model_cache(config: config).download_lid(force: force)
96
+ end
97
+
98
+ # Whether the LID artifact pair is cached (cache-only check).
99
+ #
100
+ # @param config [Configuration]
101
+ # @return [Boolean]
102
+ def setup?(config: Kotoshu.configuration)
103
+ model_cache(config: config).lid_cached?
104
+ end
105
+
106
+ # Drop the memoized native model. The next {detect} re-resolves
107
+ # from the cache — the seam tests and embedders use after
108
+ # {setup} or a cache purge (mirrors Kotoshu.reset_spellchecker).
109
+ #
110
+ # @return [nil]
111
+ def reset
112
+ @native_model = nil
113
+ nil
114
+ end
115
+
116
+ private
117
+
118
+ # The loaded native LID model, memoized per process — one
119
+ # parse of the ~3 MB pair, mirroring how spellcheckers are
120
+ # cached per language. Nil whenever the native engine cannot
121
+ # serve: extension absent, Kotoshu::Native::LidModel missing
122
+ # (extension predates plan 106), an explicit ruby backend, or
123
+ # the pair not set up.
124
+ #
125
+ # @param config [Configuration]
126
+ # @return [Kotoshu::Native::LidModel, nil]
127
+ def native_model(config:)
128
+ return nil unless native_lid_surface?
129
+ return nil if backend_opts_out?(config)
130
+
131
+ @native_model ||= begin
132
+ pair = model_cache(config: config).load_cached_lid
133
+ pair && Kotoshu::Native::LidModel.load(pair[:onnx_path], pair[:vocab_path])
134
+ end
135
+ end
136
+
137
+ # Whether the extension carries the LID surface at all.
138
+ def native_lid_surface?
139
+ Kotoshu::Native.available? && Kotoshu::Native.const_defined?(:LidModel)
140
+ end
141
+
142
+ # Backend selection for LID (plan 106).
143
+ #
144
+ # KOTOSHU_BACKEND=ruby opts the LID out of the native engine.
145
+ # When the backend is UNSET, LID still prefers the native
146
+ # engine — unlike correct?/suggest, where "ruby" is the engine
147
+ # default and "native" the explicit accelerator. The heuristic
148
+ # is strictly weaker than the 176-language model, so it stays
149
+ # the fallback for the cannot-serve cases (no extension, no
150
+ # cached model), not for an unset preference.
151
+ #
152
+ # Explicitness is read through the resolver layers (live ENV,
153
+ # constructor settings, CLI options) so the "ruby" DEFAULT
154
+ # never counts as an opt-out. Mutating
155
+ # Kotoshu.configuration.backend after construction is not
156
+ # visible here — rebuild the configuration instead.
157
+ def backend_opts_out?(config)
158
+ config.backend.to_s == "ruby" && backend_set?(config)
159
+ end
160
+
161
+ # Whether the backend was explicitly set at any resolver layer
162
+ # (rather than left at its "ruby" default).
163
+ def backend_set?(config)
164
+ resolver = config.resolver
165
+ ENV.key?("KOTOSHU_BACKEND") ||
166
+ resolver.programmatic.key?(:backend) ||
167
+ resolver.cli.key?(:backend)
168
+ end
169
+
170
+ # The heuristic fallback, wrapped in the same {Detection}
171
+ # shape the native path returns.
172
+ def heuristic_detection(text)
173
+ code, confidence = Detector.detect_with_confidence(text)
174
+ Detection.new(code: code, score: confidence)
175
+ end
176
+
177
+ # A cache over the configuration's model cache path and
178
+ # source registry (so KOTOSHU_MODELS_PIN is honored).
179
+ def model_cache(config:)
180
+ Cache::ModelCache.new(
181
+ cache_path: config.cache_path,
182
+ source_registry: config.source_registry
183
+ )
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
@@ -0,0 +1,113 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Normalizer for Arabic script.
7
+ #
8
+ # Arabic has many presentation forms (the U+FE70..U+FEFF Arabic
9
+ # Presentation Forms-A block plus the FB50..FDFF / FE70..FEFF
10
+ # blocks for Presentation Forms-B) that look the same to a
11
+ # reader but are distinct codepoints. Input text often arrives
12
+ # in any of these forms depending on the source editor — same
13
+ # word can be encoded three different ways and not match a
14
+ # dictionary lookup. This normalizer canonicalizes them all to
15
+ # the base U+0621..U+064A range.
16
+ #
17
+ # Also handles:
18
+ # - NFC composition (combines base + combining marks).
19
+ # - Optional diacritic (tashkeel) stripping for spell checking —
20
+ # Modern Standard Arabic is typically written without
21
+ # diacritics; the dictionary stores bare forms.
22
+ # - Tatweel (U+0640) removal — a decorative elongation mark
23
+ # that creates variants of the same word.
24
+ #
25
+ # @example Default (strip diacritics)
26
+ # norm = Arabic.new
27
+ # norm.normalize_word("كِتَابٌ") # => "كتاب"
28
+ class Arabic < Base
29
+ # Arabic Presentation Forms-A and Forms-B codepoints that
30
+ # have a base-form equivalent in the U+0621..U+064A range.
31
+ # Ruby's String#unicode_normalize(:nfc) handles the
32
+ # composition for most pairs; the legacy Presentation Forms-B
33
+ # range (FE70..FEFF) needs an explicit table because NFC
34
+ # doesn't cover it.
35
+ PRESENTATION_FORM_MAP = {
36
+ "ﺍ" => "ا", "ﺎ" => "ا",
37
+ "ﺏ" => "ب", "ﺑ" => "ب", "ﺒ" => "ب", "ﺓ" => "ة",
38
+ "ﺕ" => "ت", "ﺗ" => "ت", "ﺘ" => "ت", "ﺚ" => "ث",
39
+ "ﺛ" => "ث", "ﺜ" => "ث", "ﺝ" => "ج", "ﺟ" => "ج",
40
+ "ﺠ" => "ج", "ﺡ" => "ح", "ﺣ" => "ح", "ﺤ" => "ح",
41
+ "ﺥ" => "خ", "ﺧ" => "خ", "ﺨ" => "خ", "ﺩ" => "د",
42
+ "ﺪ" => "د", "ﺫ" => "ذ", "ﺬ" => "ذ", "ﺭ" => "ر",
43
+ "ﺮ" => "ر", "ﺯ" => "ز", "ﺰ" => "ز", "ﺱ" => "س",
44
+ "ﺳ" => "س", "ﺴ" => "س", "ﺵ" => "ش", "ﺷ" => "ش",
45
+ "ﺸ" => "ش", "ﺹ" => "ص", "ﺻ" => "ص", "ﺼ" => "ص",
46
+ "ﺽ" => "ض", "ﺿ" => "ض", "ﻀ" => "ض", "ﻁ" => "ط",
47
+ "ﻃ" => "ط", "ﻄ" => "ط", "ﻅ" => "ظ", "ﻇ" => "ظ",
48
+ "ﻈ" => "ظ", "ﻉ" => "ع", "ﻋ" => "ع", "ﻌ" => "ع",
49
+ "ﻍ" => "غ", "ﻏ" => "غ", "ﻐ" => "غ", "ﻑ" => "ف",
50
+ "ﻓ" => "ف", "ﻔ" => "ف", "ﻕ" => "ق", "ﻗ" => "ق",
51
+ "ﻘ" => "ق", "ﻙ" => "ك", "ﻛ" => "ك", "ﻜ" => "ك",
52
+ "ﻝ" => "ل", "ﻟ" => "ل", "ﻠ" => "ل", "ﻡ" => "م",
53
+ "ﻣ" => "م", "ﻤ" => "م", "ﻥ" => "ن", "ﻧ" => "ن",
54
+ "ﻨ" => "ن", "ﻩ" => "ه", "ﻫ" => "ه", "ﻬ" => "ه",
55
+ "ﻭ" => "و", "ﻮ" => "و", "ﻱ" => "ي", "ﻳ" => "ي",
56
+ "ﻴ" => "ي", "ﻵ" => "آ", "ﻶ" => "آ", "ﻷ" => "أ",
57
+ "ﻸ" => "أ", "ﻹ" => "إ", "ﻺ" => "إ", "ﻻ" => "لا",
58
+ "ﻼ" => "لا"
59
+ }.freeze
60
+
61
+ # Arabic diacritics (tashkeel) — harakat, tanwin, shaddah,
62
+ # sukun, dagger alef. Stripped for spell-checking since the
63
+ # dictionary stores bare forms.
64
+ DIACRITICS = /[ؐ-ًؚ-ٰٟۖ-ۜ۟-ۤۧ-۪ۨ-ۭ]/
65
+
66
+ # Tatweel (kashida) — decorative horizontal elongation.
67
+ TATWEEL = "ـ"
68
+
69
+ # @param strip_diacritics [Boolean] default true. When true,
70
+ # tashkeel (vowel points) are removed before lookup.
71
+ # @param strip_tatweel [Boolean] default true. When true,
72
+ # tatweel elongation marks are removed.
73
+ def initialize(strip_diacritics: true, strip_tatweel: true)
74
+ @strip_diacritics = strip_diacritics
75
+ @strip_tatweel = strip_tatweel
76
+ end
77
+
78
+ # Normalize text: NFC + presentation-form canonicalization +
79
+ # optional diacritic/tatweel stripping. Skips the Base class's
80
+ # downcase step because Arabic has no case.
81
+ #
82
+ # @param text [String]
83
+ # @param _options [Hash] ignored (Arabic has no case to fold)
84
+ # @return [String]
85
+ def normalize(text, _options = {})
86
+ return "" if text.nil? || text.empty?
87
+
88
+ result = text.unicode_normalize(:nfc)
89
+ result = canonicalize_presentation_forms(result)
90
+ result = result.gsub(DIACRITICS, "") if @strip_diacritics
91
+ result = result.delete(TATWEEL) if @strip_tatweel
92
+ result.strip
93
+ end
94
+
95
+ # Normalize a word: same as +normalize+ but operates on a
96
+ # single token. Arabic words don't need different treatment.
97
+ def normalize_word(word)
98
+ normalize(word)
99
+ end
100
+
101
+ private
102
+
103
+ # Replace every Presentation Forms-B character with its
104
+ # base-form equivalent. Done character-by-character because
105
+ # the mapping is irregular (one base letter may have up to
106
+ # four positional forms: isolated, initial, medial, final).
107
+ def canonicalize_presentation_forms(text)
108
+ text.chars.map { |c| PRESENTATION_FORM_MAP.fetch(c, c) }.join
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,133 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Abstract base class for text normalizers.
7
+ #
8
+ # Normalizers transform text to a standard form for comparison.
9
+ # Different languages use different normalization strategies.
10
+ #
11
+ # Examples of normalization:
12
+ # - Accent removal (café -> cafe)
13
+ # - Case folding (Hello -> hello)
14
+ # - Whitespace normalization
15
+ # - Punctuation normalization
16
+ #
17
+ # @example Implement a normalizer
18
+ # class MyNormalizer < Normalizer::Base
19
+ # def normalize(text)
20
+ # super.downcase.gsub(/[áàâä]/, 'a')
21
+ # end
22
+ # end
23
+ class Base
24
+ # Normalize text.
25
+ #
26
+ # Default implementation:
27
+ # - Strip leading/trailing whitespace
28
+ # - Collapse multiple whitespace to single space
29
+ # - Downcase (optional)
30
+ #
31
+ # @param text [String] Text to normalize
32
+ # @param options [Hash] Normalization options
33
+ # @option options [Boolean] :downcase (true) Convert to lowercase
34
+ # @option options [Boolean] :strip_punct (false) Remove punctuation
35
+ # @option options [Boolean] :collapse_ws (true) Collapse whitespace
36
+ # @return [String] Normalized text
37
+ def normalize(text, options = {})
38
+ return "" if text.nil?
39
+
40
+ defaults = {
41
+ downcase: true,
42
+ strip_punct: false,
43
+ collapse_ws: true
44
+ }
45
+ opts = defaults.merge(options)
46
+
47
+ result = text.dup
48
+
49
+ # Strip whitespace
50
+ result = result.strip
51
+
52
+ # Collapse multiple whitespace
53
+ result = result.gsub(/\s+/, " ") if opts[:collapse_ws]
54
+
55
+ # Downcase
56
+ result = result.downcase if opts[:downcase]
57
+
58
+ # Strip punctuation
59
+ result = strip_punctuation(result) if opts[:strip_punct]
60
+
61
+ result
62
+ end
63
+
64
+ # Normalize a word.
65
+ #
66
+ # @param word [String] Word to normalize
67
+ # @return [String] Normalized word
68
+ def normalize_word(word)
69
+ normalize(word)
70
+ end
71
+
72
+ # Check if two normalized strings are equal.
73
+ #
74
+ # @param str1 [String] First string
75
+ # @param str2 [String] Second string
76
+ # @return [Boolean] True if equal after normalization
77
+ def normalized_eql?(str1, str2)
78
+ normalize(str1) == normalize(str2)
79
+ end
80
+
81
+ # Strip punctuation from text.
82
+ #
83
+ # @param text [String] Text to strip
84
+ # @return [String] Text without punctuation
85
+ def strip_punctuation(text)
86
+ text.gsub(/[^\p{L}\p{N}\s]/, "")
87
+ end
88
+
89
+ # Remove accents from characters.
90
+ #
91
+ # @param text [String] Text with accents
92
+ # @return [String] Text without accents
93
+ def remove_accents(text)
94
+ # Unicode normalization form D (decompose)
95
+ normalized = text.unicode_normalize(:nfd)
96
+
97
+ # Remove combining diacritical marks
98
+ normalized.gsub(/[\u0300-\u036F]/, "")
99
+ end
100
+
101
+ # Normalize quotes to standard ASCII.
102
+ #
103
+ # @param text [String] Text with quotes
104
+ # @return [String] Text with normalized quotes
105
+ def normalize_quotes(text)
106
+ # Left double quote to straight
107
+ text = text.gsub(/[\u201C\u201D]/, '"')
108
+ # Right double quote to straight
109
+ text = text.gsub(/[\u2018\u2019]/, "'")
110
+ # Backticks to quotes
111
+ text = text.gsub('`', "'")
112
+ # Other quote variants
113
+ text = text.gsub("\u00AB", '"') # Left-pointing double angle
114
+ text = text.gsub("\u00BB", '"') # Right-pointing double angle
115
+ text = text.gsub("\u2039", "'") # Single left-pointing
116
+ text.gsub("\u203A", "'") # Single right-pointing
117
+ end
118
+
119
+ # Normalize whitespace.
120
+ #
121
+ # @param text [String] Text with irregular whitespace
122
+ # @return [String] Text with normalized whitespace
123
+ def normalize_whitespace(text)
124
+ text
125
+ .gsub(/[\u00A0\u202F\u205F]/, " ") # Various space chars
126
+ .gsub(/[\u2000-\u200B]/, " ") # Various space chars
127
+ .gsub(/\s+/, " ") # Collapse multiple spaces
128
+ .strip
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Greek-aware normalizer.
7
+ #
8
+ # Two Greek-specific behaviors over the base normalizer:
9
+ #
10
+ # 1. Final sigma: uppercase words ending in Σ downcase to σ, but
11
+ # dictionary forms end in ς. ΕΛΛΑΣ must look up as ελλας, not
12
+ # ελλασ. Applied at word level (normalize_word), where the
13
+ # word boundary is known.
14
+ # 2. Optional accent stripping (tonos and dialytika) for
15
+ # accent-insensitive comparison: πάτησις -> πατησις. Greek
16
+ # Hunspell dictionaries carry accents, so this is opt-in via
17
+ # the :strip_accents option (default false).
18
+ class Greek < Base
19
+ # Combining marks removed by accent stripping: Greek tonos,
20
+ # dialytika and the polytonic accent/breath marks
21
+ # (U+0300 through U+0345).
22
+ COMBINING_MARKS = /[\u0300-\u0345]/
23
+
24
+ private_constant :COMBINING_MARKS
25
+
26
+ # Normalize text.
27
+ #
28
+ # @param text [String] Text to normalize
29
+ # @param options [Hash] Normalization options
30
+ # @option options [Boolean] :strip_accents (false) Remove
31
+ # Greek accents after downcasing
32
+ # @return [String] Normalized text
33
+ def normalize(text, options = {})
34
+ result = super
35
+ result = strip_accents(result) if options.fetch(:strip_accents, false)
36
+ result
37
+ end
38
+
39
+ # Normalize a word for dictionary lookup.
40
+ #
41
+ # Downcase plus the final-sigma correction: a word ending in σ
42
+ # folds to ς to match dictionary forms.
43
+ #
44
+ # @param word [String] Word to normalize
45
+ # @return [String] Normalized word
46
+ def normalize_word(word)
47
+ normalized = super
48
+ normalized.gsub(/σ\z/, "ς")
49
+ end
50
+
51
+ private
52
+
53
+ # Strip Greek accents via NFD decomposition.
54
+ #
55
+ # @param text [String] Text to deaccent
56
+ # @return [String] Text without Greek combining marks
57
+ def strip_accents(text)
58
+ text.unicode_normalize(:nfd).gsub(COMBINING_MARKS, "").unicode_normalize(:nfc)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Normalizer for Hebrew script.
7
+ #
8
+ # Modern Hebrew is typically written without niqqud (vowel
9
+ # points); spell-checking dictionaries store the bare
10
+ # consonant form. This normalizer:
11
+ #
12
+ # - Applies NFC composition.
13
+ # - Strips niqqud (U+05B0..U+05BC range, plus geresh/gershayim
14
+ # when used as vowel marks — keeps them when used as
15
+ # punctuation).
16
+ # - Optionally strips dagesh (U+05BC) — a dot inside a letter
17
+ # that affects pronunciation but is not represented in bare
18
+ # dictionary entries.
19
+ # - Normalizes the maqaf (U+05BE, Hebrew hyphen) to ASCII hyphen
20
+ # for consistent tokenization.
21
+ #
22
+ # @example Default (strip niqqud)
23
+ # norm = Hebrew.new
24
+ # norm.normalize_word("שָׁלוֹם") # => "שלום"
25
+ class Hebrew < Base
26
+ # Niqqud + cantillation marks. Hebrew points range U+0591
27
+ # (etnahta) through U+05BD (meteg). Plus geresh (U+05F3),
28
+ # gershayim (U+05F4) when used as accents. Plus paseq (U+05C0),
29
+ # sof-pasuq (U+05C3), upper dot (U+05C4), lower dot (U+05C5),
30
+ # qamats qatan (U+05C7), and raph (U+05BF).
31
+ # Shin dot (U+05C1) and sin dot (U+05C2) are included so
32
+ # שׁ and שׂ both normalize to ש.
33
+ # Dagesh (U+05BC) is handled separately so callers can keep it.
34
+ NIKKUD_AND_CANTILLATION = /[֑-ֻֽֿ׀-ׇׂׅׄ]/
35
+
36
+ DAGESH = "ּ"
37
+ MAQAF = "־"
38
+
39
+ # @param strip_dagesh [Boolean] default true. When true,
40
+ # removes the dagesh dot from letters.
41
+ # @param normalize_maqaf [Boolean] default true. When true,
42
+ # replaces the Hebrew hyphen (maqaf) with an ASCII hyphen
43
+ # for consistent tokenization.
44
+ def initialize(strip_dagesh: true, normalize_maqaf: true)
45
+ @strip_dagesh = strip_dagesh
46
+ @normalize_maqaf = normalize_maqaf
47
+ end
48
+
49
+ # Normalize text: NFC + niqqud stripping + optional dagesh
50
+ # stripping + maqaf normalization. Skips Base's downcase
51
+ # step because Hebrew has no case.
52
+ #
53
+ # @param text [String]
54
+ # @param _options [Hash] ignored
55
+ # @return [String]
56
+ def normalize(text, _options = {})
57
+ return "" if text.nil? || text.empty?
58
+
59
+ result = text.unicode_normalize(:nfc)
60
+ result = result.gsub(NIKKUD_AND_CANTILLATION, "")
61
+ result = result.delete(DAGESH) if @strip_dagesh
62
+ result = result.tr(MAQAF, "-") if @normalize_maqaf
63
+ result.strip
64
+ end
65
+
66
+ def normalize_word(word)
67
+ normalize(word)
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Normalizer for Persian (Farsi) script.
7
+ #
8
+ # Extends {Arabic} with Persian-specific letter mappings.
9
+ # Persian shares the Arabic script but uses different
10
+ # codepoints for several letters that look the same to a
11
+ # reader but are distinct in Unicode:
12
+ #
13
+ # - Arabic Yeh (ي U+064A) → Persian Yeh (ی U+06CC)
14
+ # - Arabic Kaf (ك U+0643) → Persian Kaf (ک U+06A9)
15
+ #
16
+ # Input text from Arabic-origin sources may contain either
17
+ # form. This normalizer maps both to the Persian form so a
18
+ # dictionary storing Persian codepoints matches.
19
+ class Persian < Arabic
20
+ # Arabic → Persian letter canonicalization.
21
+ PERSIAN_LETTER_MAP = {
22
+ "ي" => "ی", # Arabic Yeh → Persian Yeh
23
+ "ك" => "ک" # Arabic Kaf → Persian Kaf
24
+ }.freeze
25
+
26
+ def normalize(text, _options = {})
27
+ return "" if text.nil? || text.empty?
28
+
29
+ result = super
30
+ PERSIAN_LETTER_MAP.each { |from, to| result = result.gsub(from, to) }
31
+ result
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ module Normalizer
6
+ # Turkish-aware normalizer.
7
+ #
8
+ # Turkish case folding is not the Unicode default: dotted and
9
+ # dotless i form a separate pair. A plain downcase maps I to i,
10
+ # which turns DİŞARIIDA into disarida — a word no Turkish
11
+ # dictionary contains. Ruby's :turkic case mapping gets the pair
12
+ # right: I -> ı and İ -> i, so İZMİR folds to izmir and
13
+ # ISTANBUL folds to ıstanbul.
14
+ class Turkish < Base
15
+ # Normalize text with Turkish case folding.
16
+ #
17
+ # @param text [String] Text to normalize
18
+ # @param options [Hash] Normalization options
19
+ # @return [String] Normalized text
20
+ def normalize(text, options = {})
21
+ downcase = options.fetch(:downcase, true)
22
+ result = super(text, options.merge(downcase: false))
23
+ downcase ? turkish_downcase(result) : result
24
+ end
25
+
26
+ private
27
+
28
+ # Fold case with the Turkish i pair.
29
+ #
30
+ # @param text [String] Text to fold
31
+ # @return [String] Folded text
32
+ def turkish_downcase(text)
33
+ text.downcase(:turkic)
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end