kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Embeddings module for FastText ONNX integration.
4
+ #
5
+ # Provides semantic spell checking using FastText word embeddings converted
6
+ # to ONNX format. Loading this file is cheap — every class below is
7
+ # autoloaded on first reference.
8
+ #
9
+ # @example Simple usage (recommended)
10
+ # pipeline = Kotoshu::Embeddings.from_cache(language: 'en')
11
+ # neighbors = pipeline.find_nearest('semantic', k: 5)
12
+ #
13
+ # @example Advanced usage
14
+ # vocab = Kotoshu::Embeddings::Vocabulary.from_file('vocab.json')
15
+ # model = Kotoshu::Embeddings::OnnxRuntimeModel.from_file('model.onnx')
16
+ # engine = Kotoshu::Embeddings::SimilarityEngine.new(pre_normalize: true)
17
+ #
18
+ module Kotoshu
19
+ module Embeddings
20
+ DEFAULT_DIMENSION = 300
21
+ MAX_VOCABULARY_SIZE = 100_000
22
+ VERSION = '2.0.0'
23
+
24
+ autoload :Protocol, "kotoshu/embeddings/protocol"
25
+ autoload :ProtocolError, "kotoshu/embeddings/protocol"
26
+ autoload :EmbeddingModelProtocol, "kotoshu/embeddings/protocol"
27
+ autoload :SimilarityEngineProtocol, "kotoshu/embeddings/protocol"
28
+ autoload :VocabularyProtocol, "kotoshu/embeddings/protocol"
29
+ autoload :LruCache, "kotoshu/embeddings/lru_cache"
30
+ autoload :Vocabulary, "kotoshu/embeddings/vocabulary"
31
+ autoload :OnnxRuntimeModel, "kotoshu/embeddings/onnx_runtime_model"
32
+ autoload :SimilarityEngine, "kotoshu/embeddings/similarity_engine"
33
+ autoload :Search, "kotoshu/embeddings/search"
34
+ autoload :SimilaritySearch, "kotoshu/embeddings/similarity_search"
35
+ autoload :EmbeddingPipeline, "kotoshu/embeddings/embedding_pipeline"
36
+ autoload :Registry, "kotoshu/embeddings/registry"
37
+
38
+ # Create an EmbeddingPipeline from cache
39
+ #
40
+ # @param language [String] ISO 639-1 language code
41
+ # @param preload [Boolean] Preload embeddings into memory
42
+ # @param index [:exact, :auto] Search index type
43
+ # @return [EmbeddingPipeline]
44
+ #
45
+ def self.from_cache(language:, preload: false, index: :exact)
46
+ EmbeddingPipeline.from_cache(language: language, preload: preload,
47
+ index: index)
48
+ end
49
+
50
+ # Check if a language is supported
51
+ #
52
+ # @param language [String] ISO 639-1 language code
53
+ # @return [Boolean]
54
+ #
55
+ def self.language_supported?(language)
56
+ cache = Kotoshu::Cache::ModelCache.new
57
+ cache.available_models_for(language.to_sym).include?(:onnx)
58
+ end
59
+
60
+ # List all supported languages
61
+ #
62
+ # @return [Array<String>]
63
+ #
64
+ def self.supported_languages
65
+ cache = Kotoshu::Cache::ModelCache.new
66
+ cache.all_available_models[:onnx].keys.map(&:to_s)
67
+ end
68
+
69
+ # Create a custom embedding pipeline
70
+ #
71
+ # @param vocabulary [Vocabulary] Vocabulary instance
72
+ # @param model [EmbeddingModel] Model instance
73
+ # @param preload [Boolean] Preload embeddings
74
+ # @param pre_normalize [Boolean] Pre-normalize vectors
75
+ # @return [EmbeddingPipeline]
76
+ #
77
+ def self.create_pipeline(vocabulary:, model:, preload: false,
78
+ pre_normalize: false)
79
+ EmbeddingPipeline.new(
80
+ vocabulary: vocabulary,
81
+ model: model,
82
+ preload: preload,
83
+ pre_normalize: pre_normalize,
84
+ )
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ class Spellchecker
5
+ # Fluent checker for chainable configuration.
6
+ #
7
+ # Experimental and not public API: outside the 1.0 stability
8
+ # freeze, may change or be removed without a deprecation cycle.
9
+ #
10
+ # Provides a convenient API for spell checking with method chaining.
11
+ #
12
+ # @example Basic usage
13
+ # result = Kotoshu.fluent.check("Hello wrold")
14
+ #
15
+ # @example With options
16
+ # Kotoshu.fluent
17
+ # .ignore_words(/https?:\/\/\S+/)
18
+ # .max_suggestions(5)
19
+ # .check("Hello wrold")
20
+ class FluentChecker
21
+ # @return [Spellchecker] The underlying spellchecker
22
+ attr_reader :spellchecker
23
+
24
+ # @return [Hash] Configuration options
25
+ attr_reader :options
26
+
27
+ # Create a new fluent checker.
28
+ #
29
+ # @param spellchecker [Spellchecker] The underlying spellchecker
30
+ # @param options [Hash] Configuration options
31
+ def initialize(spellchecker:, options: {})
32
+ @spellchecker = spellchecker
33
+ @options = options
34
+ @progress_callback = nil
35
+ @error_callback = nil
36
+ end
37
+
38
+ # Check text for spelling errors.
39
+ #
40
+ # @param text [String] Text to check
41
+ # @return [Models::Result::DocumentResult] Check result
42
+ def check(text)
43
+ @spellchecker.check(text)
44
+ end
45
+
46
+ # Ignore words matching pattern.
47
+ #
48
+ # @param pattern [Regexp] Pattern to ignore
49
+ # @return [FluentChecker] Self for chaining
50
+ #
51
+ # @example
52
+ # fluent.ignore_words(/https?:\/\/\S+/)
53
+ def ignore_words(pattern)
54
+ @options[:ignore_patterns] ||= []
55
+ @options[:ignore_patterns] << pattern
56
+ self
57
+ end
58
+
59
+ # Set maximum suggestions.
60
+ #
61
+ # @param max [Integer] Maximum suggestions
62
+ # @return [FluentChecker] Self for chaining
63
+ def max_suggestions(max)
64
+ @options[:max_suggestions] = max
65
+ self
66
+ end
67
+
68
+ # Set progress callback.
69
+ #
70
+ # @param block [Proc] Callback proc
71
+ # @return [FluentChecker] Self for chaining
72
+ def on_progress(&block)
73
+ @progress_callback = block
74
+ self
75
+ end
76
+
77
+ # Set error callback.
78
+ #
79
+ # @param block [Proc] Callback proc
80
+ # @return [FluentChecker] Self for chaining
81
+ def on_error(&block)
82
+ @error_callback = block
83
+ self
84
+ end
85
+
86
+ # Get the result.
87
+ #
88
+ # @return [Models::Result::ResultDocumentResult] Check result
89
+ def result
90
+ check(@text)
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Base class for pattern matchers.
7
+ #
8
+ # Pattern matchers check token sequences against specific patterns
9
+ # defined in YAML configuration.
10
+ class BaseMatcher
11
+ def initialize(pattern, exceptions = {})
12
+ @pattern = pattern
13
+ @exceptions = exceptions
14
+ end
15
+
16
+ # Match tokens against the pattern.
17
+ #
18
+ # @param _tokens [Array<Hash>] Array of token hashes (unused)
19
+ # @param _rule [Rule] The rule being checked (unused)
20
+ # @return [Array<Hash>] Array of error hashes
21
+ def match(_tokens, _rule)
22
+ []
23
+ end
24
+
25
+ protected
26
+
27
+ # Extract target tokens from context specification.
28
+ #
29
+ # @param tokens [Array<Hash>] Array of token hashes
30
+ # @param context_spec [Hash] Context specification from pattern
31
+ # @return [Array<Hash>] Array of matching tokens with their indices
32
+ def extract_tokens_from_context(tokens, context_spec)
33
+ result = []
34
+ context_spec.each do |spec|
35
+ if spec['target_token']
36
+ tokens.each_with_index do |token, idx|
37
+ if token[:token]&.downcase == spec['target_token']
38
+ result << { token: token, index: idx }
39
+ end
40
+ end
41
+ end
42
+ end
43
+ result
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,179 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for double negative rules.
7
+ #
8
+ # Detects when multiple negative words appear within a certain
9
+ # distance, with support for declared exception phrases. The
10
+ # canonical exception is the "not only...but also" correlative
11
+ # conjunction: both "not" and "also" are flagged as negatives,
12
+ # but the construction is grammatical and must not fire.
13
+ #
14
+ # Exception phrases use the literal +...+ separator to indicate
15
+ # arbitrary text between the prefix and suffix:
16
+ #
17
+ # "not only...but also"
18
+ #
19
+ # means "not only" at some position, then any text, then "but
20
+ # also". Every negative inside such a region is suppressed.
21
+ class DoubleNegativeMatcher < BaseMatcher
22
+ # Match tokens against the double negative pattern.
23
+ #
24
+ # @param tokens [Array<Hash>] Array of token hashes
25
+ # @param rule [Rule] The rule being checked
26
+ # @return [Array<Hash>] Array of error hashes
27
+ def match(tokens, rule)
28
+ errors = []
29
+ exceptions = rule.exceptions || {}
30
+ exception_phrases = exceptions['phrases'] || []
31
+
32
+ conditions = @pattern['conditions'] || []
33
+ distance_condition = conditions.find { |c| c['type'] == 'distance_check' }
34
+ max_distance = distance_condition&.dig('max_distance') || 15
35
+
36
+ regions = exception_regions(tokens, exception_phrases)
37
+
38
+ negative_indices = tokens.each_with_index.filter_map do |token, idx|
39
+ word = token[:token]&.downcase
40
+ next unless negative?(word)
41
+ next if regions.any? { |range| range.include?(idx) }
42
+
43
+ idx
44
+ end
45
+
46
+ negative_indices.each_cons(2) do |idx1, idx2|
47
+ pos1 = tokens[idx1][:position]
48
+ pos2 = tokens[idx2][:position]
49
+ distance = pos2 - pos1
50
+ next if distance > max_distance
51
+
52
+ error = build_error(tokens, idx1, idx2, rule)
53
+ errors << error if error
54
+ end
55
+ errors
56
+ end
57
+
58
+ private
59
+
60
+ # Check if a word is a negative.
61
+ #
62
+ # @param word [String] The word to check
63
+ # @return [Boolean] True if the word is a negative
64
+ def negative?(word)
65
+ return false if word.nil? || word.empty?
66
+
67
+ negatives = %w[not no neither nobody never nothing nowhere hardly barely scarcely]
68
+ return true if negatives.include?(word)
69
+ return true if word.end_with?("n't")
70
+
71
+ false
72
+ end
73
+
74
+ # Locate every (start...end) region of the token stream that
75
+ # matches one of the declared +exception_phrases+. Each phrase
76
+ # is "<prefix>...<suffix>"; a region runs from the start of the
77
+ # prefix match through the end of the corresponding suffix
78
+ # match. If the suffix never matches, the region is not
79
+ # recorded (the idiom was never closed).
80
+ #
81
+ # @param tokens [Array<Hash>]
82
+ # @param exception_phrases [Array<String>]
83
+ # @return [Array<Range>] Index ranges to suppress
84
+ def exception_regions(tokens, exception_phrases)
85
+ return [] if exception_phrases.empty?
86
+
87
+ exception_phrases.flat_map do |phrase|
88
+ phrase_regions(tokens, phrase)
89
+ end
90
+ end
91
+
92
+ # Find every region for a single phrase.
93
+ #
94
+ # Two phrase shapes are supported:
95
+ #
96
+ # - "prefix...suffix" — correlative / open-ended idiom. Region
97
+ # runs from the start of the prefix match through the end of
98
+ # the next matching suffix. If the suffix never matches, the
99
+ # region is not recorded (idiom was never closed).
100
+ # - "literal phrase" (no "...") — fixed n-gram. Region is the
101
+ # exact span of the match.
102
+ def phrase_regions(tokens, phrase)
103
+ if phrase.include?("...")
104
+ split_phrase_regions(tokens, phrase)
105
+ else
106
+ literal_phrase_regions(tokens, phrase)
107
+ end
108
+ end
109
+
110
+ # "prefix...suffix" form.
111
+ def split_phrase_regions(tokens, phrase)
112
+ prefix, suffix = phrase.split("...", 2)
113
+ prefix_tokens = prefix.to_s.split.map(&:downcase)
114
+ suffix_tokens = suffix.to_s.split.map(&:downcase)
115
+ return [] if prefix_tokens.empty? || suffix_tokens.empty?
116
+
117
+ regions = []
118
+ tokens.each_index do |start_idx|
119
+ next unless tokens_match?(tokens, start_idx, prefix_tokens)
120
+
121
+ suffix_search_from = start_idx + prefix_tokens.length
122
+ suffix_idx = (suffix_search_from...tokens.length).find do |i|
123
+ tokens_match?(tokens, i, suffix_tokens)
124
+ end
125
+ next unless suffix_idx
126
+
127
+ regions << (start_idx...(suffix_idx + suffix_tokens.length))
128
+ end
129
+ regions
130
+ end
131
+
132
+ # Literal n-gram form (no "...").
133
+ def literal_phrase_regions(tokens, phrase)
134
+ pattern = phrase.split.map(&:downcase)
135
+ return [] if pattern.empty?
136
+
137
+ regions = []
138
+ tokens.each_index do |start_idx|
139
+ next unless tokens_match?(tokens, start_idx, pattern)
140
+
141
+ regions << (start_idx...(start_idx + pattern.length))
142
+ end
143
+ regions
144
+ end
145
+
146
+ # True if tokens[idx, idx + pattern.length] matches +pattern+
147
+ # (case-insensitive, token-by-token).
148
+ def tokens_match?(tokens, idx, pattern)
149
+ return false if idx.negative?
150
+ return false if idx + pattern.length > tokens.length
151
+
152
+ pattern.each_with_index.all? do |word, offset|
153
+ tokens[idx + offset][:token]&.downcase == word
154
+ end
155
+ end
156
+
157
+ # Build an error hash.
158
+ #
159
+ # @param tokens [Array<Hash>] Array of token hashes
160
+ # @param idx1 [Integer] First negative index
161
+ # @param idx2 [Integer] Second negative index
162
+ # @param rule [Rule] The rule being checked
163
+ # @return [Hash] Error hash
164
+ def build_error(tokens, idx1, idx2, rule)
165
+ words = tokens[idx1..idx2].map { |t| t[:token] }.join(' ')
166
+
167
+ {
168
+ rule_id: rule.id,
169
+ position: tokens[idx1][:position],
170
+ message: rule.message,
171
+ suggestion: rule.suggestion,
172
+ context: words,
173
+ suggestions: rule.suggestion ? [rule.suggestion] : []
174
+ }
175
+ end
176
+ end
177
+ end
178
+ end
179
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for literal multi-word phrase confusions.
7
+ #
8
+ # Catches phrases where every word is individually valid but the
9
+ # combination is a common error — typically phonetic confusions
10
+ # like "could of" (should be "could have"). The spelling checker
11
+ # passes these because each word is in the dictionary; only a
12
+ # phrase-level grammar check catches them.
13
+ #
14
+ # The rule config declares the wrong phrase and its replacement:
15
+ #
16
+ # conditions:
17
+ # - type: phrase_check
18
+ # wrong_phrase: "could of"
19
+ # suggestion: "could have"
20
+ #
21
+ # The matcher scans the token stream for consecutive tokens whose
22
+ # downcased text matches +wrong_phrase+ (split on whitespace)
23
+ # and emits one error per match.
24
+ class PhraseMatcher < BaseMatcher
25
+ # Match tokens against the phrase-confusion pattern.
26
+ #
27
+ # @param tokens [Array<Hash>] Array of token hashes
28
+ # @param rule [Rule] The rule being checked
29
+ # @return [Array<Hash>] Array of error hashes
30
+ def match(tokens, rule)
31
+ wrong_phrase = phrase_condition&.dig('wrong_phrase')
32
+ suggestion = phrase_condition&.dig('suggestion')
33
+ return [] unless wrong_phrase && suggestion
34
+
35
+ wrong_tokens = wrong_phrase.downcase.split
36
+ return [] if wrong_tokens.empty?
37
+
38
+ matches = []
39
+ tokens.each_with_index do |start_token, start_idx|
40
+ next unless start_token[:token]&.downcase == wrong_tokens.first
41
+
42
+ match_idx = find_match(tokens, start_idx, wrong_tokens)
43
+ next unless match_idx
44
+
45
+ matches << build_error(tokens, match_idx, wrong_tokens.length,
46
+ wrong_phrase, suggestion, rule)
47
+ end
48
+ matches
49
+ end
50
+
51
+ private
52
+
53
+ def phrase_condition
54
+ @phrase_condition ||= @pattern['conditions']&.find do |c|
55
+ c['type'] == 'phrase_check'
56
+ end
57
+ end
58
+
59
+ # Walk forward from +start_idx+ and confirm the next
60
+ # +expected.length+ tokens match +expected+ (case-insensitive).
61
+ # Returns the start_idx on full match, nil otherwise.
62
+ def find_match(tokens, start_idx, expected)
63
+ expected.each_with_index do |word, offset|
64
+ tok = tokens[start_idx + offset]
65
+ return nil unless tok && tok[:token]&.downcase == word
66
+ end
67
+ start_idx
68
+ end
69
+
70
+ def build_error(tokens, start_idx, length, wrong_phrase, suggestion, rule)
71
+ end_idx = start_idx + length - 1
72
+ words = tokens[start_idx..end_idx].map { |t| t[:token] }.join(' ')
73
+ {
74
+ rule_id: rule.id,
75
+ position: tokens[start_idx][:position],
76
+ message: rule.message,
77
+ suggestion: suggestion,
78
+ context: words,
79
+ suggestions: [suggestion],
80
+ wrong_phrase: wrong_phrase
81
+ }
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for there/their/they're confusion rules.
7
+ #
8
+ # This matcher detects when "there" is used where "their"
9
+ # (possessive) is intended.
10
+ class PossessiveContextMatcher < BaseMatcher
11
+ # Match tokens against the there/their pattern.
12
+ #
13
+ # @param tokens [Array<Hash>] Array of token hashes
14
+ # @param rule [Rule] The rule being checked
15
+ # @return [Array<Hash>] Array of error hashes
16
+ def match(tokens, rule)
17
+ errors = []
18
+ exceptions = rule.exceptions || {}
19
+
20
+ location_indicators = exceptions['location_indicators'] || {}
21
+ location_verbs = location_indicators['verbs'] || []
22
+ possessive_nouns = location_indicators['possessive_nouns'] || []
23
+
24
+ tokens.each_with_index do |token, idx|
25
+ word = token[:token]&.downcase
26
+ next unless word == 'there'
27
+
28
+ next_token = tokens[idx + 1]
29
+ next unless next_token
30
+
31
+ next_word = next_token[:token]&.downcase
32
+
33
+ # Skip if followed by verb (location/existence context)
34
+ next if location_verbs.include?(next_word)
35
+
36
+ uses_their = false
37
+
38
+ # Check POS tags first
39
+ next_pos = next_token[:pos_tag]
40
+ if next_pos && ['NOUN', 'NOUN_PROPER', 'ADJ'].include?(next_pos)
41
+ uses_their = true
42
+ # Fallback to word list
43
+ elsif possessive_nouns.include?(next_word)
44
+ uses_their = true
45
+ end
46
+
47
+ if uses_their
48
+ errors << build_error(token, next_token, rule)
49
+ end
50
+ end
51
+ errors
52
+ end
53
+
54
+ private
55
+
56
+ # Build an error hash.
57
+ #
58
+ # @param token [Hash] The token with "there"
59
+ # @param next_token [Hash] The next token
60
+ # @param rule [Rule] The rule being checked
61
+ # @return [Hash] Error hash
62
+ def build_error(token, next_token, rule)
63
+ {
64
+ rule_id: rule.id,
65
+ position: token[:position],
66
+ message: rule.message,
67
+ suggestion: rule.suggestion,
68
+ context: "\"#{token[:token]} #{next_token[:token]}\"",
69
+ suggestions: [rule.suggestion]
70
+ }
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for possessive-vs-contraction confusion rules.
7
+ #
8
+ # Detects when a possessive form ("its", "your", "whose") is
9
+ # used where the corresponding contraction ("it's", "you're",
10
+ # "who's") is intended — or vice versa. The trigger is the
11
+ # following token's POS tag or word form.
12
+ #
13
+ # Rule shape:
14
+ #
15
+ # patterns:
16
+ # - context:
17
+ # target_token: "its"
18
+ # trigger_when_followed_by:
19
+ # tags: [ADJ, ADV]
20
+ # words: [been, going, getting]
21
+ # conditions:
22
+ # - type: possessive_contraction_check
23
+ #
24
+ # When the target token is followed by a token whose POS tag is
25
+ # in +tags+ OR whose lowercased form is in +words+, emit an
26
+ # error with +rule.suggestion+ as the replacement.
27
+ #
28
+ # Examples:
29
+ # "its cold" → suggest "it's" (cold is ADJ)
30
+ # "your ready" → suggest "you're"
31
+ # "whose coming" → suggest "who's"
32
+ class PossessiveContractionMatcher < BaseMatcher
33
+ def match(tokens, rule)
34
+ errors = []
35
+ target = pattern_target_token
36
+ triggers = pattern_triggers
37
+
38
+ tokens.each_with_index do |token, idx|
39
+ next unless matches_target?(token, target)
40
+
41
+ next_token = tokens[idx + 1]
42
+ next unless next_token
43
+ next unless triggers_match?(next_token, triggers)
44
+
45
+ errors << build_error(token, next_token, rule)
46
+ end
47
+ errors
48
+ end
49
+
50
+ protected
51
+
52
+ def pattern_target_token
53
+ @pattern.dig('context', 'target_token')
54
+ end
55
+
56
+ def pattern_triggers
57
+ @pattern.dig('context', 'trigger_when_followed_by') || {}
58
+ end
59
+
60
+ def matches_target?(token, target)
61
+ target && token[:token]&.downcase == target.downcase
62
+ end
63
+
64
+ def triggers_match?(next_token, triggers)
65
+ trigger_tags = triggers['tags'] || []
66
+ trigger_words = (triggers['words'] || []).map(&:downcase)
67
+
68
+ tag = next_token[:pos_tag]
69
+ word = next_token[:token]&.downcase
70
+
71
+ trigger_tags.include?(tag) || trigger_words.include?(word)
72
+ end
73
+
74
+ def build_error(token, next_token, rule)
75
+ {
76
+ rule_id: rule.id,
77
+ position: token[:position],
78
+ message: rule.message,
79
+ suggestion: rule.suggestion,
80
+ context: "\"#{token[:token]} #{next_token[:token]}\"",
81
+ suggestions: [rule.suggestion]
82
+ }
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end