kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Algorithms namespace for spell checking algorithms.
5
+ #
6
+ # Contains the core algorithms ported from Spylls:
7
+ # - Lookup::Lookuper: word correctness checking with affix support
8
+ # - Suggest::Suggester: main suggestion orchestration
9
+ # - Permutations: edit-distance permutations
10
+ # - NgramSuggest: n-gram based suggestion algorithm
11
+ # - PhonetSuggest: phonetic suggestion algorithm
12
+ # - Capitalization: capitalization handling
13
+ module Algorithms
14
+ autoload :Lookup, "kotoshu/algorithms/lookup"
15
+ autoload :Suggest, "kotoshu/algorithms/suggest"
16
+ autoload :Permutations, "kotoshu/algorithms/permutations"
17
+ autoload :NgramSuggest, "kotoshu/algorithms/ngram_suggest"
18
+ autoload :PhonetSuggest, "kotoshu/algorithms/phonet_suggest"
19
+ autoload :Capitalization, "kotoshu/algorithms/capitalization"
20
+ autoload :EditDistance, "kotoshu/algorithms/edit_distance"
21
+ autoload :Soundex, "kotoshu/algorithms/soundex"
22
+ end
23
+ end
@@ -0,0 +1,405 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Analyzers
5
+ # Unified semantic error analyzer.
6
+ #
7
+ # Uses word embeddings for context-aware error detection and suggestions.
8
+ # Provides unified semantic analysis without artificial spelling/grammar split.
9
+ #
10
+ # @example Analyzing a document
11
+ # model = FastTextModel.from_github('en')
12
+ # analyzer = SemanticAnalyzer.new(model)
13
+ # errors = analyzer.analyze(document)
14
+ #
15
+ # @example Checking a single word
16
+ # suggestions = analyzer.suggest_corrections('helo', context_words: ['hello', 'world'])
17
+ class SemanticAnalyzer
18
+ # Similarity threshold for high-confidence suggestions
19
+ HIGH_CONFIDENCE_THRESHOLD = 0.85
20
+
21
+ # Similarity threshold for medium-confidence suggestions
22
+ MEDIUM_CONFIDENCE_THRESHOLD = 0.70
23
+
24
+ # Minimum similarity for suggestions
25
+ MIN_SIMILARITY = 0.50
26
+
27
+ # Default number of suggestions to generate
28
+ DEFAULT_MAX_SUGGESTIONS = 5
29
+
30
+ attr_reader :model, :max_suggestions
31
+
32
+ # Create a new semantic analyzer.
33
+ #
34
+ # @param model [EmbeddingModel] The embedding model to use
35
+ # @param max_suggestions [Integer] Maximum suggestions per error
36
+ # @param min_similarity [Float] Minimum similarity threshold
37
+ def initialize(model, max_suggestions: DEFAULT_MAX_SUGGESTIONS, min_similarity: MIN_SIMILARITY)
38
+ raise ArgumentError, "Model must be an EmbeddingModel" unless model.is_a?(Models::EmbeddingModel)
39
+
40
+ @model = model
41
+ @max_suggestions = max_suggestions
42
+ @min_similarity = min_similarity
43
+ end
44
+
45
+ # Analyze a {Kotoshu::Documents::Document} for semantic errors.
46
+ #
47
+ # Walks every {Documents::TextNode}, tokenizes its text, and for
48
+ # each invalid word resolves a {Documents::SourceRange} via
49
+ # `document.source_range_for` so the emitted {Models::SemanticError}
50
+ # points at the original markup-bearing source rather than the
51
+ # flattened text. Context for ranking is built from the
52
+ # surrounding flattened text.
53
+ #
54
+ # @param document [Kotoshu::Documents::Document]
55
+ # @return [Array<Models::SemanticError>] Sorted by source_range
56
+ def analyze(document)
57
+ unless document.is_a?(Kotoshu::Documents::Document)
58
+ raise ArgumentError,
59
+ "document must be a Kotoshu::Documents::Document"
60
+ end
61
+
62
+ errors = []
63
+ flattened = document.flattened_text
64
+
65
+ document.text_nodes.each do |text_node|
66
+ tokenize_with_offsets(text_node.text).each do |word, offset_in_node|
67
+ next if valid_word?(word)
68
+
69
+ flattened_start = text_node.flattened_offset + offset_in_node
70
+ flattened_end = flattened_start + word.length
71
+ source_range = document.source_range_for(flattened_start, flattened_end)
72
+ context = build_context(flattened, flattened_start, flattened_end)
73
+
74
+ error = detect_error(word: word, source_range: source_range, context: context)
75
+ errors << error if error
76
+ end
77
+ end
78
+
79
+ errors.sort
80
+ end
81
+
82
+ # Detect semantic error for a single word.
83
+ #
84
+ # @param word [String] The word to check
85
+ # @param source_range [Kotoshu::Documents::SourceRange, nil] Where
86
+ # the word lives in the original source. May be nil for
87
+ # word-level checks that aren't tied to a document.
88
+ # @param context [Models::Context, nil] Context around the word
89
+ # @return [Models::SemanticError, nil] Error object or nil if valid
90
+ def detect_error(word:, source_range: nil, context: nil)
91
+ return nil if valid_word?(word)
92
+
93
+ suggestions = suggest_corrections(word, context: context)
94
+ error_type = classify_error(word, suggestions, context)
95
+ confidence = calculate_confidence(suggestions)
96
+
97
+ Models::SemanticError.new(
98
+ id: generate_error_id(word, source_range),
99
+ source_range: source_range,
100
+ original: word,
101
+ suggestions: suggestions,
102
+ error_type: error_type,
103
+ confidence: confidence,
104
+ context: context
105
+ )
106
+ rescue Models::EmptySuggestionsError
107
+ # Word is genuinely unknown — no close matches. Skip silently
108
+ # rather than crashing on the suggestions-cannot-be-empty
109
+ # invariant.
110
+ nil
111
+ end
112
+
113
+ # Suggest corrections for a word.
114
+ #
115
+ # For in-vocabulary words the embedding model returns the
116
+ # nearest neighbors. For OOV words (the typical "misspelling"
117
+ # case the analyzer exists to catch) the embedding model returns
118
+ # +[]+ because it has no vector for the input. We fall back to
119
+ # an edit-distance walk over the model's vocabulary so the OOV
120
+ # case still produces useful candidates that the rest of the
121
+ # pipeline (confidence scoring, context ranking) can refine.
122
+ #
123
+ # @param word [String] The misspelled word
124
+ # @param context [Models::Context, nil] Context for context-aware suggestions
125
+ # @return [Array<Models::Suggestion>] Suggested corrections
126
+ def suggest_corrections(word, context: nil)
127
+ return [] if word.nil? || word.empty?
128
+
129
+ neighbors = @model.nearest_neighbors(word, k: @max_suggestions * 3)
130
+ neighbors = edit_distance_fallback(word) if neighbors.empty?
131
+
132
+ # Filter by minimum similarity
133
+ neighbors = neighbors.select { |n| n.similarity >= @min_similarity }
134
+
135
+ # If we have context, rank by contextual relevance
136
+ if context.is_a?(Kotoshu::Models::Context)
137
+ neighbors = rank_by_context(neighbors, context)
138
+ end
139
+
140
+ # Convert to Suggestions
141
+ neighbors.first(@max_suggestions).map do |neighbor|
142
+ Models::Suggestion.new(
143
+ neighbor.word,
144
+ confidence: neighbor.similarity,
145
+ source: :semantic,
146
+ metadata: {
147
+ distance: neighbor.distance,
148
+ similarity: neighbor.similarity
149
+ }
150
+ )
151
+ end
152
+ end
153
+
154
+ # Check if a word is valid (exists in vocabulary).
155
+ #
156
+ # @param word [String] The word to check
157
+ # @return [Boolean] True if word is valid
158
+ def valid_word?(word)
159
+ return false if word.nil? || word.empty?
160
+
161
+ # Skip numbers
162
+ return true if /^\d+$/.match?(word)
163
+
164
+ # Skip single characters (likely abbreviations)
165
+ return true if word.length == 1
166
+
167
+ # Check if word exists in model vocabulary
168
+ @model.has_word?(word)
169
+ end
170
+
171
+ # Calculate confidence score for suggestions.
172
+ #
173
+ # @param suggestions [Array<Models::Suggestion>] List of suggestions
174
+ # @return [Float] Confidence score (0.0 to 1.0)
175
+ def calculate_confidence(suggestions)
176
+ return 0.0 unless suggestions&.any?
177
+
178
+ # Confidence is based on top suggestion quality
179
+ top = suggestions.first
180
+
181
+ # High confidence: top suggestion > 0.85 similarity
182
+ return 1.0 if top.confidence > HIGH_CONFIDENCE_THRESHOLD
183
+
184
+ # Medium confidence: top suggestion > 0.70 similarity
185
+ return 0.7 if top.confidence > MEDIUM_CONFIDENCE_THRESHOLD
186
+
187
+ # Low confidence: top suggestion < 0.70
188
+ 0.5
189
+ end
190
+
191
+ private
192
+
193
+ # Fallback for OOV words: walk the model's vocabulary and return
194
+ # candidates within edit distance 2. Scored by 1/(1+distance) so
195
+ # distance-1 matches rank above distance-2 matches. Mirrors the
196
+ # shape {EmbeddingModel#nearest_neighbors} returns
197
+ # (+NearestNeighbor+ instances with +similarity+ in [0, 1]).
198
+ #
199
+ # @param word [String] OOV query word
200
+ # @return [Array<Models::NearestNeighbor>]
201
+ def edit_distance_fallback(word)
202
+ vocab = @model.vocabulary
203
+ return [] unless vocab&.any?
204
+
205
+ downcased = word.downcase
206
+ candidates = vocab.each_with_object([]) do |candidate, acc|
207
+ next if candidate == word || candidate.downcase == downcased
208
+
209
+ distance = levenshtein(downcased, candidate.downcase)
210
+ next unless distance.positive? && distance <= 2
211
+
212
+ acc << Models::NearestNeighbor.new(
213
+ word: candidate,
214
+ similarity: 1.0 / (1.0 + distance),
215
+ distance: distance,
216
+ embedding: nil
217
+ )
218
+ end
219
+
220
+ candidates.sort.first(@max_suggestions * 3)
221
+ end
222
+
223
+ # Two-row Levenshtein edit distance.
224
+ def levenshtein(a, b)
225
+ return b.length if a.empty?
226
+ return a.length if b.empty?
227
+
228
+ prev = (0..b.length).to_a
229
+ a.each_char.with_index do |achar, i|
230
+ curr = [i + 1]
231
+ b.each_char.with_index do |bchar, j|
232
+ cost = achar == bchar ? 0 : 1
233
+ curr << [curr[j] + 1, prev[j + 1] + 1, prev[j] + cost].min
234
+ end
235
+ prev = curr
236
+ end
237
+ prev.last
238
+ end
239
+
240
+ # Tokenize text into [word, offset_in_text] pairs. The offset
241
+ # is the 0-based character position of the word's first char
242
+ # within +text+. Used by {#analyze} to compute flattened offsets
243
+ # for source-range resolution.
244
+ #
245
+ # @param text [String]
246
+ # @return [Array<Array(String, Integer)>]
247
+ def tokenize_with_offsets(text)
248
+ return [] unless text
249
+
250
+ text.downcase.scan(/[a-z]+(?:['’-][a-z]+)*/i).map do |match|
251
+ char_offset = Regexp.last_match.offset(0).first
252
+ [match.downcase, char_offset]
253
+ end
254
+ end
255
+
256
+ # Tokenize text into words. Kept for backward compat with specs.
257
+ #
258
+ # @param text [String] Text to tokenize
259
+ # @return [Array<String>] Words
260
+ def tokenize_words(text)
261
+ tokenize_with_offsets(text).map(&:first)
262
+ end
263
+
264
+ # Build a {Models::Context} around a flattened-text range.
265
+ # Slices up to 32 chars before and after the [start, end) range,
266
+ # rounding at whitespace so the context reads naturally.
267
+ #
268
+ # @param flattened [String]
269
+ # @param flattened_start [Integer]
270
+ # @param flattened_end [Integer]
271
+ # @return [Models::Context, nil] nil when the slice is empty
272
+ def build_context(flattened, flattened_start, flattened_end)
273
+ return nil if flattened.nil? || flattened.empty?
274
+
275
+ window = 32
276
+ before_start = [flattened_start - window, 0].max
277
+ after_end = [flattened_end + window, flattened.length].min
278
+ before = flattened[before_start...flattened_start] || ""
279
+ current = flattened[flattened_start...flattened_end] || ""
280
+ after = flattened[flattened_end...after_end] || ""
281
+ return nil if current.empty?
282
+
283
+ Models::Context.new(
284
+ before: before,
285
+ current: current,
286
+ after: after,
287
+ location: nil
288
+ )
289
+ end
290
+
291
+ # Classify error type based on word and suggestions.
292
+ #
293
+ # @param word [String] The error word
294
+ # @param suggestions [Array<Models::Suggestion>] Suggestions
295
+ # @param _context [Models::Context, nil] Context (unused)
296
+ # @return [Symbol] Error type
297
+ def classify_error(word, suggestions, _context)
298
+ return :orthographic if suggestions && suggestions.empty?
299
+
300
+ top_suggestion = suggestions.first
301
+
302
+ # Check if it's a capitalization error
303
+ if word.downcase == top_suggestion.word.downcase
304
+ return :capitalization
305
+ end
306
+
307
+ # Check if it's a diacritic/accent error
308
+ if similar_without_diacritics?(word, top_suggestion.word)
309
+ return :orthographic
310
+ end
311
+
312
+ # Check if it's a word choice error (semantic similarity but different word)
313
+ if suggestions.first&.source == "semantic"
314
+ return :word_choice
315
+ end
316
+
317
+ # Default to orthographic (spelling)
318
+ :orthographic
319
+ end
320
+
321
+ # Check if two words are similar ignoring diacritics.
322
+ #
323
+ # @param word1 [String] First word
324
+ # @param word2 [String] Second word
325
+ # @return [Boolean] True if similar without diacritics
326
+ def similar_without_diacritics?(word1, word2)
327
+ # Remove diacritics and compare
328
+ normalize_diacritics(word1) == normalize_diacritics(word2)
329
+ end
330
+
331
+ # Normalize diacritics from a word.
332
+ #
333
+ # @param word [String] Word with diacritics
334
+ # @return [String] Word without diacritics
335
+ def normalize_diacritics(word)
336
+ # Simple normalization (transliterate to ASCII)
337
+ word.encode('ASCII', fallback: ->(c) {
338
+ if c == 'ä'
339
+ 'ae'
340
+ elsif c == 'ö'
341
+ 'oe'
342
+ elsif c == 'ü'
343
+ 'ue'
344
+ else
345
+ c == 'ß' ? 'ss' : c
346
+ end
347
+ })
348
+ .downcase
349
+ end
350
+
351
+ # Rank neighbors by contextual relevance.
352
+ #
353
+ # @param neighbors [Array<Models::NearestNeighbor>] Neighbors to rank
354
+ # @param context [Models::Context] Context for ranking
355
+ # @return [Array<Models::NearestNeighbor>] Ranked neighbors
356
+ def rank_by_context(neighbors, context)
357
+ # Get surrounding words
358
+ surrounding = context.surrounding_words(3)
359
+ return neighbors unless surrounding&.any?
360
+
361
+ # Boost neighbors that appear in similar context
362
+ # In full implementation, would use more sophisticated context modeling
363
+ neighbors.map do |neighbor|
364
+ boost = context_boost(neighbor.word, surrounding)
365
+ # Create boosted neighbor (create new object to avoid mutation)
366
+ boosted_similarity = [neighbor.similarity + boost, 1.0].min
367
+ Models::NearestNeighbor.new(
368
+ word: neighbor.word,
369
+ similarity: boosted_similarity,
370
+ embedding: neighbor.embedding
371
+ )
372
+ end.sort.reverse
373
+ end
374
+
375
+ # Calculate context boost for a word.
376
+ #
377
+ # @param word [String] Word to boost
378
+ # @param surrounding [Array<String>] Surrounding words
379
+ # @return [Float] Boost amount (0.0 to 0.1)
380
+ def context_boost(word, surrounding)
381
+ return 0.0 unless surrounding&.any?
382
+
383
+ # Simple boost: if word is semantically similar to surrounding words
384
+ surrounding.reduce(0.0) do |boost, surrounding_word|
385
+ sim = @model.similarity(word, surrounding_word)
386
+ boost + ((sim || 0.0) * 0.02) # Small boost for each similar word
387
+ end
388
+ end
389
+
390
+ # Generate unique error ID.
391
+ #
392
+ # @param word [String] The error word
393
+ # @param source_range [Kotoshu::Documents::SourceRange, nil]
394
+ # @return [String] Truncated SHA-256 hex
395
+ def generate_error_id(word, source_range)
396
+ key = if source_range
397
+ "#{word}-#{source_range.start.offset}-#{source_range.end.offset}"
398
+ else
399
+ "word:#{word}"
400
+ end
401
+ Digest::SHA256.hexdigest(key)[0...16]
402
+ end
403
+ end
404
+ end
405
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Analyzers: higher-level text analysis (semantic, grammatical).
5
+ module Analyzers
6
+ autoload :SemanticAnalyzer, "kotoshu/analyzers/semantic_analyzer"
7
+ end
8
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Baseline
5
+ # Outcome of applying a {Store} to one check result. Plain data
6
+ # (Struct) — not a serialized model.
7
+ #
8
+ # - +result+ — the new {Models::Result::DocumentResult}: errors
9
+ # are the ones NOT covered by the baseline; baseline-suppressed
10
+ # errors moved into `suppressed_errors` (marked with
11
+ # `suppressed_by: "baseline"`), joining any inline-suppressed
12
+ # entries produced by the checker.
13
+ # - +suppressed_count+ — how many errors the baseline absorbed
14
+ # - +stale_count+ — baseline entries for the checked file whose
15
+ # errors no longer exist
16
+ Application = Struct.new(:result, :suppressed_count, :stale_count, keyword_init: true)
17
+ end
18
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "lutaml/model"
4
+
5
+ module Kotoshu
6
+ module Baseline
7
+ # One baseline record: +count+ occurrences of +word+ in +file+ are
8
+ # known debt. +line+ is the first occurrence at the time the
9
+ # baseline was written — informational only, matching is
10
+ # count-based (stable across reformatting).
11
+ #
12
+ # Serialized via lutaml-model; the canonical JSON shape is
13
+ # `{"file": ..., "line": ..., "word": ..., "count": ...}`.
14
+ class Entry < Lutaml::Model::Serializable
15
+ attribute :file, :string
16
+ attribute :line, :integer, default: 1
17
+ attribute :word, :string, default: ""
18
+ attribute :count, :integer, default: 1
19
+
20
+ # Accept framework-private keywords (e.g. +lutaml_register+)
21
+ # from lutaml-model so from_hash round-trips without lutaml
22
+ # having to poke ivars directly.
23
+ def initialize(file: nil, line: 1, word: "", count: 1, **kwargs)
24
+ super(file: file, line: line, word: word.to_s, count: count, **kwargs)
25
+ end
26
+
27
+ # Entries are keyed by (file, word); counts merge on key.
28
+ def key
29
+ [file.to_s, word.to_s]
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,175 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "lutaml/model"
4
+ require "set"
5
+
6
+ module Kotoshu
7
+ module Baseline
8
+ # A baseline file: canonical JSON `{version, entries: [{file,
9
+ # line, word, count}]}`. Serialized via lutaml-model.
10
+ #
11
+ # Semantics when applied to a check result (plan 82):
12
+ # - an error passes when the baseline has budget left for it
13
+ # (same file + word, occurrence count not exceeded);
14
+ # - errors beyond the budget, and unknown words, fail;
15
+ # - a summary reports how many entries are stale (the file no
16
+ # longer has the error), so debt shrinks visibly.
17
+ class Store < Lutaml::Model::Serializable
18
+ DEFAULT_FILENAME = ".kotoshu-baseline.json"
19
+ CURRENT_VERSION = 1
20
+
21
+ attribute :version, :integer, default: CURRENT_VERSION
22
+ attribute :entries, Entry, collection: true
23
+
24
+ # Accept framework-private keywords from lutaml-model.
25
+ def initialize(version: CURRENT_VERSION, entries: [], **kwargs)
26
+ super
27
+ end
28
+
29
+ class << self
30
+ # Read and parse a baseline file.
31
+ #
32
+ # @param path [String]
33
+ # @return [Store]
34
+ # @raise [Errno::ENOENT, Lutaml::Model::InvalidValueError] when
35
+ # missing or malformed — callers translate to usage errors
36
+ def load(path)
37
+ from_json(File.read(path))
38
+ end
39
+
40
+ # Build a Store from check results. +checks+ maps each checked
41
+ # file path to a `[DocumentResult, source_text]` pair (the
42
+ # text supplies line numbers). Entries are canonicalized: one
43
+ # row per (file, word), count = occurrences, line = line of
44
+ # the first occurrence, rows sorted by (file, word).
45
+ #
46
+ # @param checks [Hash<String => Array(Models::Result::DocumentResult, String)>]
47
+ # @return [Store]
48
+ def from_checks(checks)
49
+ entries = checks.flat_map do |file, (result, text)|
50
+ result.errors.group_by(&:word).map do |word, word_errors|
51
+ first = word_errors.min_by { |e| e.position || 0 }
52
+ Entry.new(
53
+ file: file,
54
+ line: Documents::SourcePosition.line_for_offset(text || "", first.position || 0),
55
+ word: word,
56
+ count: word_errors.size
57
+ )
58
+ end
59
+ end
60
+ new(entries: canonical_entries(entries))
61
+ end
62
+
63
+ # Collapse entries to one row per (file, word) with summed
64
+ # counts, in canonical order.
65
+ def canonical_entries(entries)
66
+ entries.group_by(&:key).map do |(file, word), group|
67
+ first_line = group.map(&:line).compact.min || 1
68
+ Entry.new(file: file, line: first_line, word: word,
69
+ count: group.sum(&:count))
70
+ end.sort_by { |e| [e.file.to_s, e.word.to_s] }
71
+ end
72
+ end
73
+
74
+ # Write the baseline to +path+ (pretty JSON).
75
+ #
76
+ # @return [self]
77
+ def save(path)
78
+ File.write(path, to_json)
79
+ self
80
+ end
81
+
82
+ # A decrementing budget view of +file+'s entries for the check
83
+ # path (plan 116): the callable answers false for the first
84
+ # +count+ occurrences of each entry word — exactly the
85
+ # occurrences {#apply} will absorb, whose suggestions nobody
86
+ # reads — and true for anything beyond, so new errors keep
87
+ # their suggestions. Occurrence order is check order, the same
88
+ # position order #apply absorbs in.
89
+ #
90
+ # @param file [String] path the text being checked came from
91
+ # @return [#call] word -> boolean
92
+ def suggestions_filter_for(file)
93
+ budget = Hash.new(0)
94
+ entries_for(file).each { |entry| budget[entry.word] += entry.count }
95
+ lambda do |word|
96
+ if budget[word].positive?
97
+ budget[word] -= 1
98
+ false
99
+ else
100
+ true
101
+ end
102
+ end
103
+ end
104
+
105
+ # Entries recorded for +file+. Paths compare canonically — a
106
+ # baseline recorded as `docs/a.md` matches a check run over
107
+ # `./docs/a.md` and vice versa — so walking `.` and explicit
108
+ # paths share one baseline file.
109
+ def entries_for(file)
110
+ target = self.class.canonical_path(file)
111
+ entries.select { |entry| self.class.canonical_path(entry.file) == target }
112
+ end
113
+
114
+ # @return [String] path with a leading `./` stripped and
115
+ # duplicate separators collapsed; absolute paths unchanged.
116
+ def self.canonical_path(path)
117
+ path.to_s.delete_prefix('./').gsub(%r{//+}, "/")
118
+ end
119
+
120
+ # Apply the baseline to +result+ (checked +file+): errors with
121
+ # baseline budget pass (moved to `suppressed_errors`, marked
122
+ # `suppressed_by: "baseline"`); the rest keep failing.
123
+ # Inline-suppressed errors were already filtered upstream and
124
+ # do not consume baseline budget.
125
+ #
126
+ # @param result [Models::Result::DocumentResult]
127
+ # @param file [String] path the result was checked against
128
+ # @return [Application]
129
+ def apply(result, file:)
130
+ budget = Hash.new(0)
131
+ entries_for(file).each { |entry| budget[entry.word] += entry.count }
132
+
133
+ occurrences = Hash.new { |hash, key| hash[key] = [] }
134
+ result.errors.each { |error| occurrences[error.word] << error }
135
+
136
+ absorbed = []
137
+ occurrences.each_value do |word_errors|
138
+ remaining = budget[word_errors.first.word]
139
+ next if remaining <= 0
140
+
141
+ in_order = word_errors.sort_by { |error| error.position || 0 }
142
+ absorbed.concat(in_order.first(remaining))
143
+ end
144
+
145
+ stale = entries_for(file).count { |entry| occurrences[entry.word].empty? }
146
+ absorbed_ids = absorbed.map(&:object_id).to_set
147
+
148
+ kept = result.errors.reject { |error| absorbed_ids.include?(error.object_id) }
149
+ suppressed = result.suppressed_errors + absorbed.map do |error|
150
+ Models::Result::WordResult.new(
151
+ word: error.word,
152
+ correct: false,
153
+ suggestions: error.suggestions.to_a,
154
+ position: error.position,
155
+ metadata: error.metadata,
156
+ suppressed: true,
157
+ suppressed_by: Models::Result::WordResult::SUPPRESSED_BY_BASELINE
158
+ )
159
+ end
160
+
161
+ Application.new(
162
+ result: Models::Result::DocumentResult.new(
163
+ file: self.class.canonical_path(result.file),
164
+ errors: kept,
165
+ suppressed_errors: suppressed,
166
+ word_count: result.word_count,
167
+ metadata: result.metadata
168
+ ),
169
+ suppressed_count: absorbed.size,
170
+ stale_count: stale
171
+ )
172
+ end
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # CI baselines (plan 82, Track B).
5
+ #
6
+ # `kotoshu check --baseline FILE` freezes existing spelling debt so
7
+ # CI fails only on NEW errors: errors covered by the baseline pass,
8
+ # anything else fails, and stale entries (whose errors no longer
9
+ # exist) are reported so debt shrinks visibly.
10
+ #
11
+ # Matching is count-based, not position-based — an entry records how
12
+ # often a word is misspelled in a file (`count`), which survives
13
+ # reformatting that moves lines around. `line` is informational
14
+ # (the first occurrence when the baseline was written).
15
+ module Baseline
16
+ autoload :Application, "kotoshu/baseline/application"
17
+ autoload :Entry, "kotoshu/baseline/entry"
18
+ autoload :Store, "kotoshu/baseline/store"
19
+ end
20
+ end