kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ # Immutable value object: a point in the original (markup-bearing)
6
+ # source.
7
+ #
8
+ # Carries both a 0-based character offset (useful for tools that
9
+ # want to slice the source by range) and a 1-based line/column pair
10
+ # (useful for editor highlighting and human-readable messages).
11
+ SourcePosition = Struct.new(:offset, :line, :column, keyword_init: true) do
12
+ include Comparable
13
+
14
+ def initialize(offset:, line:, column:)
15
+ raise ArgumentError, "offset must be >= 0" if offset.negative?
16
+ raise ArgumentError, "line must be >= 1" if line < 1
17
+ raise ArgumentError, "column must be >= 1" if column < 1
18
+
19
+ super
20
+ freeze
21
+ end
22
+
23
+ # Lexicographic order by (offset, line, column). Matches the
24
+ # natural order of positions when scanning the source left to
25
+ # right, top to bottom.
26
+ def <=>(other)
27
+ return nil unless other.is_a?(SourcePosition)
28
+
29
+ [offset, line, column] <=> [other.offset, other.line, other.column]
30
+ end
31
+
32
+ # 1-based line number of the character at +offset+ in +text+.
33
+ # Offsets past the end clamp to the last line; a nil offset
34
+ # counts as the first line.
35
+ def self.line_for_offset(text, offset)
36
+ clamped = [offset.to_i, 0].max.clamp(0, [text.length - 1, 0].max)
37
+ text[0, clamped].count("\n") + 1
38
+ end
39
+
40
+ def to_s
41
+ "line #{line}, column #{column} (offset #{offset})"
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+
5
+ module Kotoshu
6
+ module Documents
7
+ # Immutable value object: a span in the original source. The
8
+ # +start+ position is inclusive; the +end+ position is exclusive
9
+ # (matches Ruby's range convention and editor selection APIs).
10
+ #
11
+ # SourceRange is the canonical "where" that an error report
12
+ # carries. The checker resolves it from the {Document} before
13
+ # constructing an error so callers (plugins, editors, the CLI)
14
+ # never have to know about flattened vs source offsets.
15
+ class SourceRange
16
+ include Comparable
17
+ extend Forwardable
18
+
19
+ attr_reader :start, :end
20
+
21
+ def_delegators :@start, :offset, :line, :column
22
+
23
+ # @param start_pos [SourcePosition] inclusive start
24
+ # @param end_pos [SourcePosition] exclusive end
25
+ def initialize(start_pos:, end_pos:)
26
+ raise TypeError, "start must be a SourcePosition" unless start_pos.is_a?(SourcePosition)
27
+ raise TypeError, "end must be a SourcePosition" unless end_pos.is_a?(SourcePosition)
28
+ raise ArgumentError, "end must be >= start" if end_pos < start_pos
29
+
30
+ @start = start_pos
31
+ @end = end_pos
32
+ freeze
33
+ end
34
+
35
+ # Order by start position; ties broken by end position.
36
+ def <=>(other)
37
+ return nil unless other.is_a?(SourceRange)
38
+
39
+ cmp = @start <=> other.start
40
+ cmp.zero? ? (@end <=> other.end) : cmp
41
+ end
42
+
43
+ # True if +pos+ falls inside this range. End is exclusive.
44
+ def contains?(pos)
45
+ pos.is_a?(SourcePosition) && pos >= @start && pos < @end
46
+ end
47
+
48
+ # Length in characters of the source span. Uses the offset
49
+ # difference, which is the editor-friendly interpretation: a
50
+ # range from offset 7 to offset 15 covers 8 characters.
51
+ def length
52
+ @end.offset - @start.offset
53
+ end
54
+
55
+ # True when this range covers zero characters (start == end).
56
+ def empty?
57
+ @start == @end
58
+ end
59
+
60
+ # Combine +other+ with self into a range that spans both.
61
+ # @param other [SourceRange]
62
+ # @return [SourceRange]
63
+ def union(other)
64
+ raise TypeError, "other must be a SourceRange" unless other.is_a?(SourceRange)
65
+
66
+ min_start = [@start, other.start].min
67
+ max_end = [@end, other.end].max
68
+ self.class.new(start_pos: min_start, end_pos: max_end)
69
+ end
70
+
71
+ def to_s
72
+ "#{@start}..#{@end}"
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,149 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ module Suppressions
6
+ # Line-oriented scanner for inline ignore directives.
7
+ #
8
+ # ONE shared implementation of the directive semantics; the only
9
+ # per-format knowledge is how to find a comment on a line:
10
+ #
11
+ # - :plain — a line whose trimmed content IS the directive
12
+ # (whole-line only; trailing directives would be ambiguous in
13
+ # prose)
14
+ # - :markdown — HTML comments `<!-- ... -->`, whole-line or
15
+ # trailing
16
+ # - :asciidoc — `//` line comments, whole-line or trailing
17
+ # (the marker must be at line start or preceded by whitespace,
18
+ # so URLs like https://example.com are not comments)
19
+ # - :auto — all of the above. The Spellchecker facade uses this
20
+ # profile because it checks raw text without a parser.
21
+ #
22
+ # Inside a comment the directive must appear at the very start:
23
+ # prose that merely mentions `kotoshu:disable-line` is not a
24
+ # directive.
25
+ #
26
+ # Word lists are only meaningful on `disable-next-line` (the one
27
+ # form the plan defines them for); extra words after the other
28
+ # directives are ignored.
29
+ class Scanner
30
+ # A directive line: `kotoshu:<verb>` at the very start of a
31
+ # comment body, capturing the verb and any trailing word list.
32
+ DIRECTIVE_PATTERN =
33
+ /\Akotoshu:(disable-line|disable-next-line|disable-file|enable-file)\b[ \t]*(.*)\z/
34
+
35
+ # Matches an HTML comment body (`<!-- ... -->`; Markdown profile).
36
+ HTML_COMMENT_BODIES = /<!--\s*(.*?)\s*-->/
37
+
38
+ # Matches an AsciiDoc `//` line comment body. The marker must
39
+ # start the line or follow whitespace so URLs are not comments.
40
+ ASCII_DOC_COMMENT_BODY = /(?:\A|[ \t])\/\/[ \t]?(?<body>.*)\Z/
41
+
42
+ # Which comment kinds each format profile recognizes: :bare
43
+ # (whole-line directives), :html_comment (Markdown),
44
+ # :line_comment (AsciiDoc). :auto enables all three — the
45
+ # facade default, since it checks raw text without a parser.
46
+ PROFILE = {
47
+ plain: %i[bare],
48
+ markdown: %i[html_comment],
49
+ asciidoc: %i[line_comment],
50
+ auto: %i[bare html_comment line_comment]
51
+ }.freeze
52
+
53
+ EMPTY = [].freeze
54
+ private_constant :EMPTY
55
+
56
+ class << self
57
+ # Scan +source+ and return the suppressions its directives
58
+ # produce, in source order.
59
+ #
60
+ # @param source [String, nil]
61
+ # @param format [Symbol] :plain, :markdown, :asciidoc, :auto
62
+ # (unknown symbols fall back to :auto)
63
+ # @return [Array<Suppression>] frozen
64
+ def scan(source, format: :auto)
65
+ return EMPTY if source.nil? || source.empty?
66
+
67
+ extractors = PROFILE.fetch(format.to_sym, PROFILE[:auto])
68
+ suppressions = []
69
+ open_blocks = []
70
+ last_line = 1
71
+
72
+ source.each_line.with_index(1) do |line, lineno|
73
+ last_line = lineno
74
+ directives_on(line, extractors).each do |action, words|
75
+ case action
76
+ when :disable_line
77
+ suppressions << Suppression.new(
78
+ kind: :line, directive_line: lineno,
79
+ start_line: lineno, end_line: lineno
80
+ )
81
+ when :disable_next_line
82
+ suppressions << Suppression.new(
83
+ kind: :next_line, directive_line: lineno,
84
+ start_line: lineno + 1, end_line: lineno + 1,
85
+ words: words
86
+ )
87
+ when :disable_file
88
+ open_blocks << lineno
89
+ when :enable_file
90
+ close_block(open_blocks.pop, lineno, suppressions)
91
+ end
92
+ end
93
+ end
94
+
95
+ open_blocks.each { |start| push_file_block(start, last_line, suppressions) }
96
+ suppressions.freeze
97
+ end
98
+
99
+ private
100
+
101
+ # All directives found on +line+, in order, as
102
+ # [action_symbol, word_list] pairs.
103
+ def directives_on(line, extractors)
104
+ directives = []
105
+ extractors.each do |extractor|
106
+ comment_bodies(line, extractor).each do |body|
107
+ match = body.match(DIRECTIVE_PATTERN)
108
+ next unless match
109
+
110
+ action = match[1].tr("-", "_").to_sym
111
+ words = action == :disable_next_line ? match[2].split : []
112
+ directives << [action, words]
113
+ end
114
+ end
115
+ directives
116
+ end
117
+
118
+ # Comment bodies on +line+ for one extractor kind.
119
+ def comment_bodies(line, extractor)
120
+ case extractor
121
+ when :bare
122
+ [line.strip]
123
+ when :html_comment
124
+ line.scan(HTML_COMMENT_BODIES).flatten
125
+ when :line_comment
126
+ match = line.match(ASCII_DOC_COMMENT_BODY)
127
+ match ? [match[:body]] : []
128
+ end
129
+ end
130
+
131
+ def close_block(start_line, enable_line, suppressions)
132
+ return unless start_line
133
+
134
+ # The enable-file line itself belongs to the block:
135
+ # directive text is never spellchecked.
136
+ push_file_block(start_line, [enable_line, start_line].max, suppressions)
137
+ end
138
+
139
+ def push_file_block(start_line, end_line, suppressions)
140
+ suppressions << Suppression.new(
141
+ kind: :file, directive_line: start_line,
142
+ start_line: start_line, end_line: end_line
143
+ )
144
+ end
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ module Suppressions
6
+ # One inline ignore directive, resolved to the 1-based source
7
+ # lines it covers. A plain value object — never serialized.
8
+ #
9
+ # Kinds:
10
+ # - +:line+ — `kotoshu:disable-line`, covers its own line
11
+ # - +:next_line+ — `kotoshu:disable-next-line [WORDS]`, covers
12
+ # the following line (only the listed words when a word list
13
+ # is present)
14
+ # - +:file+ — a `kotoshu:disable-file` / `kotoshu:enable-file`
15
+ # block, covers the whole span (to EOF when never re-enabled)
16
+ #
17
+ # The directive line itself is always suppressed for every
18
+ # word: the directive text is an instruction, not prose to
19
+ # check.
20
+ class Suppression
21
+ KINDS = %i[line next_line file].freeze
22
+
23
+ attr_reader :kind, :words, :directive_line, :start_line, :end_line
24
+
25
+ # @param kind [Symbol] one of KINDS
26
+ # @param directive_line [Integer] 1-based line the directive is on
27
+ # @param start_line [Integer] first covered line (inclusive)
28
+ # @param end_line [Integer] last covered line (inclusive)
29
+ # @param words [Array<String>, nil] word list (downcased);
30
+ # empty means "every word"
31
+ def initialize(kind:, directive_line:, start_line:, end_line:, words: [])
32
+ raise ArgumentError, "unknown kind: #{kind.inspect}" unless KINDS.include?(kind)
33
+ raise ArgumentError, "directive_line must be >= 1" if directive_line < 1
34
+ raise ArgumentError, "start_line must be >= 1" if start_line < 1
35
+ raise ArgumentError, "end_line must cover start_line" if end_line < start_line
36
+
37
+ @kind = kind
38
+ @directive_line = directive_line
39
+ @start_line = start_line
40
+ @end_line = end_line
41
+ @words = Array(words).map { |w| w.to_s.downcase }.reject(&:empty?).freeze
42
+ freeze
43
+ end
44
+
45
+ # A `kotoshu:disable-file` block.
46
+ def file_block?
47
+ kind == :file
48
+ end
49
+
50
+ # True when this suppression covers +line+ at all (word scope
51
+ # not considered).
52
+ def covers_line?(line)
53
+ line >= start_line && line <= end_line
54
+ end
55
+
56
+ # True when only specifically listed words are suppressed.
57
+ def word_scoped?
58
+ !words.empty?
59
+ end
60
+
61
+ # True when +word+ reported on 1-based +line+ is suppressed.
62
+ # Passing +word: nil+ asks "could any word on this line be
63
+ # suppressed?" — true when the line is the directive line or
64
+ # the suppression is not word-scoped.
65
+ def applies_to?(line, word: nil)
66
+ return true if line == directive_line
67
+ return false unless covers_line?(line)
68
+ return true if words.empty?
69
+
70
+ word.nil? || words.include?(word.to_s.downcase)
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ # Inline ignore directives (plan 82, Track A).
6
+ #
7
+ # A user can mark intentionally-"wrong" words (product names, code
8
+ # identifiers in prose, dialect) as ignored directly in the
9
+ # document, instead of polluting the personal dictionary:
10
+ #
11
+ # kotoshu:disable-line suppress everything on this line
12
+ # kotoshu:disable-next-line suppress the following line
13
+ # kotoshu:disable-next-line foo bar suppress only these words
14
+ # kotoshu:disable-file / enable-file block suppression
15
+ #
16
+ # Each document format recognizes the directive inside its own
17
+ # comment syntax: HTML comments in Markdown
18
+ # (`<!-- kotoshu:disable-next-line -->`), `//` line comments in
19
+ # AsciiDoc, and bare lines in plain text. Trailing directives are
20
+ # supported in Markdown and AsciiDoc.
21
+ #
22
+ # ONE shared scanner implements the semantics; only comment
23
+ # detection is per-format (see {Scanner}). Documents attach the
24
+ # resulting {Suppression} records during parse, and the
25
+ # Spellchecker facade filters suppressed words out of its results
26
+ # (they surface via `DocumentResult#suppressed_errors`).
27
+ module Suppressions
28
+ autoload :Suppression, "kotoshu/documents/suppressions/suppression"
29
+ autoload :Scanner, "kotoshu/documents/suppressions/scanner"
30
+
31
+ # Scan +source+ for ignore directives using the comment syntax
32
+ # of +format+ (:plain, :markdown, :asciidoc, or :auto for all
33
+ # three — the facade default, since it sees raw text without a
34
+ # parser).
35
+ #
36
+ # @param source [String, nil]
37
+ # @param format [Symbol]
38
+ # @return [Array<Suppression>] frozen, possibly empty
39
+ def self.scan(source, format: :auto)
40
+ Scanner.scan(source, format: format)
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ # A contiguous run of flattened text plus its source position.
6
+ #
7
+ # The checker scans the concatenation of every text node's +text+
8
+ # (the document's flattened form). Each node carries the {SourceRange}
9
+ # it occupies in the original markup-bearing source, plus its
10
+ # +flattened_offset+ (where the node starts in the concatenated
11
+ # flattened text) so the document can map errors back to source.
12
+ #
13
+ # +format+ and +metadata+ are escape hatches for plugins that want
14
+ # to attach structural context (e.g. `format: :heading,
15
+ # metadata: { level: 2 }` for an h2). Kotoshu's core checker
16
+ # ignores them; consumers (editors, reporters) can use them to
17
+ # surface context-aware UI.
18
+ TextNode = Struct.new(:text, :source_range, :flattened_offset, :format, :metadata,
19
+ keyword_init: true) do
20
+ def initialize(text:, source_range:, flattened_offset:, format: :plain, metadata: {})
21
+ raise TypeError, "source_range must be a SourceRange" unless source_range.is_a?(SourceRange)
22
+ raise ArgumentError, "flattened_offset must be >= 0" if flattened_offset.negative?
23
+
24
+ super
25
+ freeze
26
+ end
27
+
28
+ # Length of the flattened text in this node.
29
+ def length
30
+ text.length
31
+ end
32
+
33
+ # Flattened-text range covered by this node: [flattened_offset,
34
+ # flattened_offset + text.length).
35
+ def flattened_range
36
+ flattened_offset...(flattened_offset + text.length)
37
+ end
38
+
39
+ # True when +offset+ falls within this node's flattened range.
40
+ def contains_flattened?(offset)
41
+ flattened_range.include?(offset)
42
+ end
43
+
44
+ def to_s
45
+ "#{format}(#{text.inspect} @ #{source_range})"
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Document model for structure-aware spell and grammar checking.
5
+ #
6
+ # The Document abstraction pairs the *flattened* text a checker scans
7
+ # with the *source* positions of each text run, so that errors can be
8
+ # reported against the original markup-bearing source rather than the
9
+ # stripped text. The canonical example is an AsciiDoc or Markdown
10
+ # sentence like `I'm an **friend** of Tom`: the checker sees the
11
+ # flattened `I'm an friend of Tom` and flags "an friend", but the
12
+ # error report needs to point at the original `an **friend**` range
13
+ # so an editor can highlight what the user actually wrote.
14
+ #
15
+ # Kotoshu ships only the value-object layer and a trivial
16
+ # {PlainTextDocument}. Format-specific parsers (Markdown, AsciiDoc,
17
+ # reStructuredText, etc.) live in plugins — kotoshu never owns
18
+ # document parsing (see the `kotoshu-document-plugin-boundary` design
19
+ # memory). Plugins register a parser class via {Documents.register}
20
+ # and produce {Document} instances whose {TextNode}s carry proper
21
+ # {SourceRange}s.
22
+ module Documents
23
+ autoload :SourcePosition, "kotoshu/documents/source_position"
24
+ autoload :SourceRange, "kotoshu/documents/source_range"
25
+ autoload :TextNode, "kotoshu/documents/text_node"
26
+ autoload :Document, "kotoshu/documents/document"
27
+ autoload :PlainTextDocument, "kotoshu/documents/plain_text_document"
28
+ autoload :Suppressions, "kotoshu/documents/suppressions"
29
+
30
+ @parsers = {}
31
+ @discovered_formats = []
32
+ @discovered_plugins_loaded = false
33
+
34
+ class << self
35
+ # Register a document parser for a format symbol.
36
+ #
37
+ # The parser class must respond to:
38
+ #
39
+ # .from_string(text, language_code:) -> Documents::Document
40
+ #
41
+ # Optionally also:
42
+ #
43
+ # .from_file(path, language_code:) -> Documents::Document
44
+ #
45
+ # ==== Parser contract ====
46
+ #
47
+ # The returned {Document}'s +text_nodes+ must each carry a
48
+ # {Documents::SourceRange} that points at the original
49
+ # markup-bearing source for that node. This is the structure-aware
50
+ # contract — the analyzer reads the document's flattened text
51
+ # to run spelling/grammar checks, but every error is reported
52
+ # against the original source so an editor or plugin can
53
+ # highlight the user's actual markup, not the stripped text.
54
+ #
55
+ # Example: for the source
56
+ #
57
+ # "I'm an **friend** of Tom"
58
+ #
59
+ # the parser produces TextNodes whose SourceRanges cover the
60
+ # original range, not the flattened "I'm an friend of Tom".
61
+ # A grammar error "an friend" → "a friend" then carries a
62
+ # SourceRange pointing at "an **friend**" (the bold span plus the
63
+ # preceding "an ").
64
+ #
65
+ # ==== Format symbol ====
66
+ #
67
+ # +format+ is the canonical symbol callers pass to {.parse}. The
68
+ # gem ships +:plain+ (via {PlainTextDocument}). Plugins
69
+ # commonly add +:markdown+, +:asciidoc+, +:rst+, +:latex+, etc.
70
+ # Format symbols are global — two plugins registering the same
71
+ # symbol is a name clash; the last one wins.
72
+ #
73
+ # ==== Discovery ====
74
+ #
75
+ # Plugins ship a file under +kotoshu_plugin/document/*.rb+ in
76
+ # their gem. The first lookup of {.parser_for} / {.parse} walks
77
+ # +Gem.find_files+ for those files and requires each one. Each
78
+ # file's body is expected to call {.register} for the formats
79
+ # it provides. See {.discovered_plugin_files} and
80
+ # {.discovered_formats}.
81
+ #
82
+ # @param format [Symbol] e.g. :plain, :markdown, :asciidoc
83
+ # @param parser_class [Class] responds to .from_string
84
+ # @return [void]
85
+ def register(format, parser_class)
86
+ @parsers[format.to_sym] = parser_class
87
+ end
88
+
89
+ # Look up the registered parser for a format.
90
+ #
91
+ # @param format [Symbol]
92
+ # @return [Class, nil]
93
+ def parser_for(format)
94
+ ensure_plugins_discovered!
95
+ @parsers[format.to_sym]
96
+ end
97
+
98
+ # List every registered format symbol.
99
+ #
100
+ # @return [Array<Symbol>]
101
+ def registered_formats
102
+ ensure_plugins_discovered!
103
+ @parsers.keys
104
+ end
105
+
106
+ # Parse a source string with the registered parser for +format+,
107
+ # falling back to {PlainTextDocument} when no parser is
108
+ # registered. Never raises — the fallback is intentional so
109
+ # callers can pass arbitrary format hints without first checking
110
+ # the registry.
111
+ #
112
+ # @param source [String]
113
+ # @param format [Symbol]
114
+ # @param language_code [String, nil]
115
+ # @return [Document]
116
+ def parse(source, format:, language_code: nil)
117
+ ensure_plugins_discovered!
118
+ parser = parser_for(format) || PlainTextDocument
119
+ parser.from_string(source, language_code: language_code)
120
+ end
121
+
122
+ # List every plugin file found on the load path via
123
+ # +Gem.find_files("kotoshu_plugin/document/*.rb")+. Each such
124
+ # file, when required, is expected to call {register} for the
125
+ # formats it provides. The list is the raw file paths; not
126
+ # loaded into the registry yet.
127
+ #
128
+ # @return [Array<String>]
129
+ def discovered_plugin_files
130
+ Gem.find_files("kotoshu_plugin/document/*.rb").sort
131
+ end
132
+
133
+ # List every format symbol that came from an auto-discovered
134
+ # plugin (vs. one registered explicitly at runtime). Useful for
135
+ # diagnostics and "which plugin registered this format?" UX.
136
+ #
137
+ # @return [Array<Symbol>]
138
+ def discovered_formats
139
+ ensure_plugins_discovered!
140
+ @discovered_formats.dup.freeze
141
+ end
142
+
143
+ # Clear every registration. Test-only — production code should
144
+ # register once at load time. Also resets the discovery flag so
145
+ # the next lookup re-runs Gem.find_files.
146
+ #
147
+ # @return [void]
148
+ def reset!
149
+ @parsers.clear
150
+ @discovered_formats.clear
151
+ @discovered_plugins_loaded = false
152
+ end
153
+
154
+ private
155
+
156
+ # Walk Gem.find_files for plugin files at most once per
157
+ # process. Each file is +require+d; the file's body is expected
158
+ # to call {register}. Idempotent — re-entry is a no-op once the
159
+ # flag is set.
160
+ #
161
+ # The set of formats registered by each plugin file is captured
162
+ # by snapshotting the registry before and after the require.
163
+ def ensure_plugins_discovered!
164
+ return if @discovered_plugins_loaded
165
+
166
+ @discovered_plugins_loaded = true
167
+ before = @parsers.keys
168
+ discovered_plugin_files.each do |path|
169
+ require path
170
+ rescue StandardError, LoadError => e
171
+ warn "kotoshu: failed to load document plugin #{path}: #{e.message}"
172
+ end
173
+ after = @parsers.keys
174
+ @discovered_formats.concat(after - before)
175
+ end
176
+ end
177
+ end
178
+ end