kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,236 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'zip'
4
+
5
+ module Kotoshu
6
+ module Dictionary
7
+ # Spylls-style facade over Hunspell-format .aff/.dic dictionary
8
+ # files. Distinct from {Kotoshu::Dictionary::Hunspell} (which is a
9
+ # backend subclass of {Dictionary::Base}) and from
10
+ # {Kotoshu::Spellchecker} (the high-level facade): this class is the
11
+ # low-level entry point for callers that want direct access to the
12
+ # underlying Lookuper / Suggester without the surrounding
13
+ # configuration / personal-dictionary / suggestion-generator
14
+ # machinery.
15
+ #
16
+ # Was previously declared as `class Kotoshu::Dictionary`, which
17
+ # collided with the {Dictionary} module that holds the backend
18
+ # namespace — Ruby raised TypeError on autoload. Renamed to
19
+ # {Dictionary::Unified} so it loads cleanly under the existing
20
+ # `autoload :Unified, "kotoshu/dictionary/unified"` entry.
21
+ #
22
+ # @example Loading from files
23
+ # dictionary = Dictionary::Unified.from_files('/path/to/dictionary/en_US')
24
+ # dictionary.lookup('spells') # => true
25
+ #
26
+ # @example Loading from zip archive
27
+ # dictionary = Dictionary::Unified.from_zip('/path/to/dictionary/en_US.odt')
28
+ #
29
+ # @example Loading from system
30
+ # dictionary = Dictionary::Unified.from_system('en_US')
31
+ #
32
+ # @example Getting suggestions
33
+ # dictionary.suggest('spylls') # => ["spells", "spills", ...]
34
+ #
35
+ # @example Accessing algorithms for experimentation
36
+ # dictionary.lookuper.good_forms('building') do |form|
37
+ # puts form
38
+ # end
39
+ class Unified
40
+ # System paths to search for Hunspell dictionaries
41
+ PATHES = [
42
+ '/usr/share/hunspell',
43
+ '/usr/share/myspell',
44
+ '/usr/share/myspell/dicts',
45
+ '/Library/Spelling',
46
+ '/opt/openoffice.org/basis3.0/share/dict/ooo',
47
+ '/usr/lib/openoffice.org/basis3.0/share/dict/ooo',
48
+ '/opt/openoffice.org2.4/share/dict/ooo',
49
+ '/usr/lib/openoffice.org2.4/share/dict/ooo',
50
+ '/opt/openoffice.org2.3/share/dict/ooo',
51
+ '/usr/lib/openoffice.org2.3/share/dict/ooo',
52
+ '/opt/openoffice.org2.2/share/dict/ooo',
53
+ '/usr/lib/openoffice.org2.2/share/dict/ooo',
54
+ '/opt/openoffice.org2.1/share/dict/ooo',
55
+ '/usr/lib/openoffice.org2.1/share/dict/ooo',
56
+ '/opt/openoffice.org2.0/share/dict/ooo',
57
+ '/usr/lib/openoffice.org2.0/share/dict/ooo'
58
+ ].freeze
59
+
60
+ # Distributed dictionaries for testing
61
+ DISTRIBUTED = {
62
+ 'en_US' => 'en',
63
+ 'ru' => 'ru',
64
+ 'sv_SE' => 'sv'
65
+ }.freeze
66
+
67
+ # @return [Hash] Aff data structure
68
+ attr_reader :aff
69
+
70
+ # @return [Array<Readers::Word>] Dic data structure
71
+ attr_reader :dic_words
72
+
73
+ # @return [Algorithms::Lookup::Lookuper] Lookuper instance for experimentation
74
+ attr_reader :lookuper
75
+
76
+ # @return [Algorithms::Suggest::Suggester] Suggester instance for experimentation
77
+ attr_reader :suggester
78
+
79
+ # Create a Dictionary from aff and dic data.
80
+ #
81
+ # @param aff [Hash] Aff data structure
82
+ # @param dic_words [Array<Readers::Word>] Dictionary word entries
83
+ def initialize(aff, dic_words)
84
+ @aff = aff
85
+ @dic_words = dic_words
86
+
87
+ # Build the Lookuper via LookupBuilder — it owns the aff →
88
+ # structure transformation (casing, suffixes/prefixes indexes,
89
+ # REP patterns, etc.). The Suggester needs that processed aff
90
+ # structure too, so we read it back off the Lookuper rather
91
+ # than passing the raw AffReader hash.
92
+ @lookuper = Readers::LookupBuilder.from_data(aff, dic_words).build
93
+ @suggester = Algorithms::Suggest::Suggester.new(
94
+ @lookuper.aff, @lookuper.dic, @lookuper
95
+ )
96
+ end
97
+
98
+ # Load dictionary from file path.
99
+ #
100
+ # The path should be the base name without extension, e.g., 'en_US'
101
+ # for files 'en_US.aff' and 'en_US.dic'.
102
+ #
103
+ # @param path [String] Base path to dictionary files (without extension)
104
+ # @return [Dictionary] The loaded dictionary
105
+ #
106
+ # @example
107
+ # Dictionary.from_files('en_US')
108
+ def self.from_files(path)
109
+ # Check if it's a distributed dictionary
110
+ if DISTRIBUTED.key?(path) && !File.exist?("#{path}.aff")
111
+ distributed_path = File.join(File.dirname(__FILE__), '../../data', DISTRIBUTED[path], path)
112
+ if File.exist?("#{distributed_path}.aff")
113
+ path = distributed_path
114
+ end
115
+ end
116
+
117
+ aff_path = "#{path}.aff"
118
+ dic_path = "#{path}.dic"
119
+
120
+ raise ArgumentError, "Dictionary file not found: #{aff_path}" unless File.exist?(aff_path)
121
+ raise ArgumentError, "Dictionary file not found: #{dic_path}" unless File.exist?(dic_path)
122
+
123
+ # Read aff file
124
+ aff_reader = Readers::AffReader.new(aff_path)
125
+ aff_data = aff_reader.read
126
+
127
+ # Read dic file
128
+ dic_reader = Readers::DicReader.new(dic_path,
129
+ flag_format: aff_data['FLAG'] || 'short',
130
+ flag_synonyms: aff_data['AF'] || {})
131
+ dic_words = dic_reader.read
132
+
133
+ new(aff_data, dic_words)
134
+ end
135
+
136
+ # Load dictionary from zip archive.
137
+ #
138
+ # Supports OpenOffice/LibreOffice dictionary extensions (.odt, .oxt)
139
+ # and Firefox/Thunderbird dictionary extensions (.xpi).
140
+ #
141
+ # @param zip_path [String] Path to zip archive
142
+ # @return [Dictionary] The loaded dictionary
143
+ #
144
+ # @example
145
+ # Dictionary.from_zip('en_US.odt')
146
+ def self.from_zip(zip_path)
147
+ Zip::File.open(zip_path) do |zipfile|
148
+ # Find .aff and .dic files
149
+ aff_entry = nil
150
+ dic_entry = nil
151
+
152
+ zipfile.each do |entry|
153
+ if entry.name.end_with?('.aff')
154
+ raise ArgumentError, "Multiple .aff files found in zip" if aff_entry
155
+
156
+ aff_entry = entry
157
+ elsif entry.name.end_with?('.dic')
158
+ raise ArgumentError, "Multiple .dic files found in zip" if dic_entry
159
+
160
+ dic_entry = entry
161
+ end
162
+ end
163
+
164
+ raise ArgumentError, "No .aff file found in zip" unless aff_entry
165
+ raise ArgumentError, "No .dic file found in zip" unless dic_entry
166
+
167
+ # Read aff file
168
+ aff_reader = Readers::ZipReader.new(zipfile, aff_entry.name)
169
+ aff_reader.to_a
170
+ # Parse the raw data into proper aff structure
171
+ Readers::AffReader.new(zip_path) # Temporary for context
172
+ aff_data = Readers::AffReader.new(aff_entry.name).read
173
+
174
+ # Read dic file
175
+ dic_reader = Readers::DicReader.new(dic_entry.name,
176
+ flag_format: aff_data['FLAG'] || 'short',
177
+ flag_synonyms: aff_data['AF'] || {})
178
+ dic_words = dic_reader.read
179
+
180
+ new(aff_data, dic_words)
181
+ end
182
+ end
183
+
184
+ # Load dictionary from system paths.
185
+ #
186
+ # Searches standard system locations for Hunspell dictionaries.
187
+ #
188
+ # @param name [String] Dictionary name (e.g., 'en_US', 'ru_RU')
189
+ # @return [Dictionary] The loaded dictionary
190
+ # @raise [ArgumentError] If dictionary not found in system paths
191
+ #
192
+ # @example
193
+ # Dictionary.from_system('en_US')
194
+ def self.from_system(name)
195
+ PATHES.each do |folder|
196
+ aff_path = File.join(folder, "#{name}.aff")
197
+ if File.exist?(aff_path)
198
+ base_path = aff_path.sub(/\.aff$/, '')
199
+ return from_files(base_path)
200
+ end
201
+ end
202
+
203
+ raise ArgumentError, "#{name}.aff not found in system paths: #{PATHES.inspect}"
204
+ end
205
+
206
+ # Check if a word is correct.
207
+ #
208
+ # @param word [String] Word to check
209
+ # @return [Boolean] True if the word exists in the dictionary
210
+ #
211
+ # @example
212
+ # dictionary.lookup('spells') # => true
213
+ # dictionary.lookup('spylls') # => false
214
+ def lookup(word)
215
+ @lookuper.call(word)
216
+ end
217
+
218
+ # Generate suggestions for a misspelled word.
219
+ #
220
+ # Returns suggestions in order of probability/similarity,
221
+ # with best suggestions first.
222
+ #
223
+ # @param word [String] The misspelled word
224
+ # @yield [String] Each suggestion
225
+ # @return [Enumerator] If no block given
226
+ #
227
+ # @example
228
+ # dictionary.suggest('spylls') # => ["spells", "spills", ...]
229
+ def suggest(word, &)
230
+ return enum_for(:suggest, word) unless block_given?
231
+
232
+ @suggester.suggestions(word, &)
233
+ end
234
+ end
235
+ end
236
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Dictionary
5
+ # Unix system dictionary backend.
6
+ #
7
+ # This dictionary reads from Unix-style system dictionary files,
8
+ # typically located at `/usr/share/dict/words` or symlinks to
9
+ # dictionaries like `web2` (Webster's Second International).
10
+ #
11
+ # @example Using system dictionary
12
+ # dict = UnixWords.new("/usr/share/dict/words", language_code: "en-US")
13
+ # dict.lookup?("hello") # => true
14
+ # dict.suggest("helo") # => ["hello", "help", "held", ...]
15
+ #
16
+ # @example Auto-detecting system dictionary
17
+ # dict = UnixWords.detect(language_code: "en-US")
18
+ class UnixWords < Base
19
+ # Standard system paths to check for dictionaries.
20
+ SYSTEM_PATHS = [
21
+ "/usr/share/dict/words",
22
+ "/usr/share/dict/web2",
23
+ "/usr/share/dict/american-english",
24
+ "/usr/share/dict/british-english",
25
+ "/usr/dict/words",
26
+ "/System/Library/Assets/com_apple_MobileAsset_DictionaryServices_dictionaryOS/Dictionary/words" # macOS
27
+ ].freeze
28
+
29
+ # @return [String] The path to the dictionary file
30
+ attr_reader :path
31
+
32
+ # @return [Boolean] Whether lookups are case-sensitive
33
+ attr_reader :case_sensitive
34
+
35
+ # Create a new UnixWords dictionary.
36
+ #
37
+ # @param path [String] Path to the dictionary file
38
+ # @param language_code [String] The language code
39
+ # @param locale [String, nil] The locale (optional)
40
+ # @param case_sensitive [Boolean] Whether lookups are case-sensitive
41
+ # @param metadata [Hash] Additional metadata (optional)
42
+ def initialize(path, language_code:, locale: nil, case_sensitive: false, metadata: {})
43
+ super(language_code, locale: locale, metadata: metadata)
44
+
45
+ @path = File.expand_path(path)
46
+ @case_sensitive = case_sensitive
47
+ @words = load_words(@path)
48
+ @word_set = build_word_set
49
+
50
+ # Register this dictionary type
51
+ self.class.register_type(:unix_words) unless Dictionary.registry.key?(:unix_words)
52
+ end
53
+
54
+ # Check if a word exists in the dictionary.
55
+ #
56
+ # @param word [String] The word to look up
57
+ # @return [Boolean] True if the word exists
58
+ def lookup(word)
59
+ return false if word.nil? || word.empty?
60
+
61
+ lookup_word = @case_sensitive ? word : word.downcase
62
+ @word_set.key?(lookup_word)
63
+ end
64
+
65
+ # Generate spelling suggestions.
66
+ #
67
+ # Uses edit distance to find similar words in the dictionary.
68
+ #
69
+ # @param word [String] The misspelled word
70
+ # @param max_suggestions [Integer] Maximum suggestions
71
+ # @return [Array<String>] List of suggested words
72
+ def suggest(word, max_suggestions: 10)
73
+ return [] if word.nil? || word.empty?
74
+
75
+ # For now, use simple prefix matching and edit distance
76
+ # This will be improved with the suggestion algorithms
77
+ lookup_word = @case_sensitive ? word : word.downcase
78
+
79
+ # Find words with same prefix
80
+ prefix_len = [lookup_word.length - 1, 3].max
81
+ prefix = lookup_word[0...prefix_len]
82
+ candidates = @words.select { |w| w.start_with?(prefix) }
83
+
84
+ # Calculate edit distances
85
+ candidates.map do |dict_word|
86
+ dist = edit_distance(lookup_word, dict_word)
87
+ [dict_word, dist]
88
+ end.select { |_, dist| dist.positive? && dist <= 2 } # Only close matches
89
+ .sort_by { |_, dist| dist }
90
+ .first(max_suggestions)
91
+ .map(&:first)
92
+ end
93
+
94
+ # Add a word to the dictionary.
95
+ #
96
+ # @param word [String] The word to add
97
+ # @param flags [Array<String>] Flags (ignored for UnixWords)
98
+ # @return [Boolean] True if added
99
+ def add_word(word, flags: [])
100
+ return false if word.nil? || word.empty?
101
+
102
+ lookup_word = @case_sensitive ? word : word.downcase
103
+ return false if @word_set.key?(lookup_word)
104
+
105
+ @words << lookup_word
106
+ @word_set[lookup_word] = true
107
+ reset_sweep_index
108
+
109
+ true
110
+ end
111
+
112
+ # Remove a word from the dictionary.
113
+ #
114
+ # @param word [String] The word to remove
115
+ # @return [Boolean] True if removed
116
+ def remove_word(word)
117
+ return false if word.nil? || word.empty?
118
+
119
+ lookup_word = @case_sensitive ? word : word.downcase
120
+ return false unless @word_set.key?(lookup_word)
121
+
122
+ @word_set.delete(lookup_word)
123
+ @words.delete(lookup_word)
124
+ reset_sweep_index
125
+
126
+ true
127
+ end
128
+
129
+ # Get all words in the dictionary.
130
+ #
131
+ # @return [Array<String>] All words
132
+ def words
133
+ @words.dup
134
+ end
135
+
136
+ # Detect system dictionary path.
137
+ #
138
+ # Checks standard system paths for an existing dictionary file.
139
+ #
140
+ # @return [String, nil] The detected path or nil
141
+ #
142
+ # @example
143
+ # UnixWords.detect_system_dictionary # => "/usr/share/dict/words"
144
+ def self.detect_system_dictionary
145
+ SYSTEM_PATHS.find { |p| File.exist?(p) }
146
+ end
147
+
148
+ # Create a dictionary by auto-detecting system dictionary.
149
+ #
150
+ # @param language_code [String] The language code
151
+ # @param locale [String, nil] The locale (optional)
152
+ # @param case_sensitive [Boolean] Whether lookups are case-sensitive
153
+ # @return [UnixWords, nil] The dictionary or nil if not found
154
+ #
155
+ # @example
156
+ # dict = UnixWords.detect(language_code: "en-US")
157
+ def self.detect(language_code:, locale: nil, case_sensitive: false)
158
+ path = detect_system_dictionary
159
+ return nil unless path
160
+
161
+ new(path, language_code: language_code, locale: locale,
162
+ case_sensitive: case_sensitive)
163
+ end
164
+
165
+ private
166
+
167
+ # Load words from dictionary file.
168
+ #
169
+ # @param path [String] The file path
170
+ # @return [Array<String>] List of words
171
+ def load_words(path)
172
+ raise DictionaryNotFoundError, path unless File.exist?(path)
173
+
174
+ File.foreach(path, chomp: true)
175
+ .reject { |line| line.empty? || line.start_with?("#") }
176
+ .map { |word| @case_sensitive ? word : word.downcase }
177
+ end
178
+
179
+ # Build a hash set for O(1) lookups.
180
+ #
181
+ # @return [Hash] Word membership hash (word => true)
182
+ def build_word_set
183
+ @words.to_h { |word| [word, true] }
184
+ end
185
+
186
+ # Calculate Levenshtein edit distance.
187
+ #
188
+ # @param str1 [String] First string
189
+ # @param str2 [String] Second string
190
+ # @return [Integer] Edit distance
191
+ def edit_distance(str1, str2)
192
+ return str2.length if str1.empty?
193
+ return str1.length if str2.empty?
194
+
195
+ # Use smaller string for inner loop
196
+ str1, str2 = str2, str1 if str1.length > str2.length
197
+
198
+ previous = (0..str1.length).to_a
199
+
200
+ str2.each_char.with_index do |char2, j|
201
+ current = [j + 1]
202
+
203
+ str1.each_char.with_index do |char1, i|
204
+ insert_cost = current[i] + 1
205
+ delete_cost = previous[i + 1] + 1
206
+ substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
207
+
208
+ current << [insert_cost, delete_cost, substitute_cost].min
209
+ end
210
+
211
+ previous = current
212
+ end
213
+
214
+ previous.last
215
+ end
216
+ end
217
+ end
218
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Dictionary backends for different formats (Hunspell, CSpell, plain text, etc.).
5
+ module Dictionary
6
+ autoload :Base, "kotoshu/dictionary/base"
7
+ autoload :CSpell, "kotoshu/dictionary/cspell"
8
+ autoload :Custom, "kotoshu/dictionary/custom"
9
+ autoload :Hunspell, "kotoshu/dictionary/hunspell"
10
+ autoload :PlainText, "kotoshu/dictionary/plain_text"
11
+ autoload :Repository, "kotoshu/dictionary/repository"
12
+ autoload :Unified, "kotoshu/dictionary/unified"
13
+ autoload :UnixWords, "kotoshu/dictionary/unix_words"
14
+ end
15
+ end
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ # Document abstraction for structure-aware spell and grammar
6
+ # checking. Pairs the flattened text a checker scans with the
7
+ # source positions of each {TextNode}, so errors can be reported
8
+ # against the original markup-bearing source.
9
+ #
10
+ # Subclasses (or plugins) populate +text_nodes+ with one entry per
11
+ # contiguous text run, each carrying the {SourceRange} it occupies
12
+ # in the source. The base class provides the bidirectional
13
+ # offset-mapping logic — it walks the nodes to recover a source
14
+ # range from a flattened offset.
15
+ #
16
+ # Kotoshu ships only {PlainTextDocument}; format-specific parsers
17
+ # (Markdown, AsciiDoc, etc.) live in plugins.
18
+ class Document
19
+ attr_reader :text_nodes, :source, :format, :language_code, :suppressions
20
+
21
+ # @param text_nodes [Array<TextNode>] one entry per contiguous
22
+ # text run, in source order
23
+ # @param source [String, nil] the original markup-bearing source
24
+ # (kept around so consumers can slice it for context display)
25
+ # @param format [Symbol] e.g. :plain, :markdown, :asciidoc
26
+ # @param language_code [String, nil] ISO 639-1 code when known
27
+ # @param suppressions [Array<Suppressions::Suppression>, nil] inline
28
+ # ignore directives; nil means scan +source+ with +format+ comment
29
+ # syntax (plan 82)
30
+ def initialize(text_nodes:, source: nil, format: :plain, language_code: nil, suppressions: nil)
31
+ raise ArgumentError, "text_nodes cannot be empty" if text_nodes.nil? || text_nodes.empty?
32
+
33
+ @text_nodes = text_nodes.freeze
34
+ @source = source
35
+ @format = format
36
+ @language_code = language_code
37
+ @suppressions = (suppressions || Suppressions.scan(source, format: format)).freeze
38
+ freeze
39
+ end
40
+
41
+ # The flattened text — concatenation of every node's +text+.
42
+ # This is what the checker actually scans.
43
+ #
44
+ # @return [String]
45
+ def flattened_text
46
+ @text_nodes.map(&:text).join
47
+ end
48
+
49
+ # Total character length of the flattened text.
50
+ def flattened_length
51
+ @text_nodes.sum(&:length)
52
+ end
53
+
54
+ # True when +offset+ falls within the flattened text.
55
+ def flattened_offset?(offset)
56
+ offset >= 0 && offset < flattened_length
57
+ end
58
+
59
+ # Recover the {SourceRange} of the {TextNode} that contains the
60
+ # given flattened offset. Returns nil for out-of-range offsets.
61
+ #
62
+ # Note: this returns the *node's* source range, not a one-char
63
+ # slice. For a tighter range, use {#source_range_for} with a
64
+ # flattened sub-range.
65
+ #
66
+ # @param flattened_offset [Integer] 0-based offset in flattened text
67
+ # @return [SourceRange, nil]
68
+ def source_range_at(flattened_offset)
69
+ return nil unless flattened_offset?(flattened_offset)
70
+
71
+ node = node_at(flattened_offset)
72
+ node&.source_range
73
+ end
74
+
75
+ # Recover the {SourceRange} spanning a flattened sub-range.
76
+ # The start of the result is the start of the node containing
77
+ # +flattened_start+; the end is the end of the node containing
78
+ # +flattened_end - 1+. Useful for errors that span multiple
79
+ # text nodes (e.g. a grammar error like "an **friend**" that
80
+ # crosses a markup boundary).
81
+ #
82
+ # @param flattened_start [Integer] inclusive
83
+ # @param flattened_end [Integer] exclusive; pass nil for "to end"
84
+ # @return [SourceRange, nil]
85
+ def source_range_for(flattened_start, flattened_end = nil)
86
+ return nil unless flattened_offset?(flattened_start)
87
+
88
+ end_offset = flattened_end ? flattened_end - 1 : flattened_length - 1
89
+ end_offset = [end_offset, flattened_length - 1].min
90
+ return nil unless flattened_offset?(end_offset)
91
+
92
+ start_node = node_at(flattened_start)
93
+ end_node = node_at(end_offset)
94
+ return nil unless start_node && end_node
95
+
96
+ start_node.source_range.union(end_node.source_range)
97
+ end
98
+
99
+ # Iterate text nodes. Returns an Enumerator when no block given.
100
+ def each_node(&block)
101
+ return enum_for(:each_node) unless block
102
+
103
+ @text_nodes.each(&block)
104
+ end
105
+
106
+ # True when +word+ (any word when nil) reported on 1-based +line+
107
+ # is suppressed by an inline ignore directive (plan 82).
108
+ def suppressed?(line, word: nil)
109
+ @suppressions.any? { |suppression| suppression.applies_to?(line, word: word) }
110
+ end
111
+
112
+ private
113
+
114
+ # Find the TextNode whose flattened range contains +offset+.
115
+ def node_at(offset)
116
+ @text_nodes.find { |node| node.contains_flattened?(offset) }
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Documents
5
+ # Built-in {Document} parser for plain text (no markup).
6
+ #
7
+ # The only parser kotoshu ships. Format-specific parsers (Markdown,
8
+ # AsciiDoc, etc.) live in plugins — see {Documents.register}.
9
+ #
10
+ # PlainTextDocument is intentionally simple: one {TextNode}
11
+ # covering the entire source. The flattened text equals the source,
12
+ # so source-to-flattened mapping is the identity.
13
+ class PlainTextDocument < Document
14
+ # Build a document from a string.
15
+ #
16
+ # @param text [String]
17
+ # @param language_code [String, nil]
18
+ # @return [PlainTextDocument]
19
+ # +suppressions+ defaults to scanning the text for bare
20
+ # directives (plan 82); pass [] to attach none.
21
+ def self.from_string(text, language_code: nil, suppressions: nil)
22
+ text ||= ""
23
+ range = source_range_for_string(text, 0)
24
+ new(
25
+ text_nodes: [TextNode.new(text: text, source_range: range,
26
+ flattened_offset: 0, format: :plain, metadata: {})],
27
+ source: text,
28
+ format: :plain,
29
+ language_code: language_code,
30
+ suppressions: suppressions
31
+ )
32
+ end
33
+
34
+ # Build a document from a file.
35
+ #
36
+ # @param path [String]
37
+ # @param language_code [String, nil]
38
+ # @return [PlainTextDocument]
39
+ def self.from_file(path, language_code: nil)
40
+ from_string(File.read(path), language_code: language_code)
41
+ end
42
+
43
+ class << self
44
+ private
45
+
46
+ # Compute the SourceRange for +text+ starting at +base_offset+
47
+ # in the source. Walks the string once to find the final line
48
+ # and column.
49
+ def source_range_for_string(text, base_offset)
50
+ start_pos = SourcePosition.new(offset: base_offset, line: 1, column: 1)
51
+
52
+ line = 1
53
+ column = 1
54
+ text.each_char do |char|
55
+ if char == "\n"
56
+ line += 1
57
+ column = 1
58
+ else
59
+ column += 1
60
+ end
61
+ end
62
+
63
+ end_pos = SourcePosition.new(
64
+ offset: base_offset + text.length,
65
+ line: line,
66
+ column: column
67
+ )
68
+ SourceRange.new(start_pos: start_pos, end_pos: end_pos)
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end