kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,293 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Core
5
+ # Indexed dictionary for efficient word lookup with multiple indexes.
6
+ # This is MORE model-driven than Spylls which uses simple hash indices.
7
+ #
8
+ # This is a proper domain model with rich behavior including:
9
+ # - Multiple indexes (case-sensitive, case-insensitive, prefix, suffix)
10
+ # - Rich query methods
11
+ # - Index management
12
+ # - Domain-specific behavior
13
+ class IndexedDictionary
14
+ attr_reader :words, :size
15
+
16
+ # @param words [Array<String>] Initial words to add
17
+ def initialize(words = [])
18
+ @words = []
19
+ @indexes = {
20
+ exact: {}, # case_sensitive: word => [positions]
21
+ lowercase: {}, # case_insensitive: word.downcase => [positions]
22
+ prefix: {}, # prefix => [words]
23
+ suffix: {}, # suffix => [words]
24
+ length: {}, # length => [words]
25
+ flag: {} # flag => [words] (future: for Hunspell)
26
+ }
27
+ @size = 0
28
+
29
+ words.each { |word| add_word(word) }
30
+ end
31
+
32
+ # Add a word to the dictionary with optional metadata.
33
+ #
34
+ # @param word [String] The word to add
35
+ # @param metadata [Hash] Optional metadata associated with the word
36
+ # @return [IndexedDictionary] Self for chaining
37
+ def add_word(word, metadata = {})
38
+ # Store the word with its index and metadata
39
+ entry = { word: word, index: @size, metadata: metadata }
40
+ @words << entry
41
+ @size += 1
42
+
43
+ # Update exact index (case-sensitive)
44
+ @indexes[:exact][word] ||= []
45
+ @indexes[:exact][word] << (@size - 1)
46
+
47
+ # Update lowercase index (case-insensitive)
48
+ lower = word.downcase
49
+ @indexes[:lowercase][lower] ||= []
50
+ @indexes[:lowercase][lower] << (@size - 1)
51
+
52
+ # Update prefix indexes (for prefix searching)
53
+ (1...word.length).each do |i|
54
+ prefix = word[0...i]
55
+ @indexes[:prefix][prefix] ||= []
56
+ @indexes[:prefix][prefix] << word
57
+
58
+ # Update suffix indexes (for suffix searching)
59
+ suffix = word[i..]
60
+ @indexes[:suffix][suffix] ||= []
61
+ @indexes[:suffix][suffix] << word
62
+ end
63
+
64
+ # Update length index (for length-range queries — used by
65
+ # edit-distance strategies to prune the dictionary by length).
66
+ (@indexes[:length][word.length] ||= []) << word
67
+
68
+ self
69
+ end
70
+ alias << add_word
71
+
72
+ # Add multiple words.
73
+ #
74
+ # @param new_words [Array<String>] Words to add
75
+ # @return [IndexedDictionary] Self for chaining
76
+ def add_words(new_words)
77
+ new_words.each { |word| add_word(word) }
78
+ self
79
+ end
80
+
81
+ # Check if a word exists (case-sensitive).
82
+ #
83
+ # @param word [String] The word to check
84
+ # @return [Boolean] True if word exists
85
+ def has_word?(word)
86
+ @indexes[:exact].key?(word)
87
+ end
88
+ alias include? has_word?
89
+ alias contains? has_word?
90
+
91
+ # Check if a word exists (case-insensitive).
92
+ #
93
+ # @param word [String] The word to check
94
+ # @return [Boolean] True if word exists (any case)
95
+ def has_word_ignorecase?(word)
96
+ @indexes[:lowercase].key?(word.downcase)
97
+ end
98
+
99
+ # Look up a word (case-sensitive).
100
+ #
101
+ # @param word [String] The word to look up
102
+ # @return [Hash, nil] Word entry or nil
103
+ def lookup(word)
104
+ indices = @indexes[:exact][word]
105
+ return nil if indices.nil? || indices.empty?
106
+
107
+ @words[indices.first]
108
+ end
109
+
110
+ # Look up a word (case-insensitive).
111
+ #
112
+ # @param word [String] The word to look up
113
+ # @return [Hash, nil] Word entry or nil
114
+ def lookup_ignorecase(word)
115
+ indices = @indexes[:lowercase][word.downcase]
116
+ return nil if indices.nil? || indices.empty?
117
+
118
+ @words[indices.first]
119
+ end
120
+
121
+ # Find all words with a given prefix.
122
+ #
123
+ # @param prefix [String] The prefix to match
124
+ # @param ignore_case [Boolean] Whether to ignore case
125
+ # @return [Array<String>] Words with the prefix
126
+ def find_by_prefix(prefix, ignore_case: false)
127
+ if ignore_case
128
+ prefix_lower = prefix.downcase
129
+ all_words.select { |w| w.downcase.start_with?(prefix_lower) }
130
+ else
131
+ @indexes[:prefix].fetch(prefix, []).dup
132
+ end
133
+ end
134
+
135
+ # Find all words with a given suffix.
136
+ #
137
+ # @param suffix [String] The suffix to match
138
+ # @param ignore_case [Boolean] Whether to ignore case
139
+ # @return [Array<String>] Words with the suffix
140
+ def find_by_suffix(suffix, ignore_case: false)
141
+ if ignore_case
142
+ suffix_lower = suffix.downcase
143
+ all_words.select { |w| w.downcase.end_with?(suffix_lower) }
144
+ else
145
+ @indexes[:suffix].fetch(suffix, []).dup
146
+ end
147
+ end
148
+
149
+ # Find words matching a pattern.
150
+ #
151
+ # @param pattern [Regexp] The pattern to match
152
+ # @return [Array<String>] Matching words
153
+ def find_by_pattern(pattern)
154
+ all_words.select { |w| w.match?(pattern) }
155
+ end
156
+
157
+ # Find words of a specific length.
158
+ #
159
+ # @param length [Integer] The exact length
160
+ # @return [Array<String>] Words of the given length
161
+ def find_by_length(length)
162
+ (@indexes[:length][length] || []).dup
163
+ end
164
+
165
+ # Find words within a length range.
166
+ #
167
+ # @param min_length [Integer] Minimum length
168
+ # @param max_length [Integer] Maximum length
169
+ # @return [Array<String>] Words within the length range
170
+ def find_by_length_range(min_length:, max_length:)
171
+ (min_length..max_length).flat_map { |len| @indexes[:length][len] || [] }
172
+ end
173
+
174
+ # Get all words in the dictionary.
175
+ #
176
+ # @return [Array<String>] All words
177
+ def all_words
178
+ @words.map { |entry| entry[:word] }
179
+ end
180
+
181
+ # Get random words from the dictionary.
182
+ #
183
+ # @param count [Integer] Number of random words
184
+ # @return [Array<String>] Random words
185
+ def random_words(count: 1)
186
+ return [] if @words.empty?
187
+
188
+ indices = (0...@size).to_a.sample(count)
189
+ indices.map { |i| @words[i][:word] }
190
+ end
191
+
192
+ # Get words starting with each letter (A-Z).
193
+ #
194
+ # @return [Hash] Hash of letter => word count
195
+ def count_by_first_letter
196
+ result = Hash.new(0)
197
+ all_words.each do |word|
198
+ next if word.empty?
199
+
200
+ letter = word[0].upcase
201
+ result[letter] += 1
202
+ end
203
+ result
204
+ end
205
+
206
+ # Get word length distribution.
207
+ #
208
+ # @return [Hash] Hash of length => count
209
+ def count_by_length
210
+ result = Hash.new(0)
211
+ all_words.each { |word| result[word.length] += 1 }
212
+ result
213
+ end
214
+
215
+ # Check if the dictionary is empty.
216
+ #
217
+ # @return [Boolean] True if empty
218
+ def empty?
219
+ @size.zero?
220
+ end
221
+
222
+ # Iterate over all words.
223
+ #
224
+ # @yield [word] Each word
225
+ # @return [Enumerator] Enumerator if no block given
226
+ def each_word
227
+ return enum_for(:each_word) unless block_given?
228
+
229
+ @words.each { |entry| yield entry[:word] }
230
+ end
231
+
232
+ # Iterate over all words with indices.
233
+ #
234
+ # @yield [word, index] Each word and its index
235
+ # @return [Enumerator] Enumerator if no block given
236
+ def each_with_index
237
+ return enum_for(:each_with_index) unless block_given?
238
+
239
+ @words.each { |entry| yield entry[:word], entry[:index] }
240
+ end
241
+
242
+ # Build a Trie from the dictionary words.
243
+ #
244
+ # @return [Trie] New trie containing all words
245
+ def to_trie
246
+ Trie::Builder.from_array(all_words)
247
+ end
248
+
249
+ # Get statistics about the dictionary.
250
+ #
251
+ # @return [Hash] Statistics
252
+ def statistics
253
+ lengths = all_words.map(&:length)
254
+
255
+ {
256
+ total_words: @size,
257
+ unique_words: all_words.uniq.size,
258
+ min_length: lengths.min || 0,
259
+ max_length: lengths.max || 0,
260
+ avg_length: lengths.empty? ? 0 : (lengths.sum.to_f / lengths.size).round(2),
261
+ count_by_first_letter: count_by_first_letter,
262
+ count_by_length: count_by_length
263
+ }
264
+ end
265
+
266
+ # Convert to string.
267
+ #
268
+ # @return [String] String representation
269
+ def to_s
270
+ "IndexedDictionary(size: #{@size})"
271
+ end
272
+ alias inspect to_s
273
+
274
+ # Create indexed dictionary from a file.
275
+ #
276
+ # @param path [String] Path to word list file
277
+ # @return [IndexedDictionary] New dictionary
278
+ def self.from_file(path)
279
+ words = File.foreach(path, chomp: true).reject { |l| l.empty? || l.start_with?("#") }
280
+ new(words)
281
+ end
282
+
283
+ # Create indexed dictionary from a Trie.
284
+ #
285
+ # @param trie [Trie] The trie to convert
286
+ # @return [IndexedDictionary] New dictionary
287
+ def self.from_trie(trie)
288
+ words = trie.all_words
289
+ new(words)
290
+ end
291
+ end
292
+ end
293
+ end
@@ -0,0 +1,262 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Models
5
+ # Affix rule model for Hunspell-style affix processing.
6
+ #
7
+ # Affix rules define how prefixes and suffixes can be added or removed
8
+ # from words to generate morphological variants.
9
+ #
10
+ # This is a value object that represents a single affix rule.
11
+ #
12
+ # @note This class is immutable and frozen on initialization.
13
+ #
14
+ # @example Creating a prefix rule
15
+ # rule = Models::AffixRule.new(
16
+ # type: :prefix,
17
+ # flag: "A",
18
+ # strip: "",
19
+ # add: "re",
20
+ # condition: "."
21
+ # )
22
+ # rule.prefix? # => true
23
+ # rule.suffix? # => false
24
+ class AffixRule
25
+ # @return [Symbol] The affix type (:prefix or :suffix)
26
+ attr_reader :type
27
+
28
+ # @return [String] The flag character identifying this rule
29
+ attr_reader :flag
30
+
31
+ # @return [String] Characters to strip from the word
32
+ attr_reader :strip
33
+
34
+ # @return [String] Characters to add to the word
35
+ attr_reader :add
36
+
37
+ # @return [String, Regexp] Condition for applying this rule
38
+ attr_reader :condition
39
+
40
+ # @return [Boolean] Whether this is a cross-product rule
41
+ attr_reader :cross_product
42
+
43
+ # Affix rule types.
44
+ TYPES = {
45
+ prefix: "PFX",
46
+ suffix: "SFX"
47
+ }.freeze
48
+
49
+ # Create a new AffixRule.
50
+ #
51
+ # @param type [Symbol] The affix type (:prefix or :suffix)
52
+ # @param flag [String] The flag character
53
+ # @param strip [String] Characters to strip
54
+ # @param add [String] Characters to add
55
+ # @param condition [String, Regexp] Condition for applying
56
+ # @param cross_product [Boolean] Whether this is cross-product
57
+ def initialize(type:, flag:, strip:, add:, condition: ".", cross_product: false)
58
+ raise ArgumentError, "Invalid type: #{type}" unless %i[prefix suffix].include?(type)
59
+ raise ArgumentError, "Flag cannot be empty" if flag.nil? || flag.empty?
60
+
61
+ @type = type
62
+ @flag = flag.dup.freeze
63
+ @strip = strip.dup.freeze
64
+ @add = add.dup.freeze
65
+ @condition = condition.is_a?(Regexp) ? condition : compile_condition(condition)
66
+ @cross_product = cross_product
67
+
68
+ freeze
69
+ end
70
+
71
+ # Check if this is a prefix rule.
72
+ #
73
+ # @return [Boolean] True if prefix
74
+ def prefix?
75
+ @type == :prefix
76
+ end
77
+
78
+ # Check if this is a suffix rule.
79
+ #
80
+ # @return [Boolean] True if suffix
81
+ def suffix?
82
+ @type == :suffix
83
+ end
84
+
85
+ # Check if this rule can be applied to a word.
86
+ #
87
+ # @param word [String] The word to check
88
+ # @return [Boolean] True if the rule applies
89
+ def applies_to?(word)
90
+ return false if word.nil? || word.empty?
91
+
92
+ word.match?(@condition)
93
+ end
94
+
95
+ # Apply this rule to a word.
96
+ #
97
+ # @param word [String] The word to modify
98
+ # @return [String, nil] The modified word, or nil if rule doesn't apply
99
+ def apply(word)
100
+ return nil unless applies_to?(word)
101
+
102
+ if prefix?
103
+ # Strip from beginning, add prefix
104
+ word.start_with?(@strip) ? @add + word[@strip.length..] : nil
105
+ else
106
+ # Strip from end, add suffix
107
+ word.end_with?(@strip) ? word[0...-@strip.length] + @add : nil
108
+ end
109
+ end
110
+
111
+ # Remove this affix from a word (reverse operation).
112
+ #
113
+ # Unlike #apply, the condition is checked against the candidate base
114
+ # form (the result of stripping the affix), not the input. This makes
115
+ # #remove the proper inverse of #apply: every successful apply can be
116
+ # undone by a matching remove.
117
+ #
118
+ # @param word [String] The word to modify
119
+ # @return [String, nil] The stripped word, or nil if affix doesn't match
120
+ def remove(word)
121
+ return nil if word.nil? || word.empty?
122
+
123
+ candidate =
124
+ if prefix?
125
+ word.start_with?(@add) ? @strip + word[@add.length..] : nil
126
+ else
127
+ word.end_with?(@add) ? word[0...-@add.length] + @strip : nil
128
+ end
129
+
130
+ return nil if candidate.nil?
131
+ return nil unless candidate.match?(@condition)
132
+
133
+ candidate
134
+ end
135
+
136
+ # Get the Hunspell representation.
137
+ #
138
+ # @return [String] The affix line for Hunspell format
139
+ def to_hunspell
140
+ type_code = TYPES[@type]
141
+ cross = @cross_product ? "Y" : "N"
142
+ "#{type_code} #{@flag} #{cross} #{@strip.empty? ? '0' : @strip} " \
143
+ "#{@add} #{@condition.is_a?(Regexp) ? condition_to_s : @condition}"
144
+ end
145
+
146
+ # Check equality based on all attributes.
147
+ #
148
+ # @param other [AffixRule] The other rule
149
+ # @return [Boolean] True if equal
150
+ def ==(other)
151
+ return false unless other.is_a?(AffixRule)
152
+
153
+ @type == other.type &&
154
+ @flag == other.flag &&
155
+ @strip == other.strip &&
156
+ @add == other.add &&
157
+ @condition == other.condition &&
158
+ @cross_product == other.cross_product
159
+ end
160
+ alias eql? ==
161
+
162
+ # Hash based on all attributes.
163
+ #
164
+ # @return [Integer] Hash code
165
+ def hash
166
+ [@type, @flag, @strip, @add, @cross_product].hash
167
+ end
168
+
169
+ # Compare rules by flag.
170
+ #
171
+ # @param other [AffixRule] The other rule
172
+ # @return [Integer] Comparison result
173
+ def <=>(other)
174
+ return nil unless other.is_a?(AffixRule)
175
+
176
+ @flag <=> other.flag
177
+ end
178
+
179
+ private
180
+
181
+ # Compile condition string to regex.
182
+ #
183
+ # Hunspell uses '.' for match-all, '[...]' for character classes,
184
+ # and '^[...]' for negated classes. The compiled regex is anchored:
185
+ # suffix conditions anchor to end-of-string, prefix to start.
186
+ #
187
+ # @param condition [String] The condition string
188
+ # @return [Regexp] The compiled regex
189
+ def compile_condition(condition)
190
+ return // if condition == "."
191
+
192
+ regex_str = condition.dup
193
+
194
+ # Convert [...] to Ruby character class
195
+ regex_str = regex_str.gsub(/\[([^\]]+)\]/, "(?:\\1)")
196
+
197
+ # Convert ^[...] to negative lookahead
198
+ # Convert ^ to negative lookahead for single character
199
+ regex_str = regex_str.gsub("\\^(\\w)", "(?!\\1).")
200
+
201
+ # Anchor to end for suffix, beginning for prefix.
202
+ # The anchors themselves are NOT escaped — Ruby's Regexp needs
203
+ # the bare ^/$ metacharacters to mean start/end of line.
204
+ if @type == :suffix
205
+ Regexp.new("#{regex_str}$")
206
+ else
207
+ Regexp.new("^#{regex_str}")
208
+ end
209
+ end
210
+
211
+ # Convert regex condition back to a Hunspell condition string.
212
+ #
213
+ # @return [String] The condition string
214
+ def condition_to_s
215
+ source = @condition.source
216
+ return "." if source.empty?
217
+
218
+ # Strip the anchors added by compile_condition. Using a regex
219
+ # pattern with escaped metacharacters so gsub matches the literal
220
+ # ^ and $ characters, not the zero-width anchors.
221
+ source = source.gsub('^', "").gsub('$', "")
222
+
223
+ # Convert negative lookaheads back
224
+ source = source.gsub(/\(\?!([^)]+)\)\./, "^\\1")
225
+
226
+ # Convert non-capturing groups back to character classes
227
+ source.gsub('(?:', "[").gsub(')', "]")
228
+ end
229
+
230
+ # Create an affix rule from a Hunspell affix line.
231
+ #
232
+ # @param line [String] The affix line
233
+ # @param type [Symbol] The rule type (:prefix or :suffix)
234
+ # @return [AffixRule] New affix rule
235
+ #
236
+ # @example Parsing a Hunspell prefix rule
237
+ # AffixRule.from_hunspell("PFX A Y 1 re .", :prefix)
238
+ #
239
+ # @example Parsing a Hunspell suffix rule
240
+ # AffixRule.from_hunspell("SFX V N 2 ive e", :suffix)
241
+ def self.from_hunspell(line, type)
242
+ parts = line.split
243
+ return nil if parts.length < 5
244
+
245
+ flag = parts[1]
246
+ cross_product = parts[2] == "Y"
247
+ strip = parts[3] == "0" ? "" : parts[3]
248
+ add = parts[4]
249
+ condition = parts[5] || "."
250
+
251
+ new(
252
+ type: type,
253
+ flag: flag,
254
+ strip: strip,
255
+ add: add,
256
+ condition: condition,
257
+ cross_product: cross_product
258
+ )
259
+ end
260
+ end
261
+ end
262
+ end
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "lutaml/model"
4
+
5
+ module Kotoshu
6
+ module Models
7
+ module Result
8
+ # Result object for checking a document or file.
9
+ #
10
+ # Serialized via lutaml-model. Holds an Array<WordResult> in +errors+.
11
+ class DocumentResult < Lutaml::Model::Serializable
12
+ attribute :file, :string
13
+ attribute :word_count, :integer, default: 0
14
+ attribute :errors, WordResult, collection: true
15
+ # Errors absorbed by inline ignore directives or a baseline
16
+ # (plan 82); each entry is a real misspelling marked
17
+ # `suppressed: true` with `suppressed_by` naming what absorbed it.
18
+ attribute :suppressed_errors, WordResult, collection: true
19
+ attribute :metadata, :hash, default: {}
20
+
21
+ # Accept framework-private keywords (e.g. +lutaml_register+)
22
+ # from lutaml-model's +from_hash+ so the round-trip works
23
+ # without lutaml having to poke ivars directly.
24
+ def initialize(file: nil, errors: [], word_count: 0, metadata: {}, suppressed_errors: [], **kwargs)
25
+ super(
26
+ file: file,
27
+ word_count: word_count,
28
+ errors: errors,
29
+ suppressed_errors: suppressed_errors,
30
+ metadata: metadata,
31
+ **kwargs
32
+ )
33
+ end
34
+
35
+ def success?
36
+ Array(errors).empty?
37
+ end
38
+
39
+ def failed?
40
+ !success?
41
+ end
42
+
43
+ def error_count
44
+ Array(errors).size
45
+ end
46
+
47
+ def suppressed_count
48
+ Array(suppressed_errors).size
49
+ end
50
+
51
+ def unique_error_count
52
+ errors.map(&:word).uniq.size
53
+ end
54
+
55
+ def has_error_for?(word)
56
+ errors.any? { |e| e.word == word }
57
+ end
58
+
59
+ def errors_for(word)
60
+ errors.select { |e| e.word == word }
61
+ end
62
+
63
+ def each_error(&)
64
+ return enum_for(:each_error) unless block_given?
65
+
66
+ errors.each(&)
67
+ end
68
+
69
+ def each_unique_error(&)
70
+ return enum_for(:each_unique_error) unless block_given?
71
+
72
+ errors.group_by(&:word).each(&)
73
+ end
74
+
75
+ def first_errors(n = 10)
76
+ errors.first(n)
77
+ end
78
+
79
+ def error_summary
80
+ summary = Hash.new(0)
81
+ each_error { |error| summary[error.word] += 1 }
82
+ summary
83
+ end
84
+
85
+ def ==(other)
86
+ return false unless other.is_a?(DocumentResult)
87
+
88
+ file == other.file && errors == other.errors && suppressed_errors == other.suppressed_errors
89
+ end
90
+ alias eql? ==
91
+
92
+ def hash
93
+ [file, errors, suppressed_errors].hash
94
+ end
95
+
96
+ def to_s
97
+ if success?
98
+ if file
99
+ "File '#{file}': No spelling errors found (#{word_count} words checked)"
100
+ else
101
+ "No spelling errors found (#{word_count} words checked)"
102
+ end
103
+ else
104
+ prefix = file ? "File '#{file}':" : ""
105
+ "#{prefix} #{error_count} spelling error(s) found " \
106
+ "(#{unique_error_count} unique) in #{word_count} words"
107
+ end
108
+ end
109
+ alias inspect to_s
110
+
111
+ def self.success(file: nil, word_count: 0)
112
+ new(file: file, errors: [], word_count: word_count)
113
+ end
114
+
115
+ def self.failure(file: nil, errors: [], word_count: 0)
116
+ new(file: file, errors: errors, word_count: word_count)
117
+ end
118
+
119
+ def self.merge(results)
120
+ return new if results.empty?
121
+
122
+ new(
123
+ file: nil,
124
+ errors: results.flat_map(&:errors),
125
+ suppressed_errors: results.flat_map(&:suppressed_errors),
126
+ word_count: results.sum(&:word_count)
127
+ )
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end