kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for sentence-start capitalization rules.
7
+ #
8
+ # English (and most Latin-script languages) capitalizes the
9
+ # first word of every sentence. This matcher walks the token
10
+ # stream tracking sentence boundaries and flags lowercase
11
+ # sentence starts.
12
+ #
13
+ # A sentence starts:
14
+ # - at the beginning of the stream, OR
15
+ # - immediately after a token whose final character is a
16
+ # sentence-ending punctuation mark (. ! ?).
17
+ #
18
+ # The matcher only considers alphabetic tokens. Numbers,
19
+ # symbols, and whitespace don't reset the sentence-start flag.
20
+ #
21
+ # Rule shape:
22
+ #
23
+ # patterns:
24
+ # - conditions:
25
+ # - type: sentence_start_check
26
+ #
27
+ class SentenceStartMatcher < BaseMatcher
28
+ SENTENCE_ENDERS = %w[. ! ?].freeze
29
+
30
+ def match(tokens, rule)
31
+ errors = []
32
+ expecting_capital = true
33
+
34
+ tokens.each do |token|
35
+ word = token[:token]
36
+ next if word.nil? || word.empty?
37
+
38
+ if sentence_start_word?(word, expecting_capital)
39
+ errors << build_error(token, rule)
40
+ end
41
+
42
+ # Any non-empty token resets the sentence-start expectation.
43
+ # Only sentence-ending punctuation re-arms it (below).
44
+ expecting_capital = false
45
+ expecting_capital = true if ends_sentence?(word)
46
+ end
47
+
48
+ errors
49
+ end
50
+
51
+ protected
52
+
53
+ def sentence_start_word?(word, expecting_capital)
54
+ expecting_capital && alphabetic_word?(word) && first_letter_lowercase?(word)
55
+ end
56
+
57
+ def alphabetic_word?(word)
58
+ !word.nil? && !word.empty? && word[0].match?(/[[:alpha:]]/)
59
+ end
60
+
61
+ def first_letter_lowercase?(word)
62
+ first = word[0]
63
+ first == first.downcase && first != first.upcase
64
+ end
65
+
66
+ def ends_sentence?(word)
67
+ SENTENCE_ENDERS.include?(word[-1])
68
+ end
69
+
70
+ def build_error(token, rule)
71
+ capitalized = token[:token].sub(/\A(\w)/) { $1.upcase }
72
+ {
73
+ rule_id: rule.id,
74
+ position: token[:position],
75
+ message: rule.message,
76
+ suggestion: capitalized,
77
+ context: "\"#{token[:token]}\"",
78
+ suggestions: [capitalized]
79
+ }
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for a/an article usage rules.
7
+ #
8
+ # This matcher checks if "a" or "an" is used correctly before
9
+ # vowel and consonant sounds.
10
+ class VowelSoundMatcher < BaseMatcher
11
+ VOWEL_SOUNDS = %w[a e i o u].freeze
12
+
13
+ # Match tokens against the a/an pattern.
14
+ #
15
+ # @param tokens [Array<Hash>] Array of token hashes
16
+ # @param rule [Rule] The rule being checked
17
+ # @return [Array<Hash>] Array of error hashes
18
+ def match(tokens, rule)
19
+ errors = []
20
+ tokens.each_cons(2) do |prev_token, current_token|
21
+ prev_word = prev_token[:token]&.downcase
22
+ next unless %w[a an].include?(prev_word)
23
+ next unless prev_token[:pos_tag] == 'DET' || prev_token[:pos_tag].nil?
24
+
25
+ next_word = current_token[:token]
26
+ next if next_word.nil? || next_word.empty?
27
+
28
+ expected = article_for(next_word, rule)
29
+ if prev_word != expected
30
+ errors << build_error(prev_token, current_token, expected, rule)
31
+ end
32
+ end
33
+ errors
34
+ end
35
+
36
+ private
37
+
38
+ # Determine the correct article for a word.
39
+ #
40
+ # @param word [String] The word to check
41
+ # @param rule [Rule] The rule with exceptions
42
+ # @return [String] "a" or "an"
43
+ def article_for(word, rule)
44
+ word_downcase = word.downcase
45
+ exceptions = rule.exceptions || {}
46
+
47
+ consonant_exceptions = exceptions['consonant_sound_exceptions'] || []
48
+ return 'a' if consonant_exceptions.include?(word_downcase)
49
+
50
+ silent_exceptions = exceptions['silent_consonant_exceptions'] || []
51
+ return 'an' if silent_exceptions.include?(word_downcase)
52
+
53
+ first_char = word_downcase[0]
54
+ VOWEL_SOUNDS.include?(first_char) ? 'an' : 'a'
55
+ end
56
+
57
+ # Build an error hash.
58
+ #
59
+ # @param prev_token [Hash] The previous token (article)
60
+ # @param current_token [Hash] The current token (word)
61
+ # @param expected [String] The expected article
62
+ # @param rule [Rule] The rule being checked
63
+ # @return [Hash] Error hash
64
+ def build_error(prev_token, current_token, expected, rule)
65
+ prev_word = prev_token[:token]
66
+ next_word = current_token[:token]
67
+ message = rule.message.gsub('{expected}', expected).gsub('{word}', next_word)
68
+
69
+ {
70
+ rule_id: rule.id,
71
+ position: prev_token[:position],
72
+ message: message,
73
+ suggestion: expected,
74
+ context: "#{prev_word} #{next_word}",
75
+ suggestions: [expected]
76
+ }
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ module PatternMatchers
6
+ # Matcher for word-list corrections.
7
+ #
8
+ # Flags any token whose lowercase form is a key in the rule's
9
+ # +corrections+ map. Useful for proper-noun capitalization
10
+ # (monday → Monday, english → English) where a single rule
11
+ # covers dozens of word pairs.
12
+ #
13
+ # Rule shape:
14
+ #
15
+ # patterns:
16
+ # - conditions:
17
+ # - type: word_list_check
18
+ # corrections:
19
+ # monday: Monday
20
+ # tuesday: Tuesday
21
+ # english: English
22
+ #
23
+ # The matcher only fires when the actual token text differs from
24
+ # the mapped correction (so "Monday" doesn't get flagged).
25
+ #
26
+ class WordListMatcher < BaseMatcher
27
+ def match(tokens, rule)
28
+ corrections = word_list_condition&.dig('corrections') || {}
29
+ return [] if corrections.empty?
30
+
31
+ errors = []
32
+ tokens.each do |token|
33
+ word = token[:token]
34
+ next unless word
35
+
36
+ correction = corrections[word.downcase]
37
+ next unless correction
38
+ next if word == correction # already correct
39
+
40
+ errors << build_error(token, correction, rule)
41
+ end
42
+ errors
43
+ end
44
+
45
+ private
46
+
47
+ def word_list_condition
48
+ @word_list_condition ||= @pattern['conditions']&.find do |c|
49
+ c['type'] == 'word_list_check'
50
+ end
51
+ end
52
+
53
+ def build_error(token, correction, rule)
54
+ {
55
+ rule_id: rule.id,
56
+ position: token[:position],
57
+ message: rule.message,
58
+ suggestion: correction,
59
+ context: "\"#{token[:token]}\"",
60
+ suggestions: [correction]
61
+ }
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ # Base class for grammar rules.
6
+ #
7
+ # All grammar rules inherit from this class and implement
8
+ # the #check method to validate tokens.
9
+ class Rule
10
+ attr_reader :id, :name, :category, :severity, :description,
11
+ :exceptions, :message, :suggestion
12
+
13
+ def initialize(id:, name:, category:, severity:, description:,
14
+ patterns:, message:, suggestion:, exceptions: {})
15
+ @id = id
16
+ @name = name
17
+ @category = category
18
+ @severity = severity
19
+ @description = description
20
+ @patterns = patterns
21
+ @exceptions = exceptions
22
+ @message = message
23
+ @suggestion = suggestion
24
+ end
25
+
26
+ # Factory method to create Rule from YAML configuration.
27
+ #
28
+ # @param config [Hash] YAML configuration hash
29
+ # @return [Rule] A new rule instance
30
+ def self.from_yaml(config)
31
+ new(
32
+ id: config['id'],
33
+ name: config['name'],
34
+ category: config['category'],
35
+ severity: config['severity'],
36
+ description: config['description'],
37
+ patterns: config['patterns'],
38
+ exceptions: config['exceptions'] || {},
39
+ message: config['message'],
40
+ suggestion: config['suggestion']
41
+ )
42
+ end
43
+
44
+ # Check tokens against this rule.
45
+ #
46
+ # @param tokens [Array<Hash>] Array of token hashes
47
+ # @return [Array<Hash>] Array of error hashes
48
+ def check(tokens)
49
+ errors = []
50
+ @patterns.each do |pattern|
51
+ pattern_errors = check_pattern(tokens, pattern)
52
+ errors.concat(pattern_errors)
53
+ end
54
+ errors
55
+ end
56
+
57
+ private
58
+
59
+ # Check a single pattern against tokens.
60
+ #
61
+ # @param tokens [Array<Hash>] Array of token hashes
62
+ # @param pattern [Hash] Pattern configuration hash
63
+ # @return [Array<Hash>] Array of error hashes
64
+ def check_pattern(tokens, pattern)
65
+ matcher = create_matcher(pattern)
66
+ matcher.match(tokens, self)
67
+ end
68
+
69
+ # Create appropriate pattern matcher based on pattern type.
70
+ #
71
+ # @param pattern [Hash] Pattern configuration hash
72
+ # @return [PatternMatchers::BaseMatcher] A pattern matcher instance
73
+ def create_matcher(pattern)
74
+ conditions = pattern['conditions'] || []
75
+ return PatternMatchers::BaseMatcher.new(pattern) if conditions.empty?
76
+
77
+ condition_types = conditions.map { |c| c['type'] }
78
+
79
+ if condition_types.include?('vowel_check')
80
+ PatternMatchers::VowelSoundMatcher.new(pattern, exceptions)
81
+ elsif condition_types.include?('context_check')
82
+ PatternMatchers::PossessiveContextMatcher.new(pattern, exceptions)
83
+ elsif condition_types.include?('possessive_contraction_check')
84
+ PatternMatchers::PossessiveContractionMatcher.new(pattern, exceptions)
85
+ elsif condition_types.include?('distance_check')
86
+ PatternMatchers::DoubleNegativeMatcher.new(pattern, exceptions)
87
+ elsif condition_types.include?('phrase_check')
88
+ PatternMatchers::PhraseMatcher.new(pattern, exceptions)
89
+ elsif condition_types.include?('sentence_start_check')
90
+ PatternMatchers::SentenceStartMatcher.new(pattern, exceptions)
91
+ elsif condition_types.include?('word_list_check')
92
+ PatternMatchers::WordListMatcher.new(pattern, exceptions)
93
+ else
94
+ PatternMatchers::BaseMatcher.new(pattern)
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Grammar
5
+ # Engine for loading and executing grammar rules from YAML configuration.
6
+ #
7
+ # This implements configuration-driven design where all linguistic data
8
+ # (rules, patterns, exceptions) is stored in YAML files, not hardcoded.
9
+ #
10
+ # @example Loading rules for English
11
+ # engine = RuleEngine.new(language: 'en')
12
+ # errors = engine.check(tokens)
13
+ #
14
+ class RuleEngine
15
+ attr_reader :language, :rules
16
+
17
+ # Create a new rule engine for a language.
18
+ #
19
+ # @param language [String] Language code (e.g., 'en', 'de', 'fr')
20
+ # @param rules_path [String, nil] Optional custom path to grammar rules
21
+ # @param dictionaries_path [String, nil] Optional custom path to dictionaries directory
22
+ def initialize(language:, rules_path: nil, dictionaries_path: nil)
23
+ @language = language
24
+ @rules_path = rules_path || default_rules_path(dictionaries_path)
25
+ @loader = RuleLoader.new(@rules_path)
26
+ @rules = @loader.load_rules
27
+ end
28
+
29
+ # Check tokens against all loaded rules.
30
+ #
31
+ # @param tokens [Array<Hash>] Array of token hashes with :token, :pos_tag, :position keys
32
+ # @return [Array<Hash>] Array of error hashes
33
+ def check(tokens)
34
+ errors = []
35
+ @rules.each do |rule|
36
+ rule_errors = rule.check(tokens)
37
+ errors.concat(rule_errors)
38
+ end
39
+ errors
40
+ end
41
+
42
+ # Get list of rule IDs.
43
+ #
44
+ # @return [Array<String>] Array of rule IDs
45
+ def rule_names
46
+ @rules.map(&:id)
47
+ end
48
+
49
+ # Get a specific rule by ID.
50
+ #
51
+ # @param id [String] Rule ID
52
+ # @return [Rule, nil] The rule if found, nil otherwise
53
+ def get_rule(id)
54
+ @rules.find { |r| r.id == id }
55
+ end
56
+
57
+ # Check if a rule exists.
58
+ #
59
+ # @param id [String] Rule ID
60
+ # @return [Boolean] True if rule exists
61
+ def rule_exists?(id)
62
+ @rules.any? { |r| r.id == id }
63
+ end
64
+
65
+ private
66
+
67
+ # Resolve the directory holding <language>/grammar/rules.yaml.
68
+ #
69
+ # Priority:
70
+ # 1. Explicit `dictionaries_path:` constructor arg
71
+ # 2. KOTOSHU_DICTIONARIES_PATH env var
72
+ # 3. Configuration#dictionaries_path (when set)
73
+ # 4. Gem-bundled data/grammar (always available)
74
+ #
75
+ # @param dictionaries_path [String, nil] Caller-supplied override
76
+ # @return [String] Path joined with language + "grammar"
77
+ def default_rules_path(dictionaries_path = nil)
78
+ base = dictionaries_path || resolve_dictionaries_base
79
+ File.join(base, @language, "grammar")
80
+ end
81
+
82
+ def resolve_dictionaries_base
83
+ ENV["KOTOSHU_DICTIONARIES_PATH"] ||
84
+ Configuration.instance.dictionaries_path ||
85
+ gem_bundled_data_path
86
+ end
87
+
88
+ # Path to the gem's own data/ directory. Always available after
89
+ # `gem install kotoshu`, so grammar rules work without external
90
+ # dependencies or sibling repos checked out.
91
+ def gem_bundled_data_path
92
+ File.expand_path("../../../data", __dir__)
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+
5
+ module Kotoshu
6
+ module Grammar
7
+ # Loads grammar rules from YAML configuration files.
8
+ #
9
+ # This class reads rule definitions from YAML files in the
10
+ # dictionaries/{language}/grammar/ directory.
11
+ class RuleLoader
12
+ def initialize(rules_path)
13
+ @rules_path = rules_path
14
+ end
15
+
16
+ # Load all rules from the rules.yaml file.
17
+ #
18
+ # @return [Array<Rule>] Array of rule instances
19
+ def load_rules
20
+ rules_file = File.join(@rules_path, 'rules.yaml')
21
+ return [] unless File.exist?(rules_file)
22
+
23
+ config = YAML.load_file(rules_file)
24
+ return [] unless config && config['rules']
25
+
26
+ config['rules'].map { |rule_config| Rule.from_yaml(rule_config) }
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Grammar rules infrastructure for Kotoshu.
5
+ #
6
+ # This module provides configuration-driven grammar checking
7
+ # where all linguistic data is stored in YAML files.
8
+ module Grammar
9
+ autoload :Rule, "kotoshu/grammar/rule"
10
+ autoload :RuleEngine, "kotoshu/grammar/rule_engine"
11
+ autoload :RuleLoader, "kotoshu/grammar/rule_loader"
12
+
13
+ module PatternMatchers
14
+ autoload :BaseMatcher, "kotoshu/grammar/pattern_matchers/base_matcher"
15
+ autoload :VowelSoundMatcher, "kotoshu/grammar/pattern_matchers/vowel_sound_matcher"
16
+ autoload :PossessiveContextMatcher, "kotoshu/grammar/pattern_matchers/possessive_context_matcher"
17
+ autoload :PossessiveContractionMatcher, "kotoshu/grammar/pattern_matchers/possessive_contraction_matcher"
18
+ autoload :DoubleNegativeMatcher, "kotoshu/grammar/pattern_matchers/double_negative_matcher"
19
+ autoload :PhraseMatcher, "kotoshu/grammar/pattern_matchers/phrase_matcher"
20
+ autoload :SentenceStartMatcher, "kotoshu/grammar/pattern_matchers/sentence_start_matcher"
21
+ autoload :WordListMatcher, "kotoshu/grammar/pattern_matchers/word_list_matcher"
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,167 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "time"
5
+ require "fileutils"
6
+
7
+ module Kotoshu
8
+ module Integrity
9
+ # Append-only JSON audit log of every resource download.
10
+ #
11
+ # Each entry is one JSON object per line, written to +audit.log+
12
+ # under the configured Kotoshu home directory. The log is consulted
13
+ # by users when investigating "what did Kotoshu fetch?" and by CI
14
+ # for reproducibility audits.
15
+ #
16
+ # Statuses:
17
+ # "verified" — content matched manifest entry's SHA-256
18
+ # "unverified" — no manifest entry available; bytes trusted as-is
19
+ # "mismatch" — SHA-256 mismatch (also raises IntegrityError)
20
+ # "missing" — attempted download failed (network, 404, etc.)
21
+ #
22
+ # The log is opened, appended, and closed per entry — no long-lived
23
+ # file handle. Writes are line-buffered and fsync'd so the record
24
+ # survives a crash mid-batch.
25
+ #
26
+ # == Rotation
27
+ #
28
+ # When the current file exceeds +rotation_policy.max_bytes+, the
29
+ # policy produces a rename plan (see {RotationPolicy#plan_for}) and
30
+ # the log shifts the existing rotations up by one slot before
31
+ # writing the new entry to a fresh current file. The total on-disk
32
+ # footprint is bounded at +max_bytes * (rotations + 1)+.
33
+ #
34
+ # Rotation happens under an exclusive flock on a sibling lockfile
35
+ # (+audit.log.lock+) — not on the log itself — because the log path
36
+ # moves during rotation and the lock would otherwise travel with it.
37
+ class AuditLog
38
+ # Default location: $XDG_DATA_HOME/kotoshu/audit.log
39
+ # (~/.local/share/kotoshu/audit.log), or $KOTOSHU_AUDIT_LOG.
40
+ def self.default_path
41
+ Kotoshu::Paths.audit_log_path
42
+ end
43
+
44
+ attr_reader :path, :rotation_policy
45
+
46
+ # @param path [String] Override the log file location.
47
+ # @param rotation_policy [RotationPolicy, nil] When nil, no
48
+ # rotation is performed (the log grows unbounded). Pass a
49
+ # +RotationPolicy+ instance to enable bounded rotation.
50
+ def initialize(path: self.class.default_path, rotation_policy: nil)
51
+ @path = path
52
+ @rotation_policy = rotation_policy
53
+ end
54
+
55
+ # Record one download attempt. Returns the written entry hash.
56
+ #
57
+ # @param url [String] Source URL
58
+ # @param size [Integer, nil] Bytes downloaded (nil on missing)
59
+ # @param sha256 [String, nil] Computed SHA-256 of bytes (nil on missing)
60
+ # @param manifest_sha256 [String, nil] Expected SHA-256 from manifest
61
+ # @param status [String] One of: verified, unverified, mismatch, missing
62
+ # @param resource_id [String, nil] Caller-supplied resource identifier
63
+ def record(url:, status:, size: nil, sha256: nil,
64
+ manifest_sha256: nil, resource_id: nil)
65
+ entry = {
66
+ timestamp: Time.now.utc.iso8601,
67
+ url: url,
68
+ resource_id: resource_id,
69
+ size: size,
70
+ sha256: sha256,
71
+ manifest_sha256: manifest_sha256,
72
+ status: status
73
+ }
74
+ FileUtils.mkdir_p(File.dirname(@path))
75
+ with_exclusive_lock do
76
+ rotate_if_needed!
77
+ File.open(@path, "a", encoding: "UTF-8") do |f|
78
+ f.write("#{entry.to_json}\n")
79
+ f.fsync
80
+ end
81
+ end
82
+ entry
83
+ end
84
+
85
+ # Iterate every recorded entry (parsed Hashes) across the current
86
+ # log and all rotations, newest-first.
87
+ #
88
+ # Newest-first means: within each file, lines are yielded in reverse
89
+ # write order (the last appended entry first); files are visited in
90
+ # order current, .1, .2, ... (newest file first). This guarantees a
91
+ # strictly decreasing write-time ordering across the whole log,
92
+ # which is what an auditor scanning "what did kotoshu fetch most
93
+ # recently?" expects.
94
+ def each(&block)
95
+ return enum_for(:each) unless block
96
+
97
+ each_file_reversed(@path, &block)
98
+ return unless rotation_policy&.rotations&.positive?
99
+
100
+ 1.upto(rotation_policy.rotations) do |n|
101
+ each_file_reversed("#{@path}.#{n}", &block)
102
+ end
103
+ end
104
+
105
+ def entries
106
+ each.to_a
107
+ end
108
+
109
+ def clear!
110
+ with_exclusive_lock do
111
+ FileUtils.rm_f(@path)
112
+ clear_rotations
113
+ end
114
+ end
115
+
116
+ private
117
+
118
+ # Yield parsed entries from +path+ newest-first (reverse line order).
119
+ # No-op when the file doesn't exist. Each individual rotation file is
120
+ # bounded by +RotationPolicy.max_bytes+, so loading one file's lines
121
+ # into memory before reversing is acceptable.
122
+ def each_file_reversed(path)
123
+ return unless File.exist?(path)
124
+
125
+ File.readlines(path, encoding: "UTF-8")
126
+ .map(&:strip)
127
+ .reject(&:empty?)
128
+ .reverse_each { |line| yield JSON.parse(line) }
129
+ end
130
+
131
+ def clear_rotations
132
+ return unless rotation_policy&.rotations&.positive?
133
+
134
+ 1.upto(rotation_policy.rotations) do |n|
135
+ FileUtils.rm_f("#{@path}.#{n}")
136
+ end
137
+ end
138
+
139
+ def rotate_if_needed!
140
+ return unless rotation_policy
141
+ return unless File.exist?(@path)
142
+ return unless rotation_policy.rotate?(File.size(@path))
143
+
144
+ plan = rotation_policy.plan_for(@path)
145
+ plan[:deletes].each { |p| FileUtils.rm_f(p) }
146
+ plan[:moves].each do |src, dst|
147
+ FileUtils.mv(src, dst) if File.exist?(src)
148
+ end
149
+ end
150
+
151
+ # Exclusive flock on a stable sibling lockfile. The log itself is
152
+ # a poor lock target because rotation renames it out from under
153
+ # any writer mid-flight.
154
+ def with_exclusive_lock
155
+ FileUtils.mkdir_p(File.dirname(@path))
156
+ File.open(lockfile_path, File::RDWR | File::CREAT, 0o644) do |lock|
157
+ lock.flock(File::LOCK_EX)
158
+ yield
159
+ end
160
+ end
161
+
162
+ def lockfile_path
163
+ "#{@path}.lock"
164
+ end
165
+ end
166
+ end
167
+ end