kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,210 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ module Kotoshu
6
+ module Cli
7
+ # Catalog of commands the completion scripts know about, plus pure
8
+ # template builders that render shell-specific scripts from the
9
+ # catalog. Kept in a single module so the data and the rendering
10
+ # logic live next to the Thor subcommand that exposes them.
11
+ module Completions
12
+ # Describes a single CLI command for completion purposes.
13
+ Command = Struct.new(:name, :description, keyword_init: true)
14
+
15
+ # Top-level commands the bash/zsh/fish scripts will offer. Kept as
16
+ # a static catalog — descriptions are user-facing, so they are
17
+ # curated here rather than introspected from Thor (whose
18
+ # `description` field is often terse).
19
+ COMMANDS = [
20
+ Command.new(name: "check", description: "Check spelling in files, directories, or stdin"),
21
+ Command.new(name: "setup", description: "Set up languages (download or register local files)"),
22
+ Command.new(name: "status", description: "Show setup, cache, and runtime status"),
23
+ Command.new(name: "dict", description: "Dictionary operations"),
24
+ Command.new(name: "cache", description: "Cache management"),
25
+ Command.new(name: "completions", description: "Emit shell completion scripts"),
26
+ Command.new(name: "version", description: "Show version information"),
27
+ Command.new(name: "fetch", description: "Alias for `setup` (deprecated)")
28
+ ].freeze
29
+
30
+ # Commands whose first positional argument is a language code.
31
+ LANGUAGE_ARGUMENT_COMMANDS = %w[setup fetch].freeze
32
+
33
+ # Pure template builders. Each takes the catalog and renders a
34
+ # shell-specific completion script. They have no IO dependencies
35
+ # and are unit-tested in isolation.
36
+ module ScriptBuilders
37
+ module_function
38
+
39
+ # Indent every line of +text+ by +n+ spaces. Used to align
40
+ # interpolated blocks within their surrounding heredoc body.
41
+ def indent(text, n)
42
+ pad = " " * n
43
+ text.lines.map { |line| "#{pad}#{line}" }.join
44
+ end
45
+
46
+ module Bash
47
+ module_function
48
+
49
+ def build(commands, language_argument_commands)
50
+ word_list = commands.map(&:name).join(" ")
51
+ lang_branch = language_branch(language_argument_commands)
52
+ <<~BASH
53
+ # kotoshu bash completion — install with:
54
+ # kotoshu completions bash > /etc/bash_completion.d/kotoshu
55
+ # or for a single user:
56
+ # kotoshu completions bash > ~/.local/share/bash-completion/completions/kotoshu
57
+ _kotoshu_completions() {
58
+ local cur prev
59
+ cur="${COMP_WORDS[COMP_CWORD]}"
60
+ prev="${COMP_WORDS[COMP_CWORD-1]}"
61
+
62
+ if [ "$COMP_CWORD" -eq 1 ]; then
63
+ COMPREPLY=( $(compgen -W "#{word_list}" -- "$cur") )
64
+ return 0
65
+ fi
66
+
67
+ #{ScriptBuilders.indent(lang_branch, 2)}
68
+
69
+ return 0
70
+ }
71
+ complete -F _kotoshu_completions kotoshu
72
+ BASH
73
+ end
74
+
75
+ def language_branch(language_argument_commands)
76
+ return "# (no language-argument commands registered)" if language_argument_commands.empty?
77
+
78
+ matcher = language_argument_commands.join("|")
79
+ <<~BRANCH
80
+ case "$prev" in
81
+ #{matcher})
82
+ COMPREPLY=( $(compgen -W "$(kotoshu completions languages 2>/dev/null)" -- "$cur") )
83
+ return 0
84
+ ;;
85
+ esac
86
+ BRANCH
87
+ end
88
+ private_class_method :language_branch
89
+ end
90
+
91
+ module Zsh
92
+ module_function
93
+
94
+ def build(commands, language_argument_commands)
95
+ describe_block = commands.map { |c| "'#{c.name}:#{escape(c.description)}'" }.join(" ")
96
+ lang_case = language_case(language_argument_commands)
97
+ <<~ZSH
98
+ #compdef kotoshu
99
+ # kotoshu zsh completion — install with:
100
+ # kotoshu completions zsh > "${fpath[1]}/_kotoshu"
101
+
102
+ _kotoshu() {
103
+ local -a commands
104
+ commands=(
105
+ #{describe_block}
106
+ )
107
+
108
+ if (( CURRENT == 2 )); then
109
+ _describe 'command' commands
110
+ return
111
+ fi
112
+
113
+ #{ScriptBuilders.indent(lang_case, 2)}
114
+ }
115
+
116
+ _kotoshu "$@"
117
+ ZSH
118
+ end
119
+
120
+ def language_case(language_argument_commands)
121
+ return "# (no language-argument commands registered)" if language_argument_commands.empty?
122
+
123
+ matcher = language_argument_commands.join("|")
124
+ <<~CASE
125
+ case "$words[2]" in
126
+ #{matcher})
127
+ local -a langs
128
+ langs=("${(@f)$(kotoshu completions languages 2>/dev/null)}")
129
+ _describe 'language' langs
130
+ ;;
131
+ esac
132
+ CASE
133
+ end
134
+ private_class_method :language_case
135
+
136
+ def escape(text)
137
+ text.gsub("'", "''")
138
+ end
139
+ private_class_method :escape
140
+ end
141
+
142
+ module Fish
143
+ module_function
144
+
145
+ def build(commands, language_argument_commands)
146
+ command_lines = commands.map do |c|
147
+ "complete -c kotoshu -n \"__fish_use_subcommand\" -a \"#{c.name}\" -d \"#{escape(c.description)}\""
148
+ end.join("\n")
149
+ language_lines = language_argument_commands.map do |cmd|
150
+ matcher = "__fish_seen_subcommand_from #{cmd}"
151
+ "complete -c kotoshu -n \"#{matcher}\" -a \"(kotoshu completions languages 2>/dev/null)\""
152
+ end.join("\n")
153
+ <<~FISH
154
+ # kotoshu fish completion — install with:
155
+ # kotoshu completions fish > ~/.config/fish/completions/kotoshu.fish
156
+ #{command_lines}
157
+ #{language_lines}
158
+ FISH
159
+ end
160
+
161
+ def escape(text)
162
+ text.gsub('\\', '\\\\\\\\').gsub('"', '\\"')
163
+ end
164
+ private_class_method :escape
165
+ end
166
+ end
167
+ end
168
+
169
+ # Thor subcommand wired as `kotoshu completions <shell>`.
170
+ #
171
+ # Emits shell completion scripts for bash, zsh, and fish, plus a
172
+ # `languages` helper that prints supported language codes one per
173
+ # line. The shell scripts shell out to `kotoshu completions
174
+ # languages` for dynamic language completion, so newly registered
175
+ # languages appear without regenerating the script.
176
+ class CompletionsCommand < Thor
177
+ desc "bash", "Emit a bash completion script for the kotoshu CLI"
178
+ def bash
179
+ $stdout.puts Completions::ScriptBuilders::Bash.build(
180
+ Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
181
+ )
182
+ end
183
+
184
+ desc "zsh", "Emit a zsh completion script for the kotoshu CLI"
185
+ def zsh
186
+ $stdout.puts Completions::ScriptBuilders::Zsh.build(
187
+ Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
188
+ )
189
+ end
190
+
191
+ desc "fish", "Emit a fish completion script for the kotoshu CLI"
192
+ def fish
193
+ $stdout.puts Completions::ScriptBuilders::Fish.build(
194
+ Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
195
+ )
196
+ end
197
+
198
+ desc "languages", "Emit supported language codes, one per line"
199
+ long_desc <<~DESC
200
+ Prints every language code registered with
201
+ `Kotoshu::Language::Registry`, one per line. Used by the bash,
202
+ zsh, and fish completion scripts to populate the language list
203
+ dynamically.
204
+ DESC
205
+ def languages
206
+ Kotoshu::Language::Registry.supported_codes.each { |c| $stdout.puts c }
207
+ end
208
+ end
209
+ end
210
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ module Kotoshu
6
+ module Cli
7
+ # Dictionary command class.
8
+ #
9
+ # @example
10
+ # kotoshu dict list
11
+ # kotoshu dict info en-US
12
+ class DictCommand < Thor
13
+ desc "list", "List available dictionaries"
14
+ # Print the available dictionary types (one line each).
15
+ #
16
+ # @return [void]
17
+ def list
18
+ puts "Available dictionary types:"
19
+ puts " - unix_words: Unix system dictionary"
20
+ puts " - plain_text: Plain text word list"
21
+ puts " - custom: Custom in-memory dictionary"
22
+ puts " - hunspell: Hunspell (.dic/.aff)"
23
+ puts " - cspell: CSpell (.txt/.trie)"
24
+ end
25
+
26
+ desc "info TYPE", "Show information about a dictionary type"
27
+ # Print a short description of one dictionary type, or a hint to
28
+ # run `kotoshu dict list` for an unknown type.
29
+ #
30
+ # @param type [String, Symbol] One of unix_words, plain_text,
31
+ # custom, hunspell, cspell
32
+ # @return [void]
33
+ def info(type)
34
+ case type.to_sym
35
+ when :unix_words
36
+ puts "UnixWords Dictionary:"
37
+ puts " Reads from Unix system dictionary files"
38
+ puts " Default paths:"
39
+ puts " - /usr/share/dict/words"
40
+ puts " - /usr/share/dict/web2"
41
+ puts " - /usr/share/dict/american-english"
42
+ when :plain_text
43
+ puts "PlainText Dictionary:"
44
+ puts " Reads from plain text word lists"
45
+ puts " One word per line, # comments supported"
46
+ when :custom
47
+ puts "Custom Dictionary:"
48
+ puts " In-memory dictionary for user-defined words"
49
+ when :hunspell
50
+ puts "Hunspell Dictionary:"
51
+ puts " Reads Hunspell .dic and .aff files"
52
+ puts " Supports morphological affix rules"
53
+ when :cspell
54
+ puts "CSpell Dictionary:"
55
+ puts " Reads CSpell .txt or .trie files"
56
+ puts " Uses trie data structure for fast lookups"
57
+ else
58
+ puts "Unknown dictionary type: #{type}"
59
+ puts "Run 'kotoshu dict list' for available types"
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,146 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Cli
5
+ # Runs the check pipeline over a set of files and directories
6
+ # (plan 88): the CLI file mode, one target at a time, sharing one
7
+ # language resolution, baseline, and per-file output.
8
+ #
9
+ # Directories are expanded by DirectoryWalker; explicit file
10
+ # targets are checked as given. A baseline (when provided) is
11
+ # applied per file, exactly as single-file check does.
12
+ class DirectoryCheck
13
+ # Outcome of checking one file: the checked path, the check
14
+ # result, and the baseline application (nil without a
15
+ # baseline).
16
+ FileOutcome = Struct.new(:path, :result, :application, keyword_init: true)
17
+
18
+ attr_reader :outcomes
19
+
20
+ # @param roots [Array<String>] files and directories to check
21
+ # @param check [#call] callable taking the file text and an
22
+ # optional suggestions_filter: keyword and returning a
23
+ # Models::Result::DocumentResult (the CLI file-mode
24
+ # pipeline, including inline suppressions)
25
+ # @param baseline_path [String, nil] --baseline file
26
+ # @param include_globs [Array<String>] --include globs
27
+ # @param exclude_globs [Array<String>] --exclude globs
28
+ def initialize(roots, check:, baseline_path: nil, include_globs: [], exclude_globs: [])
29
+ @roots = Array(roots)
30
+ @check = check
31
+ @baseline_path = baseline_path
32
+ @walker = DirectoryWalker.new(include_globs: include_globs, exclude_globs: exclude_globs)
33
+ @outcomes = []
34
+ end
35
+
36
+ # Check every selected file.
37
+ #
38
+ # @return [Array<FileOutcome>] one outcome per checked file, in
39
+ # walk order
40
+ def run
41
+ @outcomes = target_files.map { |path| check_file(path) }
42
+ @outcomes
43
+ end
44
+
45
+ # The number of files checked.
46
+ #
47
+ # @return [Integer]
48
+ def file_count
49
+ @outcomes.size
50
+ end
51
+
52
+ # Total errors across all files, after baseline application.
53
+ #
54
+ # @return [Integer]
55
+ def error_count
56
+ @outcomes.sum { |outcome| outcome.result.error_count }
57
+ end
58
+
59
+ # Total words across all files.
60
+ #
61
+ # @return [Integer]
62
+ def word_count
63
+ @outcomes.sum { |outcome| outcome.result.word_count }
64
+ end
65
+
66
+ # Whether any file failed.
67
+ #
68
+ # @return [Boolean]
69
+ def failed?
70
+ @outcomes.any? { |outcome| outcome.result.failed? }
71
+ end
72
+
73
+ # Baseline-suppressed error count across all files.
74
+ #
75
+ # @return [Integer]
76
+ def suppressed_count
77
+ applications.sum(&:suppressed_count)
78
+ end
79
+
80
+ # Stale baseline entry count across all files.
81
+ #
82
+ # @return [Integer]
83
+ def stale_count
84
+ applications.sum(&:stale_count)
85
+ end
86
+
87
+ private
88
+
89
+ # The applications of the baseline across outcomes.
90
+ #
91
+ # @return [Array[Baseline::Store::Application>]
92
+ def applications
93
+ @outcomes.filter_map(&:application)
94
+ end
95
+
96
+ # Expand roots into a deduplicated file list: explicit files
97
+ # first (in argument order), then walked directory files.
98
+ #
99
+ # @return [Array<String>] file paths
100
+ def target_files
101
+ files = @roots.select { |root| File.file?(root) }
102
+ @roots.select { |root| File.directory?(root) }.each do |dir|
103
+ files.concat(@walker.files(dir))
104
+ end
105
+ files.each_with_object([]) do |path, unique|
106
+ unique << path unless unique.include?(path)
107
+ end
108
+ end
109
+
110
+ # Check one file and apply the baseline to its result.
111
+ #
112
+ # @param path [String] file path
113
+ # @return [FileOutcome]
114
+ def check_file(path)
115
+ text = File.read(path, encoding: Kotoshu.configuration.encoding).scrub
116
+ # Baseline-covered occurrences skip the suggestion sweep
117
+ # (plan 116): the filter answers false for exactly the
118
+ # occurrences apply_baseline will absorb.
119
+ filter = @baseline_path ? baseline_store.suggestions_filter_for(path) : nil
120
+ result = @check.call(text, suggestions_filter: filter)
121
+ application = apply_baseline(result, path)
122
+ result = application.result if application
123
+ FileOutcome.new(path: path, result: result, application: application)
124
+ end
125
+
126
+ # Apply the baseline to one file result, mirroring the CLI
127
+ # file-mode semantics.
128
+ #
129
+ # @param result [Models::Result::DocumentResult] check result
130
+ # @param path [String] file path
131
+ # @return [Baseline::Store::Application, nil]
132
+ def apply_baseline(result, path)
133
+ return nil unless @baseline_path
134
+
135
+ baseline_store.apply(result, file: path)
136
+ end
137
+
138
+ # Load the baseline store once.
139
+ #
140
+ # @return [Baseline::Store]
141
+ def baseline_store
142
+ @baseline_store ||= Baseline::Store.load(@baseline_path)
143
+ end
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Cli
5
+ # Walks directory trees and selects the text files to check
6
+ # (plan 88).
7
+ #
8
+ # Selection rules:
9
+ # - known text extensions by default: md markdown asciidoc adoc
10
+ # txt rst mdx
11
+ # - --include globs replace the extension default; --exclude
12
+ # globs always win
13
+ # - globs without a slash match the file basename; globs with a
14
+ # slash match the path relative to the checked root, with *
15
+ # and ? never crossing / and ** crossing directories
16
+ # - hidden files and directories (dot-prefixed) are skipped, as
17
+ # are .git, node_modules, vendor and target directories
18
+ # - .gitignore and .ignore files are honored (see IgnoreMatcher
19
+ # for the supported subset); nested ignore files only apply to
20
+ # their own subtree
21
+ class DirectoryWalker
22
+ TEXT_EXTENSIONS = %w[.md .markdown .asciidoc .adoc .txt .rst .mdx].freeze
23
+ DEFAULT_SKIP_DIRS = %w[.git node_modules vendor target].freeze
24
+
25
+ # @param include_globs [Array<String>] --include globs
26
+ # @param exclude_globs [Array<String>] --exclude globs
27
+ def initialize(include_globs: [], exclude_globs: [])
28
+ @include_globs = Array(include_globs)
29
+ @exclude_globs = Array(exclude_globs)
30
+ end
31
+
32
+ # Collect the checkable files under one root, sorted
33
+ # deterministically.
34
+ #
35
+ # @param root [String] directory to walk
36
+ # @return [Array<String>] file paths rooted at +root+
37
+ def files(root)
38
+ walk(root, [], [])
39
+ end
40
+
41
+ private
42
+
43
+ # Recursively collect files, carrying the ignore matchers of
44
+ # every ancestor directory paired with their depth below the
45
+ # root, so each matcher sees paths relative to its own
46
+ # directory (gitignore scoping).
47
+ #
48
+ # Selection and ignore matching use forward-slash relative
49
+ # paths (the gitignore/glob convention), independent of the
50
+ # platforms path separator; returned file paths are
51
+ # platform-native.
52
+ #
53
+ # @param directory [String] current absolute directory
54
+ # @param relative [Array<String>] path segments below the root
55
+ # @param matchers [Array<[IgnoreMatcher, Integer]>] ancestor
56
+ # matchers with their depths
57
+ # @return [Array<String>] collected file paths
58
+ def walk(directory, relative, matchers)
59
+ if IgnoreMatcher.present_in?(directory)
60
+ matchers = matchers + [[IgnoreMatcher.load_directory(directory), relative.length]]
61
+ end
62
+
63
+ files = []
64
+ Dir.children(directory).sort.each do |entry|
65
+ path = File.join(directory, entry)
66
+ entry_relative = relative + [entry]
67
+ next if hidden?(entry)
68
+ next if ignored_by_any?(matchers, entry_relative, File.directory?(path))
69
+
70
+ if File.directory?(path)
71
+ next if DEFAULT_SKIP_DIRS.include?(entry)
72
+
73
+ files.concat(walk(path, entry_relative, matchers))
74
+ elsif selected?(entry, entry_relative.join("/"))
75
+ files << path
76
+ end
77
+ end
78
+ files
79
+ end
80
+
81
+ # Whether a path is ignored by any scoped matcher.
82
+ #
83
+ # @param matchers [Array<[IgnoreMatcher, Integer]>] matchers in
84
+ # scope with their depths
85
+ # @param entry_relative [Array<String>] path segments below root
86
+ # @param directory [Boolean] whether the entry is a directory
87
+ # @return [Boolean]
88
+ def ignored_by_any?(matchers, entry_relative, directory)
89
+ matchers.any? do |matcher, depth|
90
+ scoped = entry_relative[depth..].join("/")
91
+ matcher.ignored?(scoped, directory: directory)
92
+ end
93
+ end
94
+
95
+ # Whether an entry name is hidden.
96
+ #
97
+ # @param entry [String] directory entry name
98
+ # @return [Boolean]
99
+ def hidden?(entry)
100
+ entry.start_with?(".")
101
+ end
102
+
103
+ # Whether a file is selected for checking.
104
+ #
105
+ # @param basename [String] file name
106
+ # @param relative [String] path relative to the root
107
+ # @return [Boolean]
108
+ def selected?(basename, relative)
109
+ return false if globbed?(@exclude_globs, basename, relative)
110
+ return globbed?(@include_globs, basename, relative) unless @include_globs.empty?
111
+
112
+ TEXT_EXTENSIONS.include?(File.extname(basename))
113
+ end
114
+
115
+ # Whether any glob matches the file.
116
+ #
117
+ # @param globs [Array<String>] glob patterns
118
+ # @param basename [String] file name
119
+ # @param relative [String] path relative to the root
120
+ # @return [Boolean]
121
+ def globbed?(globs, basename, relative)
122
+ globs.any? do |glob|
123
+ if glob.include?("/")
124
+ File.fnmatch(glob, relative, File::FNM_PATHNAME)
125
+ else
126
+ File.fnmatch(glob, basename)
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end