kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
data/CLAUDE.md ADDED
@@ -0,0 +1,176 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## What Kotoshu Is
6
+
7
+ Kotoshu 「言修」 is a **semantic** spell checker for Ruby. It pairs a traditional
8
+ dictionary/affix backend (Hunspell-style) with ONNX-converted FastText word
9
+ embeddings for context-aware suggestions. The README.adoc is the authoritative
10
+ user-facing description; this file is the contributor-facing map.
11
+
12
+ Key dependencies (`kotoshu.gemspec`): `thor` (CLI), `suika` (tokenizer),
13
+ `onnxruntime` (semantic inference). Ruby 3.1+.
14
+
15
+ ## Development Commands
16
+
17
+ ```bash
18
+ bundle exec rspec # Run the full test suite
19
+ bundle exec rspec spec/path/to_spec.rb # Run one file
20
+ bundle exec rspec -e "matches a word" # Run examples matching a name
21
+ bundle exec rspec --only-failures # Rerun just failing examples (uses .rspec_status)
22
+
23
+ NETWORK_TESTS=1 bundle exec rspec # Opt INTO tests that download dictionaries
24
+ ONNX_TESTS=1 bundle exec rspec # Opt INTO tests that need onnxruntime + cached models
25
+ SLOW_TESTS=1 bundle exec rspec # Opt INTO benchmarks and full-dictionary sweeps
26
+ bundle exec rubocop # Lint
27
+ bundle exec rubocop -A # Lint with safe auto-fix
28
+ bundle exec rake # default task = spec + rubocop
29
+
30
+ bundle exec bin/console # IRB with Kotoshu loaded
31
+ bundle exec exe/kotoshu check FILE # Run the CLI locally
32
+ gem build kotoshu.gemspec && gem install kotoshu-*.gem
33
+ ```
34
+
35
+ Notes that aren't obvious from the Rakefile:
36
+ - `spec/spec_helper.rb` excludes anything tagged `:network` unless `NETWORK_TESTS=1` is set — those specs download dictionaries from GitHub and are slow/flaky.
37
+ - `spec/spec_helper.rb` excludes anything tagged `:onnx` unless `ONNX_TESTS=1` is set — those specs need `onnxruntime` installed **and** the language model cached via `kotoshu setup :en --model`.
38
+ - `spec/spec_helper.rb` excludes anything tagged `:slow` unless `SLOW_TESTS=1` is set — those are timing-sensitive benchmarks and full-dictionary sweeps.
39
+ - SimpleCov runs on every `rspec` invocation (configured in `spec_helper.rb`).
40
+ - `spec/spylls_test_helper.rb` is mixed into every spec. It ports Hunspell's reference test fixtures from [Splylls](https://github.com/neolithos/spylls) (the Python Hunspell port); many specs assert behavior against those fixtures.
41
+
42
+ ## Architecture
43
+
44
+ Kotoshu has **two parallel checking paths** that share infrastructure:
45
+
46
+ 1. **Traditional path** — `Kotoshu::Spellchecker` (facade) → `Suggestions::Generator` → pluggable `Dictionary::*` backends + `Suggestions::Strategies::*` algorithms. This is what `Kotoshu.correct?` / `Kotoshu.suggest` / `Kotoshu.check` use.
47
+ 2. **Semantic path** — `Analyzers::SemanticAnalyzer` driven by an `Models::EmbeddingModel` (`FastTextModel` or `OnnxModel`). Used for context-aware reranking and OOV handling. This path is **opt-in** and only loads when needed.
48
+
49
+ ### Layer map
50
+
51
+ ```
52
+ exe/kotoshu ─► lib/kotoshu/cli.rb (Kotoshu::Cli::Cli < Thor)
53
+ subcommands: check, dict (DictCommand), cache (CacheCommand)
54
+ helpers: cli/interactive_reviewer, cli/batch_reporter,
55
+ cli/navigation_manager, cli/display_formatter
56
+
57
+ Kotoshu module (lib/kotoshu.rb) ─► public facade methods
58
+ .correct? .suggest .check .check_file .detect_language ...
59
+ all delegate to a singleton Spellchecker
60
+
61
+ Spellchecker ─► Configuration ─► Dictionary::Repository ─► Dictionary::*
62
+ │
63
+ └─► Suggestions::Generator
64
+ └─► Strategies::CompositeStrategy
65
+ (edit_distance, phonetic,
66
+ keyboard_proximity, ngram,
67
+ symspell, semantic)
68
+
69
+ SemanticAnalyzer ─► Models::OnnxModel | Models::FastTextModel
70
+ └─► Embeddings::* (vocabulary, similarity search, LRU cache)
71
+ ```
72
+
73
+ ### Loading strategy
74
+
75
+ `lib/kotoshu.rb` eagerly `require_relative`s the traditional path (core models, dictionaries, strategies, configuration, spellchecker) and `autoload`s the heavier / optional pieces (ONNX models, documents, interactive CLI, caches, language detection, debug/metrics). When adding a new top-level component, follow the existing split: eager-load only what the facade needs at boot; autoload the rest.
76
+
77
+ **ONNX is a soft dependency.** `onnxruntime` is NOT in `kotoshu.gemspec` — `gem install kotoshu` succeeds without it. `Models::OnnxModel` soft-requires it at load time and exposes `ONNX_LOADED` (true/false). When false, semantic methods raise `Models::OnnxModel::OnnxUnavailable` with a caller-friendly message. `KOTOSHU_NO_ONNX=1` forces semantic off even when the gem is present. The traditional spell-checking path never touches onnxruntime.
78
+
79
+ ### Resource lifecycle — two-stage model
80
+
81
+ Resources (dictionaries, frequency lists, ONNX models) flow through a strict two-stage API in `ResourceManager`:
82
+
83
+ 1. **Setup** (`Kotoshu.setup(:en, want: %i[spelling frequency model])`, or `Kotoshu.setup(:en, aff:, dic:)` / `from:` for local sources). Slow, network-required, explicit. Writes into the cache.
84
+ 2. **Resolve** (`Kotoshu::ResourceManager.resolve(language:, want:)`). Instant, cache-only, raises `ResourceNotSetupError` on miss.
85
+
86
+ The hot path (`Kotoshu.correct?`, `.check`, `.suggest`, `.spellchecker_for`) calls `resolve` and lets the error propagate — **setup is never implicit**. This is intentional: users on metered networks or air-gapped hosts must not get a surprise download. `Kotoshu.setup?(:en, resource: :spelling|:frequency|:model)` is the predicate for "is this already in cache?".
87
+
88
+ `ResourceBundle` (the resolve result) carries `dictionary`, `frequency`, `model`, and `rules`. `SetupResult` (the setup result) reports per-resource status (`:downloaded | :local | :cached | :unavailable`).
89
+
90
+ ### Paths — XDG Base Directory
91
+
92
+ All on-disk locations are resolved through `Kotoshu::Paths`, which honors `XDG_CACHE_HOME`, `XDG_CONFIG_HOME`, `XDG_LOCAL_HOME` and the override envs `KOTOSHU_CACHE_PATH`, `KOTOSHU_CONFIG_PATH`, `KOTOSHU_DATA_PATH`. Defaults:
93
+
94
+ | Concern | Default path |
95
+ |---|---|
96
+ | Language dictionaries, frequency lists, ONNX models | `~/.cache/kotoshu/` |
97
+ | Personal dictionary, kotoshu.cfg | `~/.config/kotoshu/` |
98
+ | Audit log | `~/.local/share/kotoshu/audit.log` |
99
+
100
+ ### Resource caching
101
+
102
+ Three caches under `~/.cache/kotoshu/` (see `CACHE_ARCHITECTURE.md` for detail, README.adoc for the user-facing version):
103
+
104
+ | Cache | Path | Source | TTL |
105
+ |---|---|---|---|
106
+ | `Cache::LanguageCache` | `~/.cache/kotoshu/languages/{code}/spelling/` | `github.com/kotoshu/dictionaries` | 7 days |
107
+ | `Cache::FrequencyCache` | `~/.cache/kotoshu/frequency-lists/{code}/` | `github.com/kotoshu/frequency-list-kelly` | 7 days |
108
+ | `Cache::ModelCache` | `~/.cache/kotoshu/models/{code}/...` | `github.com/kotoshu/models-fasttext-onnx` (FastText `.vec` → ONNX converted upstream) | 30 days |
109
+
110
+ `FrequencyCache` feeds `frequency_bonus` in `Suggestions::Strategies::EditDistanceStrategy` — high-frequency words get a ranking boost. The `kotoshu cache` subcommand exposes list/status/download/info/purge/clean operations.
111
+
112
+ ### Configuration
113
+
114
+ `Configuration` (singleton via `.instance`) is built from a `SCHEMA` hash that declares each option's ENV var, default, type, and description. The `Configuration::Resolver` enforces the priority chain: **CLI flags > ENV (`KOTOSHU_*`) > programmatic > defaults**. When adding a config option, add it to `SCHEMA` (and probably `DEFAULTS`) rather than sprinkling `attr_accessor`s — that's how it picks up ENV support automatically.
115
+
116
+ `dictionary_type` selects the backend: `:unix_words | :plain_text | :custom | :hunspell | :cspell`. The dictionary is lazy-loaded through `Configuration#dictionary` (cached on the instance; call `reset_dictionary` to reload).
117
+
118
+ ### Language support
119
+
120
+ Full features (dictionary + affixes + FastText + ONNX + keyboard layout): `de, en, es, fr, pt, ru`.
121
+ Kelly frequency only: `ar, zh, el, it, no, sv` (and `ru`).
122
+ `Language::Identifier` does automatic detection (FastText LID model, 127 languages). Per-language behavior (tokenizer, normalizer) lives in `languages/{code}/language.rb` and `language/tokenizer/*`. Keyboard layouts (`keyboard/layouts/*`) feed `KeyboardProximityStrategy`.
123
+
124
+ ### Suggestion strategies
125
+
126
+ `Suggestions::Generator::DEFAULT_ALGORITHMS` = `[EditDistanceStrategy, PhoneticStrategy, KeyboardProximityStrategy, NgramStrategy]`, composed via `Strategies::CompositeStrategy`. Also available: `SymspellStrategy`, `SemanticStrategy`. Register new algorithms via `Kotoshu.register_suggestion_algorithm(:name, Klass)` (uses `BaseStrategy.register_type`).
127
+
128
+ ## Code Layout (lib/kotoshu/)
129
+
130
+ | Path | Responsibility |
131
+ |---|---|
132
+ | `kotoshu.rb` | Public facade + eager/autoload wiring |
133
+ | `spellchecker.rb`, `spellchecker/parallel_checker.rb` | Traditional check facade |
134
+ | `paths.rb` | XDG path resolution (cache, config, data, audit log, personal dict) |
135
+ | `resource_manager.rb`, `resource_bundle.rb` | Two-stage setup/resolve flow + result structs |
136
+ | `configuration.rb`, `configuration/{builder,resolver}.rb` | Config + priority resolution |
137
+ | `core/` | Domain models (`Word`, `AffixRule`, `result/*`), `IndexedDictionary`, `Trie/*`, `exceptions` |
138
+ | `dictionary/` | Backends: `base`, `hunspell`, `cspell`, `unix_words`, `plain_text`, `custom`, `unified`, `repository` |
139
+ | `readers/` | Parsers for Hunspell `.aff` / `.dic` (aff_data, aff_reader, dic_reader, condition_checker, lookup_builder) |
140
+ | `suggestions/` | `generator`, `context`, `suggestion{,_set}`, `pipeline`, `strategies/*` |
141
+ | `algorithms/` | Lower-level Hunspell-style suggestion primitives (ported from Spylls): `ngram_suggest`, `phonet_suggest`, `suggest`, `lookup`, `permutations`, `capitalization` |
142
+ | `analyzers/` | `semantic_analyzer` — the embedding-based checker |
143
+ | `models/` | `embedding_model` (abstract), `fasttext_model`, `onnx_model`, `word_embedding`, `nearest_neighbor`, `semantic_error`, `context`, `suggestion` |
144
+ | `embeddings/` | ONNX runtime glue: `onnx_runtime_model`, `vocabulary`, `similarity_engine`, `similarity_search`, `search`, `embedding_pipeline`, `protocols{,_registry}`, `lru_cache` |
145
+ | `cache/` | `base_cache`, `language_cache`, `model_cache`, `frequency_cache`, plus `lookup_cache` / `suggestion_cache` runtime caches |
146
+ | `language/`, `languages/` | Detection (`identifier`, `detector`), registry, per-language modules, tokenizers, normalizers |
147
+ | `documents/` | Document abstraction: `plain_text_document`, `markdown_document`, `asciidoc_document`, `location` |
148
+ | `cli/` | CLI helpers (interactive reviewer, batch reporter, navigation, display) |
149
+ | `commands/` | Thor subcommands: `check_command`, `cache_command`, `model_command` |
150
+ | `grammar/` | Rule engine + pattern matchers (`rule`, `rule_engine`, `rule_loader`, `pattern_matchers/*`) |
151
+ | `keyboard/` | Layout registry + per-layout files (qwerty, qwertz, azerty, jcuken, dvorak) |
152
+ | `components/`, `plugins/`, `data_structures/`, `results/`, `data/` | Tokenizer/POS/synthesizer components, plugin registry, bloom filter, result base, common-words loader |
153
+
154
+ The exe uses `Kotoshu::Cli::Cli` (in `cli.rb`), which registers `dict` → `DictCommand` and `cache` → `CacheCommand` as subcommands. A richer `Kotoshu::CheckCommand` exists in `commands/check_command.rb` (with `--interactive`, `--format sarif/json`, `--model`, `--language auto`) — check which one is actually wired before assuming a CLI flag exists.
155
+
156
+ ## Specs
157
+
158
+ Spec layout mirrors lib: `spec/kotoshu/...`, plus `spec/integration/`, `spec/integrational/`, `spec/performance/`, `spec/benchmark/`, `spec/properties/`, `spec/unit/`, `spec/hunspell_tests/` (Splylls-ported fixtures), `spec/fixtures/`, `spec/support/`.
159
+
160
+ Global rules that apply here (see `~/.claude/CLAUDE.md`): **no `double()` in specs** — use real instances or `Struct.new`; **no hand-rolled serialization** (`to_h`/`from_h` on models).
161
+
162
+ ## Reference Implementations (read-only, on disk)
163
+
164
+ When implementing features, study these alongside Kotoshu:
165
+
166
+ - `/Users/mulgogi/src/external/hunspell/` — morphological rules, affix processing, suggestions (C++ reference).
167
+ - `/Users/mulgogi/src/external/cspell/` — trie/DAFSA dictionaries, code-aware checking (TypeScript reference).
168
+ - `/Users/mulgogi/src/external/languagetool/` — rule-based grammar, multi-interface (library + HTTP), caching patterns (Java reference).
169
+ - Spylls (Python Hunspell port) — the algorithms in `algorithms/` and the fixtures in `spec/hunspell_tests/` derive from here.
170
+
171
+ ## Other Notes
172
+
173
+ - License is **BSD-2-Clause** (not MIT — the README's "License" section is wrong).
174
+ - RBS signatures live in `sig/kotoshu.rbs` (the `sig/kotoshu/` subdirectory is empty). Update signatures when changing public APIs.
175
+ - `scripts/` contains one-off utilities (FastText→ONNX conversion in Python, Kelly frequency parsing, diagnostics). `examples/` has numbered walkthrough scripts (`01_*.rb` … `07_*.rb`).
176
+ - Design history and superseded planning docs live in `docs/` (`architecture.md`, `cache-architecture.md`, `performance.md`, `plugins.md`, `getting-started.md`, plus integrated planning docs like `KOTOSHU_SOLIDIFICATION_PLAN.md`, `ARCHITECTURE_IMPROVEMENTS.md`, `TDD_ITERATION_STRATEGY.md`). Treat them as historical context, verify against current code before relying on them. `TODO.impl/` is the current source of truth for execution plans.
@@ -0,0 +1,132 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ We as members, contributors, and leaders pledge to make participation in our
6
+ community a harassment-free experience for everyone, regardless of age, body
7
+ size, visible or invisible disability, ethnicity, sex characteristics, gender
8
+ identity and expression, level of experience, education, socio-economic status,
9
+ nationality, personal appearance, race, caste, color, religion, or sexual
10
+ identity and orientation.
11
+
12
+ We pledge to act and interact in ways that contribute to an open, welcoming,
13
+ diverse, inclusive, and healthy community.
14
+
15
+ ## Our Standards
16
+
17
+ Examples of behavior that contributes to a positive environment for our
18
+ community include:
19
+
20
+ * Demonstrating empathy and kindness toward other people
21
+ * Being respectful of differing opinions, viewpoints, and experiences
22
+ * Giving and gracefully accepting constructive feedback
23
+ * Accepting responsibility and apologizing to those affected by our mistakes,
24
+ and learning from the experience
25
+ * Focusing on what is best not just for us as individuals, but for the overall
26
+ community
27
+
28
+ Examples of unacceptable behavior include:
29
+
30
+ * The use of sexualized language or imagery, and sexual attention or advances of
31
+ any kind
32
+ * Trolling, insulting or derogatory comments, and personal or political attacks
33
+ * Public or private harassment
34
+ * Publishing others' private information, such as a physical or email address,
35
+ without their explicit permission
36
+ * Other conduct which could reasonably be considered inappropriate in a
37
+ professional setting
38
+
39
+ ## Enforcement Responsibilities
40
+
41
+ Community leaders are responsible for clarifying and enforcing our standards of
42
+ acceptable behavior and will take appropriate and fair corrective action in
43
+ response to any behavior that they deem inappropriate, threatening, offensive,
44
+ or harmful.
45
+
46
+ Community leaders have the right and responsibility to remove, edit, or reject
47
+ comments, commits, code, wiki edits, issues, and other contributions that are
48
+ not aligned to this Code of Conduct, and will communicate reasons for moderation
49
+ decisions when appropriate.
50
+
51
+ ## Scope
52
+
53
+ This Code of Conduct applies within all community spaces, and also applies when
54
+ an individual is officially representing the community in public spaces.
55
+ Examples of representing our community include using an official email address,
56
+ posting via an official social media account, or acting as an appointed
57
+ representative at an online or offline event.
58
+
59
+ ## Enforcement
60
+
61
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
62
+ reported to the community leaders responsible for enforcement at
63
+ [INSERT CONTACT METHOD].
64
+ All complaints will be reviewed and investigated promptly and fairly.
65
+
66
+ All community leaders are obligated to respect the privacy and security of the
67
+ reporter of any incident.
68
+
69
+ ## Enforcement Guidelines
70
+
71
+ Community leaders will follow these Community Impact Guidelines in determining
72
+ the consequences for any action they deem in violation of this Code of Conduct:
73
+
74
+ ### 1. Correction
75
+
76
+ **Community Impact**: Use of inappropriate language or other behavior deemed
77
+ unprofessional or unwelcome in the community.
78
+
79
+ **Consequence**: A private, written warning from community leaders, providing
80
+ clarity around the nature of the violation and an explanation of why the
81
+ behavior was inappropriate. A public apology may be requested.
82
+
83
+ ### 2. Warning
84
+
85
+ **Community Impact**: A violation through a single incident or series of
86
+ actions.
87
+
88
+ **Consequence**: A warning with consequences for continued behavior. No
89
+ interaction with the people involved, including unsolicited interaction with
90
+ those enforcing the Code of Conduct, for a specified period of time. This
91
+ includes avoiding interactions in community spaces as well as external channels
92
+ like social media. Violating these terms may lead to a temporary or permanent
93
+ ban.
94
+
95
+ ### 3. Temporary Ban
96
+
97
+ **Community Impact**: A serious violation of community standards, including
98
+ sustained inappropriate behavior.
99
+
100
+ **Consequence**: A temporary ban from any sort of interaction or public
101
+ communication with the community for a specified period of time. No public or
102
+ private interaction with the people involved, including unsolicited interaction
103
+ with those enforcing the Code of Conduct, is allowed during this period.
104
+ Violating these terms may lead to a permanent ban.
105
+
106
+ ### 4. Permanent Ban
107
+
108
+ **Community Impact**: Demonstrating a pattern of violation of community
109
+ standards, including sustained inappropriate behavior, harassment of an
110
+ individual, or aggression toward or disparagement of classes of individuals.
111
+
112
+ **Consequence**: A permanent ban from any sort of public interaction within the
113
+ community.
114
+
115
+ ## Attribution
116
+
117
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage],
118
+ version 2.1, available at
119
+ [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
120
+
121
+ Community Impact Guidelines were inspired by
122
+ [Mozilla's code of conduct enforcement ladder][Mozilla CoC].
123
+
124
+ For answers to common questions about this code of conduct, see the FAQ at
125
+ [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
126
+ [https://www.contributor-covenant.org/translations][translations].
127
+
128
+ [homepage]: https://www.contributor-covenant.org
129
+ [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
130
+ [Mozilla CoC]: https://github.com/mozilla/diversity
131
+ [FAQ]: https://www.contributor-covenant.org/faq
132
+ [translations]: https://www.contributor-covenant.org/translations
data/Cargo.lock ADDED
@@ -0,0 +1,359 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.5"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "bindgen"
16
+ version = "0.72.1"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
19
+ dependencies = [
20
+ "bitflags",
21
+ "cexpr",
22
+ "clang-sys",
23
+ "itertools",
24
+ "proc-macro2",
25
+ "quote",
26
+ "regex",
27
+ "rustc-hash",
28
+ "shlex",
29
+ "syn 2.0.119",
30
+ ]
31
+
32
+ [[package]]
33
+ name = "bitflags"
34
+ version = "2.13.1"
35
+ source = "registry+https://github.com/rust-lang/crates.io-index"
36
+ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
37
+
38
+ [[package]]
39
+ name = "cexpr"
40
+ version = "0.6.0"
41
+ source = "registry+https://github.com/rust-lang/crates.io-index"
42
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
43
+ dependencies = [
44
+ "nom",
45
+ ]
46
+
47
+ [[package]]
48
+ name = "cfg-if"
49
+ version = "1.0.4"
50
+ source = "registry+https://github.com/rust-lang/crates.io-index"
51
+ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
52
+
53
+ [[package]]
54
+ name = "clang-sys"
55
+ version = "1.9.1"
56
+ source = "registry+https://github.com/rust-lang/crates.io-index"
57
+ checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
58
+ dependencies = [
59
+ "glob",
60
+ "libc",
61
+ "libloading",
62
+ ]
63
+
64
+ [[package]]
65
+ name = "either"
66
+ version = "1.18.0"
67
+ source = "registry+https://github.com/rust-lang/crates.io-index"
68
+ checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
69
+
70
+ [[package]]
71
+ name = "glob"
72
+ version = "0.3.4"
73
+ source = "registry+https://github.com/rust-lang/crates.io-index"
74
+ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
75
+
76
+ [[package]]
77
+ name = "itertools"
78
+ version = "0.13.0"
79
+ source = "registry+https://github.com/rust-lang/crates.io-index"
80
+ checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
81
+ dependencies = [
82
+ "either",
83
+ ]
84
+
85
+ [[package]]
86
+ name = "itoa"
87
+ version = "1.0.18"
88
+ source = "registry+https://github.com/rust-lang/crates.io-index"
89
+ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
90
+
91
+ [[package]]
92
+ name = "kotoshu"
93
+ version = "0.1.0"
94
+ source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#e574c445df5f6eee8e978f1f4d26dc48cfb1947b"
95
+ dependencies = [
96
+ "magnus",
97
+ "serde",
98
+ "serde_json",
99
+ ]
100
+
101
+ [[package]]
102
+ name = "kotoshu_native"
103
+ version = "0.0.0"
104
+ dependencies = [
105
+ "kotoshu",
106
+ "magnus",
107
+ "rb-sys",
108
+ ]
109
+
110
+ [[package]]
111
+ name = "lazy_static"
112
+ version = "1.5.0"
113
+ source = "registry+https://github.com/rust-lang/crates.io-index"
114
+ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
115
+
116
+ [[package]]
117
+ name = "libc"
118
+ version = "0.2.189"
119
+ source = "registry+https://github.com/rust-lang/crates.io-index"
120
+ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
121
+
122
+ [[package]]
123
+ name = "libloading"
124
+ version = "0.8.9"
125
+ source = "registry+https://github.com/rust-lang/crates.io-index"
126
+ checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
127
+ dependencies = [
128
+ "cfg-if",
129
+ "windows-link",
130
+ ]
131
+
132
+ [[package]]
133
+ name = "magnus"
134
+ version = "0.8.2"
135
+ source = "registry+https://github.com/rust-lang/crates.io-index"
136
+ checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
137
+ dependencies = [
138
+ "magnus-macros",
139
+ "rb-sys",
140
+ "rb-sys-env",
141
+ "seq-macro",
142
+ ]
143
+
144
+ [[package]]
145
+ name = "magnus-macros"
146
+ version = "0.8.0"
147
+ source = "registry+https://github.com/rust-lang/crates.io-index"
148
+ checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
149
+ dependencies = [
150
+ "proc-macro2",
151
+ "quote",
152
+ "syn 2.0.119",
153
+ ]
154
+
155
+ [[package]]
156
+ name = "memchr"
157
+ version = "2.8.3"
158
+ source = "registry+https://github.com/rust-lang/crates.io-index"
159
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
160
+
161
+ [[package]]
162
+ name = "minimal-lexical"
163
+ version = "0.2.1"
164
+ source = "registry+https://github.com/rust-lang/crates.io-index"
165
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
166
+
167
+ [[package]]
168
+ name = "nom"
169
+ version = "7.1.3"
170
+ source = "registry+https://github.com/rust-lang/crates.io-index"
171
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
172
+ dependencies = [
173
+ "memchr",
174
+ "minimal-lexical",
175
+ ]
176
+
177
+ [[package]]
178
+ name = "proc-macro2"
179
+ version = "1.0.107"
180
+ source = "registry+https://github.com/rust-lang/crates.io-index"
181
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
182
+ dependencies = [
183
+ "unicode-ident",
184
+ ]
185
+
186
+ [[package]]
187
+ name = "quote"
188
+ version = "1.0.47"
189
+ source = "registry+https://github.com/rust-lang/crates.io-index"
190
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
191
+ dependencies = [
192
+ "proc-macro2",
193
+ ]
194
+
195
+ [[package]]
196
+ name = "rb-sys"
197
+ version = "0.9.130"
198
+ source = "registry+https://github.com/rust-lang/crates.io-index"
199
+ checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
200
+ dependencies = [
201
+ "rb-sys-build",
202
+ ]
203
+
204
+ [[package]]
205
+ name = "rb-sys-build"
206
+ version = "0.9.130"
207
+ source = "registry+https://github.com/rust-lang/crates.io-index"
208
+ checksum = "05be6f9c86fe5482808162826f6c047d093b3670582296c770de1d94f8066694"
209
+ dependencies = [
210
+ "bindgen",
211
+ "lazy_static",
212
+ "proc-macro2",
213
+ "quote",
214
+ "regex",
215
+ "shell-words",
216
+ "syn 2.0.119",
217
+ ]
218
+
219
+ [[package]]
220
+ name = "rb-sys-env"
221
+ version = "0.2.3"
222
+ source = "registry+https://github.com/rust-lang/crates.io-index"
223
+ checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
224
+
225
+ [[package]]
226
+ name = "regex"
227
+ version = "1.13.1"
228
+ source = "registry+https://github.com/rust-lang/crates.io-index"
229
+ checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
230
+ dependencies = [
231
+ "aho-corasick",
232
+ "memchr",
233
+ "regex-automata",
234
+ "regex-syntax",
235
+ ]
236
+
237
+ [[package]]
238
+ name = "regex-automata"
239
+ version = "0.4.18"
240
+ source = "registry+https://github.com/rust-lang/crates.io-index"
241
+ checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
242
+ dependencies = [
243
+ "aho-corasick",
244
+ "memchr",
245
+ "regex-syntax",
246
+ ]
247
+
248
+ [[package]]
249
+ name = "regex-syntax"
250
+ version = "0.8.11"
251
+ source = "registry+https://github.com/rust-lang/crates.io-index"
252
+ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
253
+
254
+ [[package]]
255
+ name = "rustc-hash"
256
+ version = "2.1.3"
257
+ source = "registry+https://github.com/rust-lang/crates.io-index"
258
+ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
259
+
260
+ [[package]]
261
+ name = "seq-macro"
262
+ version = "0.3.6"
263
+ source = "registry+https://github.com/rust-lang/crates.io-index"
264
+ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
265
+
266
+ [[package]]
267
+ name = "serde"
268
+ version = "1.0.229"
269
+ source = "registry+https://github.com/rust-lang/crates.io-index"
270
+ checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
271
+ dependencies = [
272
+ "serde_core",
273
+ "serde_derive",
274
+ ]
275
+
276
+ [[package]]
277
+ name = "serde_core"
278
+ version = "1.0.229"
279
+ source = "registry+https://github.com/rust-lang/crates.io-index"
280
+ checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
281
+ dependencies = [
282
+ "serde_derive",
283
+ ]
284
+
285
+ [[package]]
286
+ name = "serde_derive"
287
+ version = "1.0.229"
288
+ source = "registry+https://github.com/rust-lang/crates.io-index"
289
+ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
290
+ dependencies = [
291
+ "proc-macro2",
292
+ "quote",
293
+ "syn 3.0.5",
294
+ ]
295
+
296
+ [[package]]
297
+ name = "serde_json"
298
+ version = "1.0.151"
299
+ source = "registry+https://github.com/rust-lang/crates.io-index"
300
+ checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
301
+ dependencies = [
302
+ "itoa",
303
+ "memchr",
304
+ "serde",
305
+ "serde_core",
306
+ "zmij",
307
+ ]
308
+
309
+ [[package]]
310
+ name = "shell-words"
311
+ version = "1.1.1"
312
+ source = "registry+https://github.com/rust-lang/crates.io-index"
313
+ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
314
+
315
+ [[package]]
316
+ name = "shlex"
317
+ version = "1.3.0"
318
+ source = "registry+https://github.com/rust-lang/crates.io-index"
319
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
320
+
321
+ [[package]]
322
+ name = "syn"
323
+ version = "2.0.119"
324
+ source = "registry+https://github.com/rust-lang/crates.io-index"
325
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
326
+ dependencies = [
327
+ "proc-macro2",
328
+ "quote",
329
+ "unicode-ident",
330
+ ]
331
+
332
+ [[package]]
333
+ name = "syn"
334
+ version = "3.0.5"
335
+ source = "registry+https://github.com/rust-lang/crates.io-index"
336
+ checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
337
+ dependencies = [
338
+ "proc-macro2",
339
+ "quote",
340
+ "unicode-ident",
341
+ ]
342
+
343
+ [[package]]
344
+ name = "unicode-ident"
345
+ version = "1.0.24"
346
+ source = "registry+https://github.com/rust-lang/crates.io-index"
347
+ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
348
+
349
+ [[package]]
350
+ name = "windows-link"
351
+ version = "0.2.1"
352
+ source = "registry+https://github.com/rust-lang/crates.io-index"
353
+ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
354
+
355
+ [[package]]
356
+ name = "zmij"
357
+ version = "1.0.23"
358
+ source = "registry+https://github.com/rust-lang/crates.io-index"
359
+ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
data/Cargo.toml ADDED
@@ -0,0 +1,15 @@
1
+ # Kotoshu gem Rust workspace (plan 66 / P4b) — mirrors parsanol-ruby's
2
+ # root Cargo.toml: rb_sys's compile task runs `cargo metadata` from the gem
3
+ # root, so the extension crate must be reachable through a root manifest.
4
+ [workspace]
5
+ resolver = "2"
6
+ members = ["ext/kotoshu_native"]
7
+
8
+ # Release profile follows the kotoshu-rs workspace policy (plan 66): fat
9
+ # LTO and single codegen unit so the shipped accelerator is as fast as the
10
+ # core's own optimized builds. Local development builds use the dev profile
11
+ # via RB_SYS_CARGO_PROFILE (see ext/kotoshu_native/extconf.rb).
12
+ [profile.release]
13
+ opt-level = 3
14
+ lto = "fat"
15
+ codegen-units = 1