kotoshu 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. checksums.yaml +4 -4
  2. data/.pre-commit-hooks.yaml +18 -0
  3. data/.yardopts +8 -0
  4. data/CHANGELOG.md +149 -0
  5. data/Cargo.lock +291 -0
  6. data/Cargo.toml +15 -0
  7. data/README.adoc +206 -0
  8. data/Rakefile +103 -0
  9. data/TODO.impl/00-vision.md +212 -0
  10. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  11. data/TODO.impl/23-fix-url-defaults.md +57 -0
  12. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  13. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  14. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  15. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  16. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  17. data/TODO.impl/29-status-command.md +84 -0
  18. data/TODO.impl/30-language-auto-detection.md +74 -0
  19. data/TODO.impl/31-download-progress.md +58 -0
  20. data/TODO.impl/32-sarif-output-verification.md +72 -0
  21. data/TODO.impl/36-cleanup-regressions-0.3.1.md +17 -0
  22. data/TODO.impl/37-hunspell-correctness-tier2.md +21 -0
  23. data/TODO.impl/38-onnx-semantic-gating.md +63 -0
  24. data/TODO.impl/39-tier3-and-beyond.md +36 -0
  25. data/TODO.impl/65-universal-kotoshu.md +251 -0
  26. data/TODO.impl/66-kotoshu-core.md +169 -0
  27. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  28. data/TODO.impl/68-sota-adoption.md +152 -0
  29. data/TODO.impl/69-sota-typo-eval.md +74 -0
  30. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  31. data/TODO.impl/71-sota-nested-model.md +63 -0
  32. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  33. data/TODO.impl/73-rl-reranker.md +42 -0
  34. data/TODO.impl/74-distilled-reranker.md +36 -0
  35. data/TODO.impl/75-llm-evaluator.md +35 -0
  36. data/TODO.impl/76-optional-llm-backend.md +37 -0
  37. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  38. data/TODO.impl/78-in-browser-playground.md +69 -0
  39. data/TODO.impl/79-news-and-releases.md +60 -0
  40. data/TODO.impl/80-ecosystem-docs.md +56 -0
  41. data/TODO.impl/81-editor-reach.md +68 -0
  42. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  43. data/TODO.impl/83-models-batch-2.md +56 -0
  44. data/TODO.impl/84-language-modules-wave2.md +75 -0
  45. data/TODO.impl/85-wasm-model-api.md +48 -0
  46. data/TODO.impl/86-site-truth-wave2.md +45 -0
  47. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  48. data/TODO.impl/88-cli-directory-mode.md +54 -0
  49. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  50. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  51. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  52. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  53. data/TODO.impl/93-python-wheels.md +32 -0
  54. data/TODO.impl/94-site-integrations-docs.md +35 -0
  55. data/TODO.impl/95-editor-action-polish.md +27 -0
  56. data/TODO.impl/96-server-semantic-models.md +50 -0
  57. data/TODO.impl/97-yard-execution.md +73 -0
  58. data/conformance/vectors.jsonl +2630 -0
  59. data/data/en/grammar/rules.yaml +109 -0
  60. data/docs/cache-architecture.md +502 -0
  61. data/ext/kotoshu_native/Cargo.toml +37 -0
  62. data/ext/kotoshu_native/extconf.rb +15 -0
  63. data/ext/kotoshu_native/src/lib.rs +15 -0
  64. data/lib/kotoshu/algorithms/edit_distance.rb +57 -7
  65. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  66. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  67. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  68. data/lib/kotoshu/algorithms/lookup.rb +95 -236
  69. data/lib/kotoshu/algorithms/suggest.rb +231 -50
  70. data/lib/kotoshu/analyzers/semantic_analyzer.rb +1 -1
  71. data/lib/kotoshu/baseline/application.rb +18 -0
  72. data/lib/kotoshu/baseline/entry.rb +33 -0
  73. data/lib/kotoshu/baseline/store.rb +152 -0
  74. data/lib/kotoshu/baseline.rb +20 -0
  75. data/lib/kotoshu/cache/base_cache.rb +46 -3
  76. data/lib/kotoshu/cache/frequency_cache.rb +2 -2
  77. data/lib/kotoshu/cache/language_cache.rb +33 -11
  78. data/lib/kotoshu/cache/model_cache.rb +449 -7
  79. data/lib/kotoshu/cache/model_registry.rb +109 -0
  80. data/lib/kotoshu/cache.rb +1 -0
  81. data/lib/kotoshu/cli/baseline_command.rb +81 -0
  82. data/lib/kotoshu/cli/cache_command.rb +44 -38
  83. data/lib/kotoshu/cli/completions_command.rb +1 -1
  84. data/lib/kotoshu/cli/directory_check.rb +141 -0
  85. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  86. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  87. data/lib/kotoshu/cli.rb +336 -61
  88. data/lib/kotoshu/configuration.rb +69 -9
  89. data/lib/kotoshu/conformance_exporter.rb +256 -0
  90. data/lib/kotoshu/conformance_runner.rb +176 -0
  91. data/lib/kotoshu/core/indexed_dictionary.rb +7 -2
  92. data/lib/kotoshu/core/models/result/document_result.rb +13 -3
  93. data/lib/kotoshu/core/models/result/word_result.rb +19 -1
  94. data/lib/kotoshu/data/common_words_loader.rb +0 -1
  95. data/lib/kotoshu/debug_mode.rb +2 -0
  96. data/lib/kotoshu/dictionary/base.rb +14 -0
  97. data/lib/kotoshu/dictionary/cspell.rb +3 -8
  98. data/lib/kotoshu/dictionary/plain_text.rb +17 -0
  99. data/lib/kotoshu/documents/document.rb +12 -2
  100. data/lib/kotoshu/documents/plain_text_document.rb +5 -2
  101. data/lib/kotoshu/documents/source_position.rb +8 -0
  102. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  103. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  104. data/lib/kotoshu/documents/suppressions.rb +44 -0
  105. data/lib/kotoshu/documents.rb +1 -0
  106. data/lib/kotoshu/embeddings/embedding_pipeline.rb +3 -5
  107. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +11 -1
  108. data/lib/kotoshu/embeddings/vocabulary.rb +25 -1
  109. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +2 -2
  110. data/lib/kotoshu/jekyll.rb +98 -0
  111. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  112. data/lib/kotoshu/keyboard/layouts/latin.rb +220 -0
  113. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  114. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  115. data/lib/kotoshu/keyboard/registry.rb +20 -0
  116. data/lib/kotoshu/keyboard.rb +8 -0
  117. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  118. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  119. data/lib/kotoshu/language/tokenizer/base.rb +17 -0
  120. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  121. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  122. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +12 -0
  123. data/lib/kotoshu/language.rb +4 -0
  124. data/lib/kotoshu/languages/ca/language.rb +22 -0
  125. data/lib/kotoshu/languages/cs/language.rb +22 -0
  126. data/lib/kotoshu/languages/da/language.rb +22 -0
  127. data/lib/kotoshu/languages/el/language.rb +50 -0
  128. data/lib/kotoshu/languages/hu/language.rb +22 -0
  129. data/lib/kotoshu/languages/it/language.rb +22 -0
  130. data/lib/kotoshu/languages/latin_base.rb +53 -0
  131. data/lib/kotoshu/languages/nl/language.rb +22 -0
  132. data/lib/kotoshu/languages/pl/language.rb +22 -0
  133. data/lib/kotoshu/languages/ro/language.rb +22 -0
  134. data/lib/kotoshu/languages/sv/language.rb +22 -0
  135. data/lib/kotoshu/languages/tr/language.rb +36 -0
  136. data/lib/kotoshu/languages/uk/language.rb +38 -0
  137. data/lib/kotoshu/languages/vi/language.rb +22 -0
  138. data/lib/kotoshu/languages.rb +18 -0
  139. data/lib/kotoshu/metrics_collector.rb +18 -2
  140. data/lib/kotoshu/metrics_module.rb +11 -0
  141. data/lib/kotoshu/models/nearest_neighbor.rb +11 -3
  142. data/lib/kotoshu/models/onnx_model.rb +22 -23
  143. data/lib/kotoshu/native.rb +83 -0
  144. data/lib/kotoshu/native_backend.rb +131 -0
  145. data/lib/kotoshu/paths.rb +26 -0
  146. data/lib/kotoshu/readers/aff_data.rb +94 -19
  147. data/lib/kotoshu/readers/aff_reader.rb +44 -35
  148. data/lib/kotoshu/readers/lookup_builder.rb +2 -16
  149. data/lib/kotoshu/resource_bundle.rb +32 -7
  150. data/lib/kotoshu/resource_manager.rb +313 -23
  151. data/lib/kotoshu/rspec.rb +163 -0
  152. data/lib/kotoshu/source_registry.rb +26 -4
  153. data/lib/kotoshu/spellchecker.rb +88 -22
  154. data/lib/kotoshu/suggestions/frequency_provider.rb +59 -0
  155. data/lib/kotoshu/suggestions/generator.rb +4 -5
  156. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  157. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +10 -0
  158. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +62 -3
  159. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +70 -199
  160. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +11 -2
  161. data/lib/kotoshu/suggestions/suggestion_set.rb +9 -2
  162. data/lib/kotoshu/suggestions.rb +2 -0
  163. data/lib/kotoshu/tasks/check_task.rb +98 -0
  164. data/lib/kotoshu/tasks.rb +11 -0
  165. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  166. data/lib/kotoshu/validators.rb +9 -0
  167. data/lib/kotoshu/version.rb +1 -1
  168. data/lib/kotoshu.rb +42 -12
  169. data/rakelib/compile.rake +10 -0
  170. data/scripts/extract_eval_grids.rb +34 -0
  171. data/sig/kotoshu.rbs +331 -1
  172. metadata +123 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 503b4db65b4971cdc8854123771d9c1855af0de6f408f625d63219d5470c848c
4
- data.tar.gz: caa57938fc7a76f3c0f7d026346db8e7f247db34b1a9b51d79ab2e9962668f57
3
+ metadata.gz: f90290a36de788d46588e6883b0619f9fb1dff8e841f629fd021c07e156256ae
4
+ data.tar.gz: 113cbf58fb1b9c297cfb8551a9076875f4882be9e7b17bbebe330ecc53aac94a
5
5
  SHA512:
6
- metadata.gz: 9bb38cd338777586b8bf3c4700d36c5bd083b81f002fc9df0ee3ab667bde36a2a31b0dd8ebd9096543b57de8e151c18b9265df900306c7c71ae06cdc8e33825f
7
- data.tar.gz: 6a11f2cbc34632d9533af0665308943063f06d952816e277fd1e995adfbbe84a56915693a6b9a9a53a4c64589ff8d09399c66ace9c7f7c657f865e4b0ef7adc4
6
+ metadata.gz: 1520bf0ce2667b31872c137b55099834a93954f821c93903b193306d537703173ad37e84446dfd41e082e25db50d4a54c694d1623b43875fd5b61294b4cf67f4
7
+ data.tar.gz: f22ca5507daa129090a2d292d4e526200f197f32520b864c67e3ffbab82862a431d155303613e670aab97e48d213bad32ad0ba4b4cdf1178a8de1dff940a1cbd
@@ -0,0 +1,18 @@
1
+ # pre-commit hook definitions (https://pre-commit.com).
2
+ #
3
+ # Honest requirements: this hook is `language: system` — it runs the
4
+ # `kotoshu` executable from PATH, so the machine needs a Ruby
5
+ # environment with the gem installed:
6
+ #
7
+ # gem install kotoshu
8
+ # kotoshu setup en # or the languages you write in
9
+ #
10
+ # pre-commit passes the matched filenames to the entry; `kotoshu check`
11
+ # exits 1 when a file has unsuppressed misspellings. The WebAssembly
12
+ # client (in development) is the future zero-Ruby path.
13
+ - id: kotoshu
14
+ name: kotoshu semantic spell check
15
+ description: Semantic spell checking for prose and documentation
16
+ entry: kotoshu check
17
+ language: system
18
+ files: \.(txt|text|md|markdown|mdown|mkd|adoc|asciidoc|ad|rst|rest|creole|rdoc|org|pod|textile|wiki|mail|article|log|notes)$
data/.yardopts ADDED
@@ -0,0 +1,8 @@
1
+ --title "Kotoshu API Reference"
2
+ --markup markdown
3
+ --readme README.adoc
4
+ --exclude ^spec/
5
+ --exclude ^tasks/
6
+ --exclude ^ext/
7
+ --exclude ^conformance/
8
+ --hide-void-return
data/CHANGELOG.md CHANGED
@@ -5,6 +5,155 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [Unreleased]
9
+
10
+ ### Fixed
11
+ - Remote `kotoshu setup` for staged languages: the dictionaries repo ships only
12
+ `en` under the `{lang}/spelling/` layout while staged languages sit flat at
13
+ `{lang}/index.*`; downloads now try the sublayout first and fall back to the
14
+ flat layout (verified end-to-end for `it`).
15
+ - `require "kotoshu/tasks"` / `require "kotoshu/jekyll"` standalone now load
16
+ without needing `require "kotoshu"` first.
17
+ - Baseline paths compare canonically: entries recorded as `docs/a.md` match a
18
+ directory-walk check over `./docs/a.md`, and recorded entries are stored
19
+ canonical.
20
+ ### Fixed
21
+ - **Unicode word detection** (plan 91 Track A) - `Spellchecker` word
22
+ extraction accepted only ASCII letters, so Greek and Ukrainian users
23
+ could not check any text through the CLI. Extraction now follows the
24
+ configured language's tokenizer: Greek words for `el`, Cyrillic for
25
+ `uk` (in-word apostrophe included, for names like Мар'яна), and
26
+ Latin-script letters including capitals such as Å and Ä for Latin
27
+ languages. Wrong-script words stay uncheckable; languages without a
28
+ script tokenizer (en, de, ...) keep the historical ASCII behavior;
29
+ the frozen conformance vectors are unchanged.
30
+ - **Swedish aff parsing** (plan 91 Track B) - `Readers::AffReader`
31
+ raised `RegexpError: unmatched close parenthesis` on the Swedish
32
+ dictionary (dictionaries pin 1829a3e). `COMPOUNDRULE )k` uses `)`
33
+ itself as a flag character, but the rule compiler escaped it only in
34
+ some positions. Every flag character is now escaped at compile time,
35
+ so the dictionary loads and its compound rules match.
36
+ ### Added
37
+ - **Framework integrations** (plan 89) - four opt-in integration
38
+ layers, all inside the gem with NO new runtime dependencies
39
+ (soft-dependency pattern, like onnxruntime/suika):
40
+ - `Kotoshu::Validators::SpellingValidator`, an ActiveModel
41
+ EachValidator (`validates :body, spelling: true` after the
42
+ documented one-line alias, or `validates_with` directly): one
43
+ validation error per misspelling with the top suggestion in the
44
+ message; `language:` and `personal_words:` options. Loads the
45
+ real ActiveModel standalone - Rails is not required.
46
+ - RSpec matchers (`require "kotoshu/rspec"` +
47
+ `include Kotoshu::Rspec::Matchers`):
48
+ `expect_words("helo").to all_be_spelled_correctly`, and
49
+ `expect(text) / expect_document(path).to be_spelled_correctly`
50
+ for words or whole documents, with failure messages that list
51
+ each misspelling and its suggestions.
52
+ - Rake task (`require "kotoshu/tasks"`): `rake kotoshu:check`
53
+ over the repository text files using the plan-88 file selection
54
+ (known extensions, .gitignore/.ignore, default skips);
55
+ `Kotoshu::Tasks::CheckTask` exposes files/language/fail_on_error
56
+ /baseline configuration.
57
+ - Jekyll generator (`Kotoshu::Jekyll::Generator`, `safe`):
58
+ checks posts and drafts and fails the build on new spelling
59
+ errors; a `.kotoshu-baseline.json` in the site source is
60
+ respected so baselined errors do not block builds.
61
+ - **CLI directory mode** (plan 88) - `kotoshu check DIR [DIR ...]`
62
+ walks trees and checks every file with a known text extension
63
+ (md markdown asciidoc adoc txt rst mdx) unless `--include` /
64
+ `--exclude` globs say otherwise. `.gitignore` and `.ignore` files
65
+ are honored through a standard glob subset implemented in Ruby
66
+ (`*`, `?`, `**` as a whole segment, `!` negation with last-match
67
+ wins, trailing `/` for directory-only patterns, anchoring, nested
68
+ scoping; files inside ignored directories cannot be re-included).
69
+ Hidden files and `.git` / `node_modules` / `vendor` / `target` are
70
+ skipped by default. Text output prints one section per file plus a
71
+ summary; JSON and SARIF emit one combined document with per-file
72
+ entries/runs; `--baseline` applies per file as shipped; exit codes
73
+ are unchanged; interactive mode stays file-only with an explicit
74
+ notice for directory targets. Single-file and stdin behavior is
75
+ byte-identical to before.
76
+ - **Wave-1 language modules and keyboard layouts** (plan 84) -
77
+ language modules for the thirteen model-registry newcomers (ca cs
78
+ da el hu it nl pl ro sv tr uk vi): thin compositions over the shared
79
+ Latin tokenizer/normalizer plus registry entries, with real care
80
+ where the script needs it — Turkish dotless-i case folding
81
+ (`İZMİR` -> `izmir`, `ISTANBUL` -> `ıstanbul`), Greek-aware
82
+ normalization (final sigma `ΕΛΛΑΣ` -> `ελλας`, opt-in accent
83
+ stripping) and Greek/Cyrillic script tokenizers that keep
84
+ non-Latin words. Keyboard layouts grow from 5 to 19: Turkish-Q,
85
+ Ukrainian-JCUKEN and Greek-Phonetic national grids mirrored from
86
+ the models repo eval harness (a drift spec asserts the gem grids
87
+ cover the eval grids; `scripts/extract_eval_grids.rb` refreshes the
88
+ committed snapshot), plus one parameterized Latin family
89
+ (`Keyboard::Layouts::Latin` and members) over the qwerty/qwertz
90
+ base grids with real Nordic å/æ/ø and å/ä/ö keys. The pre-existing
91
+ five layouts and ten language modules are unchanged.
92
+ - **Inline ignore directives** (plan 82) - `kotoshu:disable-line`,
93
+ `kotoshu:disable-next-line [WORDS]`, and the nestable
94
+ `kotoshu:disable-file` / `kotoshu:enable-file` block, recognized in
95
+ each format's comment syntax: HTML comments in Markdown, `//` line
96
+ comments in AsciiDoc, bare lines in plain text (trailing directives
97
+ in Markdown/AsciiDoc). One shared scanner in `Documents::Suppressions`
98
+ attaches them to every `Document`, and the `Spellchecker` facade
99
+ filters them out of `check`: suppressed words move to
100
+ `DocumentResult#suppressed_errors` (marked `suppressed: true`,
101
+ `suppressed_by: "inline"`), and `kotoshu check --show-suppressed`
102
+ lists them. Result JSON gains `suppressed` / `suppressed_by` on
103
+ errors and `suppressedCount` / `suppressedErrors` at the top level.
104
+ - **CI baselines** (plan 82) - `kotoshu baseline init FILE ...` writes
105
+ `.kotoshu-baseline.json` (canonical `{version, entries: [{file, line,
106
+ word, count}]}` via lutaml-model; count-based, so it survives
107
+ reformatting); `kotoshu check --baseline FILE` lets covered errors
108
+ pass, fails on new ones, and reports stale entries. SARIF output
109
+ marks baseline-suppressed results with a `suppressions` entry
110
+ (`kind: external`, `justification: "baseline"`), and JSON output
111
+ adds a `baseline` block.
112
+ - **pre-commit hook** (plan 81) - `.pre-commit-hooks.yaml` ships hook
113
+ id `kotoshu` (`language: system`, entry `kotoshu check`, text-format
114
+ file pattern); requires Ruby plus the gem, documented honestly in the
115
+ README.
116
+
117
+ ## [0.7.0] — 2026-09-05
118
+
119
+ The universal-kotoshu cut (plan 65's roadmap milestone): tiered
120
+ models, the Rust core, and the native extension.
121
+
122
+ ### Added
123
+
124
+ - **Model tiers** — `full` (~120 MB) / `fluency` (~15 MB, the new
125
+ default) / `mini` (~3 MB), resolved through the models repo's
126
+ registry (`kotoshu://models/{lang}/{tier}`) with sha256-verified
127
+ primary→mirror→vocab downloads; `KOTOSHU_MODEL_TIER`, `setup
128
+ LANG --model --tier`, and a tier-less legacy-cache bridge.
129
+ - **Native extension** (`ext/kotoshu_native`, optional Rust
130
+ accelerator): `KOTOSHU_BACKEND=ruby|native|auto` (default ruby;
131
+ pure-Ruby installs unchanged), `Kotoshu::Native.available?`, and
132
+ `rake kotoshu:conformance:compare` — 2630 vectors, zero divergences
133
+ between the Ruby and Rust engines.
134
+ - **Conformance export** — `rake kotoshu:conformance:export` freezes
135
+ engine behavior as golden vectors (2630 lines) consumed by
136
+ kotoshu-rs CI.
137
+ - **Confidence cascade** — `semantic_cascade_threshold`
138
+ (`KOTOSHU_SEMANTIC_CASCADE_THRESHOLD`): skip the ONNX rerank when
139
+ the composite strategies are already confident.
140
+ - Hunspell correctness: CHECKCOMPOUNDPATTERN replacement support,
141
+ dot-split casing, INITCAP ngram-root skip, AF alias flag-format
142
+ parsing, and a long tail of fixture fixes (integrational suite
143
+ 176 examples, 0 failures, 1 documented pending).
144
+
145
+ ### Fixed
146
+
147
+ - Model downloads cached 134-byte LFS pointer stubs whose own sha256
148
+ verified forever — the model source now fetches from the media host
149
+ and rejects stubs at download and verify.
150
+ - `Embeddings::Vocabulary` crashed on the shipped wrapped vocab
151
+ format; shared normalizer added.
152
+ - `Models::OnnxModel` used a nonexistent ONNX Runtime API and wrong
153
+ IO names — rewritten against `InferenceSession`.
154
+ - CLI JSON output restored to the documented camelCase contract;
155
+ `DictionaryNotFoundError` mapping restored at the cache boundary.
156
+
8
157
  ## [0.5.0] — 2026-06-30
9
158
 
10
159
  Tier 2.5 release. Adds the personal-dictionary CLI, completes direct
data/Cargo.lock ADDED
@@ -0,0 +1,291 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.5"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "bindgen"
16
+ version = "0.72.1"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
19
+ dependencies = [
20
+ "bitflags",
21
+ "cexpr",
22
+ "clang-sys",
23
+ "itertools",
24
+ "proc-macro2",
25
+ "quote",
26
+ "regex",
27
+ "rustc-hash",
28
+ "shlex",
29
+ "syn",
30
+ ]
31
+
32
+ [[package]]
33
+ name = "bitflags"
34
+ version = "2.13.1"
35
+ source = "registry+https://github.com/rust-lang/crates.io-index"
36
+ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
37
+
38
+ [[package]]
39
+ name = "cexpr"
40
+ version = "0.6.0"
41
+ source = "registry+https://github.com/rust-lang/crates.io-index"
42
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
43
+ dependencies = [
44
+ "nom",
45
+ ]
46
+
47
+ [[package]]
48
+ name = "cfg-if"
49
+ version = "1.0.4"
50
+ source = "registry+https://github.com/rust-lang/crates.io-index"
51
+ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
52
+
53
+ [[package]]
54
+ name = "clang-sys"
55
+ version = "1.9.1"
56
+ source = "registry+https://github.com/rust-lang/crates.io-index"
57
+ checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
58
+ dependencies = [
59
+ "glob",
60
+ "libc",
61
+ "libloading",
62
+ ]
63
+
64
+ [[package]]
65
+ name = "either"
66
+ version = "1.18.0"
67
+ source = "registry+https://github.com/rust-lang/crates.io-index"
68
+ checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
69
+
70
+ [[package]]
71
+ name = "glob"
72
+ version = "0.3.4"
73
+ source = "registry+https://github.com/rust-lang/crates.io-index"
74
+ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
75
+
76
+ [[package]]
77
+ name = "itertools"
78
+ version = "0.13.0"
79
+ source = "registry+https://github.com/rust-lang/crates.io-index"
80
+ checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
81
+ dependencies = [
82
+ "either",
83
+ ]
84
+
85
+ [[package]]
86
+ name = "kotoshu"
87
+ version = "0.1.0"
88
+ source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#9941f1ea1a6157c57659e428f475959508d4d0b3"
89
+ dependencies = [
90
+ "magnus",
91
+ ]
92
+
93
+ [[package]]
94
+ name = "kotoshu_native"
95
+ version = "0.0.0"
96
+ dependencies = [
97
+ "kotoshu",
98
+ "magnus",
99
+ "rb-sys",
100
+ ]
101
+
102
+ [[package]]
103
+ name = "lazy_static"
104
+ version = "1.5.0"
105
+ source = "registry+https://github.com/rust-lang/crates.io-index"
106
+ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
107
+
108
+ [[package]]
109
+ name = "libc"
110
+ version = "0.2.189"
111
+ source = "registry+https://github.com/rust-lang/crates.io-index"
112
+ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
113
+
114
+ [[package]]
115
+ name = "libloading"
116
+ version = "0.8.9"
117
+ source = "registry+https://github.com/rust-lang/crates.io-index"
118
+ checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
119
+ dependencies = [
120
+ "cfg-if",
121
+ "windows-link",
122
+ ]
123
+
124
+ [[package]]
125
+ name = "magnus"
126
+ version = "0.8.2"
127
+ source = "registry+https://github.com/rust-lang/crates.io-index"
128
+ checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
129
+ dependencies = [
130
+ "magnus-macros",
131
+ "rb-sys",
132
+ "rb-sys-env",
133
+ "seq-macro",
134
+ ]
135
+
136
+ [[package]]
137
+ name = "magnus-macros"
138
+ version = "0.8.0"
139
+ source = "registry+https://github.com/rust-lang/crates.io-index"
140
+ checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
141
+ dependencies = [
142
+ "proc-macro2",
143
+ "quote",
144
+ "syn",
145
+ ]
146
+
147
+ [[package]]
148
+ name = "memchr"
149
+ version = "2.8.3"
150
+ source = "registry+https://github.com/rust-lang/crates.io-index"
151
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
152
+
153
+ [[package]]
154
+ name = "minimal-lexical"
155
+ version = "0.2.1"
156
+ source = "registry+https://github.com/rust-lang/crates.io-index"
157
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
158
+
159
+ [[package]]
160
+ name = "nom"
161
+ version = "7.1.3"
162
+ source = "registry+https://github.com/rust-lang/crates.io-index"
163
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
164
+ dependencies = [
165
+ "memchr",
166
+ "minimal-lexical",
167
+ ]
168
+
169
+ [[package]]
170
+ name = "proc-macro2"
171
+ version = "1.0.107"
172
+ source = "registry+https://github.com/rust-lang/crates.io-index"
173
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
174
+ dependencies = [
175
+ "unicode-ident",
176
+ ]
177
+
178
+ [[package]]
179
+ name = "quote"
180
+ version = "1.0.47"
181
+ source = "registry+https://github.com/rust-lang/crates.io-index"
182
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
183
+ dependencies = [
184
+ "proc-macro2",
185
+ ]
186
+
187
+ [[package]]
188
+ name = "rb-sys"
189
+ version = "0.9.130"
190
+ source = "registry+https://github.com/rust-lang/crates.io-index"
191
+ checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
192
+ dependencies = [
193
+ "rb-sys-build",
194
+ ]
195
+
196
+ [[package]]
197
+ name = "rb-sys-build"
198
+ version = "0.9.130"
199
+ source = "registry+https://github.com/rust-lang/crates.io-index"
200
+ checksum = "05be6f9c86fe5482808162826f6c047d093b3670582296c770de1d94f8066694"
201
+ dependencies = [
202
+ "bindgen",
203
+ "lazy_static",
204
+ "proc-macro2",
205
+ "quote",
206
+ "regex",
207
+ "shell-words",
208
+ "syn",
209
+ ]
210
+
211
+ [[package]]
212
+ name = "rb-sys-env"
213
+ version = "0.2.3"
214
+ source = "registry+https://github.com/rust-lang/crates.io-index"
215
+ checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
216
+
217
+ [[package]]
218
+ name = "regex"
219
+ version = "1.13.1"
220
+ source = "registry+https://github.com/rust-lang/crates.io-index"
221
+ checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
222
+ dependencies = [
223
+ "aho-corasick",
224
+ "memchr",
225
+ "regex-automata",
226
+ "regex-syntax",
227
+ ]
228
+
229
+ [[package]]
230
+ name = "regex-automata"
231
+ version = "0.4.18"
232
+ source = "registry+https://github.com/rust-lang/crates.io-index"
233
+ checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
234
+ dependencies = [
235
+ "aho-corasick",
236
+ "memchr",
237
+ "regex-syntax",
238
+ ]
239
+
240
+ [[package]]
241
+ name = "regex-syntax"
242
+ version = "0.8.11"
243
+ source = "registry+https://github.com/rust-lang/crates.io-index"
244
+ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
245
+
246
+ [[package]]
247
+ name = "rustc-hash"
248
+ version = "2.1.3"
249
+ source = "registry+https://github.com/rust-lang/crates.io-index"
250
+ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
251
+
252
+ [[package]]
253
+ name = "seq-macro"
254
+ version = "0.3.6"
255
+ source = "registry+https://github.com/rust-lang/crates.io-index"
256
+ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
257
+
258
+ [[package]]
259
+ name = "shell-words"
260
+ version = "1.1.1"
261
+ source = "registry+https://github.com/rust-lang/crates.io-index"
262
+ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
263
+
264
+ [[package]]
265
+ name = "shlex"
266
+ version = "1.3.0"
267
+ source = "registry+https://github.com/rust-lang/crates.io-index"
268
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
269
+
270
+ [[package]]
271
+ name = "syn"
272
+ version = "2.0.119"
273
+ source = "registry+https://github.com/rust-lang/crates.io-index"
274
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
275
+ dependencies = [
276
+ "proc-macro2",
277
+ "quote",
278
+ "unicode-ident",
279
+ ]
280
+
281
+ [[package]]
282
+ name = "unicode-ident"
283
+ version = "1.0.24"
284
+ source = "registry+https://github.com/rust-lang/crates.io-index"
285
+ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
286
+
287
+ [[package]]
288
+ name = "windows-link"
289
+ version = "0.2.1"
290
+ source = "registry+https://github.com/rust-lang/crates.io-index"
291
+ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
data/Cargo.toml ADDED
@@ -0,0 +1,15 @@
1
+ # Kotoshu gem Rust workspace (plan 66 / P4b) — mirrors parsanol-ruby's
2
+ # root Cargo.toml: rb_sys's compile task runs `cargo metadata` from the gem
3
+ # root, so the extension crate must be reachable through a root manifest.
4
+ [workspace]
5
+ resolver = "2"
6
+ members = ["ext/kotoshu_native"]
7
+
8
+ # Release profile follows the kotoshu-rs workspace policy (plan 66): fat
9
+ # LTO and single codegen unit so the shipped accelerator is as fast as the
10
+ # core's own optimized builds. Local development builds use the dev profile
11
+ # via RB_SYS_CARGO_PROFILE (see ext/kotoshu_native/extconf.rb).
12
+ [profile.release]
13
+ opt-level = 3
14
+ lto = "fat"
15
+ codegen-units = 1