kotoshu 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.pre-commit-hooks.yaml +18 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +149 -0
- data/Cargo.lock +291 -0
- data/Cargo.toml +15 -0
- data/README.adoc +206 -0
- data/Rakefile +103 -0
- data/TODO.impl/00-vision.md +212 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +17 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +21 -0
- data/TODO.impl/38-onnx-semantic-gating.md +63 -0
- data/TODO.impl/39-tier3-and-beyond.md +36 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +109 -0
- data/docs/cache-architecture.md +502 -0
- data/ext/kotoshu_native/Cargo.toml +37 -0
- data/ext/kotoshu_native/extconf.rb +15 -0
- data/ext/kotoshu_native/src/lib.rs +15 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +57 -7
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +95 -236
- data/lib/kotoshu/algorithms/suggest.rb +231 -50
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +1 -1
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +152 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +46 -3
- data/lib/kotoshu/cache/frequency_cache.rb +2 -2
- data/lib/kotoshu/cache/language_cache.rb +33 -11
- data/lib/kotoshu/cache/model_cache.rb +449 -7
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache.rb +1 -0
- data/lib/kotoshu/cli/baseline_command.rb +81 -0
- data/lib/kotoshu/cli/cache_command.rb +44 -38
- data/lib/kotoshu/cli/completions_command.rb +1 -1
- data/lib/kotoshu/cli/directory_check.rb +141 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli.rb +336 -61
- data/lib/kotoshu/configuration.rb +69 -9
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +7 -2
- data/lib/kotoshu/core/models/result/document_result.rb +13 -3
- data/lib/kotoshu/core/models/result/word_result.rb +19 -1
- data/lib/kotoshu/data/common_words_loader.rb +0 -1
- data/lib/kotoshu/debug_mode.rb +2 -0
- data/lib/kotoshu/dictionary/base.rb +14 -0
- data/lib/kotoshu/dictionary/cspell.rb +3 -8
- data/lib/kotoshu/dictionary/plain_text.rb +17 -0
- data/lib/kotoshu/documents/document.rb +12 -2
- data/lib/kotoshu/documents/plain_text_document.rb +5 -2
- data/lib/kotoshu/documents/source_position.rb +8 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents.rb +1 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +3 -5
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +11 -1
- data/lib/kotoshu/embeddings/vocabulary.rb +25 -1
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +2 -2
- data/lib/kotoshu/jekyll.rb +98 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +220 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +20 -0
- data/lib/kotoshu/keyboard.rb +8 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +17 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +12 -0
- data/lib/kotoshu/language.rb +4 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +18 -0
- data/lib/kotoshu/metrics_collector.rb +18 -2
- data/lib/kotoshu/metrics_module.rb +11 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +11 -3
- data/lib/kotoshu/models/onnx_model.rb +22 -23
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +26 -0
- data/lib/kotoshu/readers/aff_data.rb +94 -19
- data/lib/kotoshu/readers/aff_reader.rb +44 -35
- data/lib/kotoshu/readers/lookup_builder.rb +2 -16
- data/lib/kotoshu/resource_bundle.rb +32 -7
- data/lib/kotoshu/resource_manager.rb +313 -23
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/source_registry.rb +26 -4
- data/lib/kotoshu/spellchecker.rb +88 -22
- data/lib/kotoshu/suggestions/frequency_provider.rb +59 -0
- data/lib/kotoshu/suggestions/generator.rb +4 -5
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +10 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +62 -3
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +70 -199
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +11 -2
- data/lib/kotoshu/suggestions/suggestion_set.rb +9 -2
- data/lib/kotoshu/suggestions.rb +2 -0
- data/lib/kotoshu/tasks/check_task.rb +98 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +42 -12
- data/rakelib/compile.rake +10 -0
- data/scripts/extract_eval_grids.rb +34 -0
- data/sig/kotoshu.rbs +331 -1
- metadata +123 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f90290a36de788d46588e6883b0619f9fb1dff8e841f629fd021c07e156256ae
|
|
4
|
+
data.tar.gz: 113cbf58fb1b9c297cfb8551a9076875f4882be9e7b17bbebe330ecc53aac94a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1520bf0ce2667b31872c137b55099834a93954f821c93903b193306d537703173ad37e84446dfd41e082e25db50d4a54c694d1623b43875fd5b61294b4cf67f4
|
|
7
|
+
data.tar.gz: f22ca5507daa129090a2d292d4e526200f197f32520b864c67e3ffbab82862a431d155303613e670aab97e48d213bad32ad0ba4b4cdf1178a8de1dff940a1cbd
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# pre-commit hook definitions (https://pre-commit.com).
|
|
2
|
+
#
|
|
3
|
+
# Honest requirements: this hook is `language: system` — it runs the
|
|
4
|
+
# `kotoshu` executable from PATH, so the machine needs a Ruby
|
|
5
|
+
# environment with the gem installed:
|
|
6
|
+
#
|
|
7
|
+
# gem install kotoshu
|
|
8
|
+
# kotoshu setup en # or the languages you write in
|
|
9
|
+
#
|
|
10
|
+
# pre-commit passes the matched filenames to the entry; `kotoshu check`
|
|
11
|
+
# exits 1 when a file has unsuppressed misspellings. The WebAssembly
|
|
12
|
+
# client (in development) is the future zero-Ruby path.
|
|
13
|
+
- id: kotoshu
|
|
14
|
+
name: kotoshu semantic spell check
|
|
15
|
+
description: Semantic spell checking for prose and documentation
|
|
16
|
+
entry: kotoshu check
|
|
17
|
+
language: system
|
|
18
|
+
files: \.(txt|text|md|markdown|mdown|mkd|adoc|asciidoc|ad|rst|rest|creole|rdoc|org|pod|textile|wiki|mail|article|log|notes)$
|
data/.yardopts
ADDED
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,155 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Remote `kotoshu setup` for staged languages: the dictionaries repo ships only
|
|
12
|
+
`en` under the `{lang}/spelling/` layout while staged languages sit flat at
|
|
13
|
+
`{lang}/index.*`; downloads now try the sublayout first and fall back to the
|
|
14
|
+
flat layout (verified end-to-end for `it`).
|
|
15
|
+
- `require "kotoshu/tasks"` / `require "kotoshu/jekyll"` standalone now load
|
|
16
|
+
without needing `require "kotoshu"` first.
|
|
17
|
+
- Baseline paths compare canonically: entries recorded as `docs/a.md` match a
|
|
18
|
+
directory-walk check over `./docs/a.md`, and recorded entries are stored
|
|
19
|
+
canonical.
|
|
20
|
+
### Fixed
|
|
21
|
+
- **Unicode word detection** (plan 91 Track A) - `Spellchecker` word
|
|
22
|
+
extraction accepted only ASCII letters, so Greek and Ukrainian users
|
|
23
|
+
could not check any text through the CLI. Extraction now follows the
|
|
24
|
+
configured language's tokenizer: Greek words for `el`, Cyrillic for
|
|
25
|
+
`uk` (in-word apostrophe included, for names like Мар'яна), and
|
|
26
|
+
Latin-script letters including capitals such as Å and Ä for Latin
|
|
27
|
+
languages. Wrong-script words stay uncheckable; languages without a
|
|
28
|
+
script tokenizer (en, de, ...) keep the historical ASCII behavior;
|
|
29
|
+
the frozen conformance vectors are unchanged.
|
|
30
|
+
- **Swedish aff parsing** (plan 91 Track B) - `Readers::AffReader`
|
|
31
|
+
raised `RegexpError: unmatched close parenthesis` on the Swedish
|
|
32
|
+
dictionary (dictionaries pin 1829a3e). `COMPOUNDRULE )k` uses `)`
|
|
33
|
+
itself as a flag character, but the rule compiler escaped it only in
|
|
34
|
+
some positions. Every flag character is now escaped at compile time,
|
|
35
|
+
so the dictionary loads and its compound rules match.
|
|
36
|
+
### Added
|
|
37
|
+
- **Framework integrations** (plan 89) - four opt-in integration
|
|
38
|
+
layers, all inside the gem with NO new runtime dependencies
|
|
39
|
+
(soft-dependency pattern, like onnxruntime/suika):
|
|
40
|
+
- `Kotoshu::Validators::SpellingValidator`, an ActiveModel
|
|
41
|
+
EachValidator (`validates :body, spelling: true` after the
|
|
42
|
+
documented one-line alias, or `validates_with` directly): one
|
|
43
|
+
validation error per misspelling with the top suggestion in the
|
|
44
|
+
message; `language:` and `personal_words:` options. Loads the
|
|
45
|
+
real ActiveModel standalone - Rails is not required.
|
|
46
|
+
- RSpec matchers (`require "kotoshu/rspec"` +
|
|
47
|
+
`include Kotoshu::Rspec::Matchers`):
|
|
48
|
+
`expect_words("helo").to all_be_spelled_correctly`, and
|
|
49
|
+
`expect(text) / expect_document(path).to be_spelled_correctly`
|
|
50
|
+
for words or whole documents, with failure messages that list
|
|
51
|
+
each misspelling and its suggestions.
|
|
52
|
+
- Rake task (`require "kotoshu/tasks"`): `rake kotoshu:check`
|
|
53
|
+
over the repository text files using the plan-88 file selection
|
|
54
|
+
(known extensions, .gitignore/.ignore, default skips);
|
|
55
|
+
`Kotoshu::Tasks::CheckTask` exposes files/language/fail_on_error
|
|
56
|
+
/baseline configuration.
|
|
57
|
+
- Jekyll generator (`Kotoshu::Jekyll::Generator`, `safe`):
|
|
58
|
+
checks posts and drafts and fails the build on new spelling
|
|
59
|
+
errors; a `.kotoshu-baseline.json` in the site source is
|
|
60
|
+
respected so baselined errors do not block builds.
|
|
61
|
+
- **CLI directory mode** (plan 88) - `kotoshu check DIR [DIR ...]`
|
|
62
|
+
walks trees and checks every file with a known text extension
|
|
63
|
+
(md markdown asciidoc adoc txt rst mdx) unless `--include` /
|
|
64
|
+
`--exclude` globs say otherwise. `.gitignore` and `.ignore` files
|
|
65
|
+
are honored through a standard glob subset implemented in Ruby
|
|
66
|
+
(`*`, `?`, `**` as a whole segment, `!` negation with last-match
|
|
67
|
+
wins, trailing `/` for directory-only patterns, anchoring, nested
|
|
68
|
+
scoping; files inside ignored directories cannot be re-included).
|
|
69
|
+
Hidden files and `.git` / `node_modules` / `vendor` / `target` are
|
|
70
|
+
skipped by default. Text output prints one section per file plus a
|
|
71
|
+
summary; JSON and SARIF emit one combined document with per-file
|
|
72
|
+
entries/runs; `--baseline` applies per file as shipped; exit codes
|
|
73
|
+
are unchanged; interactive mode stays file-only with an explicit
|
|
74
|
+
notice for directory targets. Single-file and stdin behavior is
|
|
75
|
+
byte-identical to before.
|
|
76
|
+
- **Wave-1 language modules and keyboard layouts** (plan 84) -
|
|
77
|
+
language modules for the thirteen model-registry newcomers (ca cs
|
|
78
|
+
da el hu it nl pl ro sv tr uk vi): thin compositions over the shared
|
|
79
|
+
Latin tokenizer/normalizer plus registry entries, with real care
|
|
80
|
+
where the script needs it — Turkish dotless-i case folding
|
|
81
|
+
(`İZMİR` -> `izmir`, `ISTANBUL` -> `ıstanbul`), Greek-aware
|
|
82
|
+
normalization (final sigma `ΕΛΛΑΣ` -> `ελλας`, opt-in accent
|
|
83
|
+
stripping) and Greek/Cyrillic script tokenizers that keep
|
|
84
|
+
non-Latin words. Keyboard layouts grow from 5 to 19: Turkish-Q,
|
|
85
|
+
Ukrainian-JCUKEN and Greek-Phonetic national grids mirrored from
|
|
86
|
+
the models repo eval harness (a drift spec asserts the gem grids
|
|
87
|
+
cover the eval grids; `scripts/extract_eval_grids.rb` refreshes the
|
|
88
|
+
committed snapshot), plus one parameterized Latin family
|
|
89
|
+
(`Keyboard::Layouts::Latin` and members) over the qwerty/qwertz
|
|
90
|
+
base grids with real Nordic å/æ/ø and å/ä/ö keys. The pre-existing
|
|
91
|
+
five layouts and ten language modules are unchanged.
|
|
92
|
+
- **Inline ignore directives** (plan 82) - `kotoshu:disable-line`,
|
|
93
|
+
`kotoshu:disable-next-line [WORDS]`, and the nestable
|
|
94
|
+
`kotoshu:disable-file` / `kotoshu:enable-file` block, recognized in
|
|
95
|
+
each format's comment syntax: HTML comments in Markdown, `//` line
|
|
96
|
+
comments in AsciiDoc, bare lines in plain text (trailing directives
|
|
97
|
+
in Markdown/AsciiDoc). One shared scanner in `Documents::Suppressions`
|
|
98
|
+
attaches them to every `Document`, and the `Spellchecker` facade
|
|
99
|
+
filters them out of `check`: suppressed words move to
|
|
100
|
+
`DocumentResult#suppressed_errors` (marked `suppressed: true`,
|
|
101
|
+
`suppressed_by: "inline"`), and `kotoshu check --show-suppressed`
|
|
102
|
+
lists them. Result JSON gains `suppressed` / `suppressed_by` on
|
|
103
|
+
errors and `suppressedCount` / `suppressedErrors` at the top level.
|
|
104
|
+
- **CI baselines** (plan 82) - `kotoshu baseline init FILE ...` writes
|
|
105
|
+
`.kotoshu-baseline.json` (canonical `{version, entries: [{file, line,
|
|
106
|
+
word, count}]}` via lutaml-model; count-based, so it survives
|
|
107
|
+
reformatting); `kotoshu check --baseline FILE` lets covered errors
|
|
108
|
+
pass, fails on new ones, and reports stale entries. SARIF output
|
|
109
|
+
marks baseline-suppressed results with a `suppressions` entry
|
|
110
|
+
(`kind: external`, `justification: "baseline"`), and JSON output
|
|
111
|
+
adds a `baseline` block.
|
|
112
|
+
- **pre-commit hook** (plan 81) - `.pre-commit-hooks.yaml` ships hook
|
|
113
|
+
id `kotoshu` (`language: system`, entry `kotoshu check`, text-format
|
|
114
|
+
file pattern); requires Ruby plus the gem, documented honestly in the
|
|
115
|
+
README.
|
|
116
|
+
|
|
117
|
+
## [0.7.0] — 2026-09-05
|
|
118
|
+
|
|
119
|
+
The universal-kotoshu cut (plan 65's roadmap milestone): tiered
|
|
120
|
+
models, the Rust core, and the native extension.
|
|
121
|
+
|
|
122
|
+
### Added
|
|
123
|
+
|
|
124
|
+
- **Model tiers** — `full` (~120 MB) / `fluency` (~15 MB, the new
|
|
125
|
+
default) / `mini` (~3 MB), resolved through the models repo's
|
|
126
|
+
registry (`kotoshu://models/{lang}/{tier}`) with sha256-verified
|
|
127
|
+
primary→mirror→vocab downloads; `KOTOSHU_MODEL_TIER`, `setup
|
|
128
|
+
LANG --model --tier`, and a tier-less legacy-cache bridge.
|
|
129
|
+
- **Native extension** (`ext/kotoshu_native`, optional Rust
|
|
130
|
+
accelerator): `KOTOSHU_BACKEND=ruby|native|auto` (default ruby;
|
|
131
|
+
pure-Ruby installs unchanged), `Kotoshu::Native.available?`, and
|
|
132
|
+
`rake kotoshu:conformance:compare` — 2630 vectors, zero divergences
|
|
133
|
+
between the Ruby and Rust engines.
|
|
134
|
+
- **Conformance export** — `rake kotoshu:conformance:export` freezes
|
|
135
|
+
engine behavior as golden vectors (2630 lines) consumed by
|
|
136
|
+
kotoshu-rs CI.
|
|
137
|
+
- **Confidence cascade** — `semantic_cascade_threshold`
|
|
138
|
+
(`KOTOSHU_SEMANTIC_CASCADE_THRESHOLD`): skip the ONNX rerank when
|
|
139
|
+
the composite strategies are already confident.
|
|
140
|
+
- Hunspell correctness: CHECKCOMPOUNDPATTERN replacement support,
|
|
141
|
+
dot-split casing, INITCAP ngram-root skip, AF alias flag-format
|
|
142
|
+
parsing, and a long tail of fixture fixes (integrational suite
|
|
143
|
+
176 examples, 0 failures, 1 documented pending).
|
|
144
|
+
|
|
145
|
+
### Fixed
|
|
146
|
+
|
|
147
|
+
- Model downloads cached 134-byte LFS pointer stubs whose own sha256
|
|
148
|
+
verified forever — the model source now fetches from the media host
|
|
149
|
+
and rejects stubs at download and verify.
|
|
150
|
+
- `Embeddings::Vocabulary` crashed on the shipped wrapped vocab
|
|
151
|
+
format; shared normalizer added.
|
|
152
|
+
- `Models::OnnxModel` used a nonexistent ONNX Runtime API and wrong
|
|
153
|
+
IO names — rewritten against `InferenceSession`.
|
|
154
|
+
- CLI JSON output restored to the documented camelCase contract;
|
|
155
|
+
`DictionaryNotFoundError` mapping restored at the cache boundary.
|
|
156
|
+
|
|
8
157
|
## [0.5.0] — 2026-06-30
|
|
9
158
|
|
|
10
159
|
Tier 2.5 release. Adds the personal-dictionary CLI, completes direct
|
data/Cargo.lock
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.5"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "bindgen"
|
|
16
|
+
version = "0.72.1"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"bitflags",
|
|
21
|
+
"cexpr",
|
|
22
|
+
"clang-sys",
|
|
23
|
+
"itertools",
|
|
24
|
+
"proc-macro2",
|
|
25
|
+
"quote",
|
|
26
|
+
"regex",
|
|
27
|
+
"rustc-hash",
|
|
28
|
+
"shlex",
|
|
29
|
+
"syn",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "bitflags"
|
|
34
|
+
version = "2.13.1"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
|
37
|
+
|
|
38
|
+
[[package]]
|
|
39
|
+
name = "cexpr"
|
|
40
|
+
version = "0.6.0"
|
|
41
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
42
|
+
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
|
43
|
+
dependencies = [
|
|
44
|
+
"nom",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[[package]]
|
|
48
|
+
name = "cfg-if"
|
|
49
|
+
version = "1.0.4"
|
|
50
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
51
|
+
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
|
52
|
+
|
|
53
|
+
[[package]]
|
|
54
|
+
name = "clang-sys"
|
|
55
|
+
version = "1.9.1"
|
|
56
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
57
|
+
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
|
|
58
|
+
dependencies = [
|
|
59
|
+
"glob",
|
|
60
|
+
"libc",
|
|
61
|
+
"libloading",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[[package]]
|
|
65
|
+
name = "either"
|
|
66
|
+
version = "1.18.0"
|
|
67
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
|
+
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
|
|
69
|
+
|
|
70
|
+
[[package]]
|
|
71
|
+
name = "glob"
|
|
72
|
+
version = "0.3.4"
|
|
73
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
74
|
+
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
|
|
75
|
+
|
|
76
|
+
[[package]]
|
|
77
|
+
name = "itertools"
|
|
78
|
+
version = "0.13.0"
|
|
79
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
80
|
+
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
|
81
|
+
dependencies = [
|
|
82
|
+
"either",
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
[[package]]
|
|
86
|
+
name = "kotoshu"
|
|
87
|
+
version = "0.1.0"
|
|
88
|
+
source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#9941f1ea1a6157c57659e428f475959508d4d0b3"
|
|
89
|
+
dependencies = [
|
|
90
|
+
"magnus",
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
[[package]]
|
|
94
|
+
name = "kotoshu_native"
|
|
95
|
+
version = "0.0.0"
|
|
96
|
+
dependencies = [
|
|
97
|
+
"kotoshu",
|
|
98
|
+
"magnus",
|
|
99
|
+
"rb-sys",
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
[[package]]
|
|
103
|
+
name = "lazy_static"
|
|
104
|
+
version = "1.5.0"
|
|
105
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
106
|
+
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
|
107
|
+
|
|
108
|
+
[[package]]
|
|
109
|
+
name = "libc"
|
|
110
|
+
version = "0.2.189"
|
|
111
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
112
|
+
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|
113
|
+
|
|
114
|
+
[[package]]
|
|
115
|
+
name = "libloading"
|
|
116
|
+
version = "0.8.9"
|
|
117
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
118
|
+
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
|
119
|
+
dependencies = [
|
|
120
|
+
"cfg-if",
|
|
121
|
+
"windows-link",
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
[[package]]
|
|
125
|
+
name = "magnus"
|
|
126
|
+
version = "0.8.2"
|
|
127
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
128
|
+
checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
|
|
129
|
+
dependencies = [
|
|
130
|
+
"magnus-macros",
|
|
131
|
+
"rb-sys",
|
|
132
|
+
"rb-sys-env",
|
|
133
|
+
"seq-macro",
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
[[package]]
|
|
137
|
+
name = "magnus-macros"
|
|
138
|
+
version = "0.8.0"
|
|
139
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
140
|
+
checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
|
|
141
|
+
dependencies = [
|
|
142
|
+
"proc-macro2",
|
|
143
|
+
"quote",
|
|
144
|
+
"syn",
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
[[package]]
|
|
148
|
+
name = "memchr"
|
|
149
|
+
version = "2.8.3"
|
|
150
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
151
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
152
|
+
|
|
153
|
+
[[package]]
|
|
154
|
+
name = "minimal-lexical"
|
|
155
|
+
version = "0.2.1"
|
|
156
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
157
|
+
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|
158
|
+
|
|
159
|
+
[[package]]
|
|
160
|
+
name = "nom"
|
|
161
|
+
version = "7.1.3"
|
|
162
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
163
|
+
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
|
164
|
+
dependencies = [
|
|
165
|
+
"memchr",
|
|
166
|
+
"minimal-lexical",
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
[[package]]
|
|
170
|
+
name = "proc-macro2"
|
|
171
|
+
version = "1.0.107"
|
|
172
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
173
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
174
|
+
dependencies = [
|
|
175
|
+
"unicode-ident",
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
[[package]]
|
|
179
|
+
name = "quote"
|
|
180
|
+
version = "1.0.47"
|
|
181
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
182
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
183
|
+
dependencies = [
|
|
184
|
+
"proc-macro2",
|
|
185
|
+
]
|
|
186
|
+
|
|
187
|
+
[[package]]
|
|
188
|
+
name = "rb-sys"
|
|
189
|
+
version = "0.9.130"
|
|
190
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
191
|
+
checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
|
|
192
|
+
dependencies = [
|
|
193
|
+
"rb-sys-build",
|
|
194
|
+
]
|
|
195
|
+
|
|
196
|
+
[[package]]
|
|
197
|
+
name = "rb-sys-build"
|
|
198
|
+
version = "0.9.130"
|
|
199
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
200
|
+
checksum = "05be6f9c86fe5482808162826f6c047d093b3670582296c770de1d94f8066694"
|
|
201
|
+
dependencies = [
|
|
202
|
+
"bindgen",
|
|
203
|
+
"lazy_static",
|
|
204
|
+
"proc-macro2",
|
|
205
|
+
"quote",
|
|
206
|
+
"regex",
|
|
207
|
+
"shell-words",
|
|
208
|
+
"syn",
|
|
209
|
+
]
|
|
210
|
+
|
|
211
|
+
[[package]]
|
|
212
|
+
name = "rb-sys-env"
|
|
213
|
+
version = "0.2.3"
|
|
214
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
215
|
+
checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
|
|
216
|
+
|
|
217
|
+
[[package]]
|
|
218
|
+
name = "regex"
|
|
219
|
+
version = "1.13.1"
|
|
220
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
221
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
222
|
+
dependencies = [
|
|
223
|
+
"aho-corasick",
|
|
224
|
+
"memchr",
|
|
225
|
+
"regex-automata",
|
|
226
|
+
"regex-syntax",
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
[[package]]
|
|
230
|
+
name = "regex-automata"
|
|
231
|
+
version = "0.4.18"
|
|
232
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
233
|
+
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
|
234
|
+
dependencies = [
|
|
235
|
+
"aho-corasick",
|
|
236
|
+
"memchr",
|
|
237
|
+
"regex-syntax",
|
|
238
|
+
]
|
|
239
|
+
|
|
240
|
+
[[package]]
|
|
241
|
+
name = "regex-syntax"
|
|
242
|
+
version = "0.8.11"
|
|
243
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
244
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
245
|
+
|
|
246
|
+
[[package]]
|
|
247
|
+
name = "rustc-hash"
|
|
248
|
+
version = "2.1.3"
|
|
249
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
250
|
+
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
|
251
|
+
|
|
252
|
+
[[package]]
|
|
253
|
+
name = "seq-macro"
|
|
254
|
+
version = "0.3.6"
|
|
255
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
256
|
+
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|
257
|
+
|
|
258
|
+
[[package]]
|
|
259
|
+
name = "shell-words"
|
|
260
|
+
version = "1.1.1"
|
|
261
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
262
|
+
checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
|
|
263
|
+
|
|
264
|
+
[[package]]
|
|
265
|
+
name = "shlex"
|
|
266
|
+
version = "1.3.0"
|
|
267
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
268
|
+
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
269
|
+
|
|
270
|
+
[[package]]
|
|
271
|
+
name = "syn"
|
|
272
|
+
version = "2.0.119"
|
|
273
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
274
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
275
|
+
dependencies = [
|
|
276
|
+
"proc-macro2",
|
|
277
|
+
"quote",
|
|
278
|
+
"unicode-ident",
|
|
279
|
+
]
|
|
280
|
+
|
|
281
|
+
[[package]]
|
|
282
|
+
name = "unicode-ident"
|
|
283
|
+
version = "1.0.24"
|
|
284
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
285
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
286
|
+
|
|
287
|
+
[[package]]
|
|
288
|
+
name = "windows-link"
|
|
289
|
+
version = "0.2.1"
|
|
290
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
291
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
data/Cargo.toml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Kotoshu gem Rust workspace (plan 66 / P4b) — mirrors parsanol-ruby's
|
|
2
|
+
# root Cargo.toml: rb_sys's compile task runs `cargo metadata` from the gem
|
|
3
|
+
# root, so the extension crate must be reachable through a root manifest.
|
|
4
|
+
[workspace]
|
|
5
|
+
resolver = "2"
|
|
6
|
+
members = ["ext/kotoshu_native"]
|
|
7
|
+
|
|
8
|
+
# Release profile follows the kotoshu-rs workspace policy (plan 66): fat
|
|
9
|
+
# LTO and single codegen unit so the shipped accelerator is as fast as the
|
|
10
|
+
# core's own optimized builds. Local development builds use the dev profile
|
|
11
|
+
# via RB_SYS_CARGO_PROFILE (see ext/kotoshu_native/extconf.rb).
|
|
12
|
+
[profile.release]
|
|
13
|
+
opt-level = 3
|
|
14
|
+
lto = "fat"
|
|
15
|
+
codegen-units = 1
|