kotoshu 1.0.3-aarch64-linux → 1.0.5-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +75 -2
- data/README.adoc +23 -0
- data/Rakefile +46 -0
- data/TODO.impl/135-post-train-improvements.md +38 -0
- data/TODO.impl/136-prebuilt-typo-matrices.md +30 -0
- data/TODO.impl/137-typo-matrix-backfill.md +50 -0
- data/TODO.impl/138-ext-pin-drift-guard.md +43 -0
- data/TODO.impl/139-typo-observability.md +40 -0
- data/lib/kotoshu/cache/model_cache.rb +82 -1
- data/lib/kotoshu/cli.rb +14 -1
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/resource_manager.rb +26 -2
- data/lib/kotoshu/typo/engine.rb +39 -9
- data/lib/kotoshu/version.rb +1 -1
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e57abdf7e47c7816e3643aec62143988c871e33554aebf822b4faf5265362767
|
|
4
|
+
data.tar.gz: d533281c50e637484e73e3c1f94babbb5cc5fb55a3f9b5c4b65a829f8fe79294
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bc4f36fff3ceea95ab3434c2df5049d47f4ae0f694a4da48945babc2134d7511af0d8a61ec3357df8c0dbe226a242d4c75c9447ce75a9b6de56e4c1d0f17c56d
|
|
7
|
+
data.tar.gz: 127248fea50d94d462a6bc7f1f49dea7a861a10b3ce9d6f02dbfd857297565b3846a3ed43567cf836aead62d8d6f90c25e42b503a18dedb664c279f484c9bff5
|
data/Cargo.lock
CHANGED
|
@@ -35,6 +35,15 @@ version = "2.13.1"
|
|
|
35
35
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
36
|
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
|
37
37
|
|
|
38
|
+
[[package]]
|
|
39
|
+
name = "block-buffer"
|
|
40
|
+
version = "0.10.4"
|
|
41
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
42
|
+
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
|
43
|
+
dependencies = [
|
|
44
|
+
"generic-array",
|
|
45
|
+
]
|
|
46
|
+
|
|
38
47
|
[[package]]
|
|
39
48
|
name = "cexpr"
|
|
40
49
|
version = "0.6.0"
|
|
@@ -61,12 +70,51 @@ dependencies = [
|
|
|
61
70
|
"libloading",
|
|
62
71
|
]
|
|
63
72
|
|
|
73
|
+
[[package]]
|
|
74
|
+
name = "cpufeatures"
|
|
75
|
+
version = "0.2.17"
|
|
76
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
77
|
+
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
|
78
|
+
dependencies = [
|
|
79
|
+
"libc",
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
[[package]]
|
|
83
|
+
name = "crypto-common"
|
|
84
|
+
version = "0.1.7"
|
|
85
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
86
|
+
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
|
87
|
+
dependencies = [
|
|
88
|
+
"generic-array",
|
|
89
|
+
"typenum",
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
[[package]]
|
|
93
|
+
name = "digest"
|
|
94
|
+
version = "0.10.7"
|
|
95
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
96
|
+
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|
97
|
+
dependencies = [
|
|
98
|
+
"block-buffer",
|
|
99
|
+
"crypto-common",
|
|
100
|
+
]
|
|
101
|
+
|
|
64
102
|
[[package]]
|
|
65
103
|
name = "either"
|
|
66
104
|
version = "1.18.0"
|
|
67
105
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
106
|
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
|
|
69
107
|
|
|
108
|
+
[[package]]
|
|
109
|
+
name = "generic-array"
|
|
110
|
+
version = "0.14.7"
|
|
111
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
112
|
+
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
|
113
|
+
dependencies = [
|
|
114
|
+
"typenum",
|
|
115
|
+
"version_check",
|
|
116
|
+
]
|
|
117
|
+
|
|
70
118
|
[[package]]
|
|
71
119
|
name = "glob"
|
|
72
120
|
version = "0.3.4"
|
|
@@ -90,12 +138,14 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
|
90
138
|
|
|
91
139
|
[[package]]
|
|
92
140
|
name = "kotoshu"
|
|
93
|
-
version = "0.
|
|
94
|
-
source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#
|
|
141
|
+
version = "0.2.0"
|
|
142
|
+
source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#0001b50056319204c3ea4eeb0ff5886a6effde64"
|
|
95
143
|
dependencies = [
|
|
96
144
|
"magnus",
|
|
145
|
+
"rb-sys",
|
|
97
146
|
"serde",
|
|
98
147
|
"serde_json",
|
|
148
|
+
"sha2",
|
|
99
149
|
]
|
|
100
150
|
|
|
101
151
|
[[package]]
|
|
@@ -306,6 +356,17 @@ dependencies = [
|
|
|
306
356
|
"zmij",
|
|
307
357
|
]
|
|
308
358
|
|
|
359
|
+
[[package]]
|
|
360
|
+
name = "sha2"
|
|
361
|
+
version = "0.10.9"
|
|
362
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
363
|
+
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
|
364
|
+
dependencies = [
|
|
365
|
+
"cfg-if",
|
|
366
|
+
"cpufeatures",
|
|
367
|
+
"digest",
|
|
368
|
+
]
|
|
369
|
+
|
|
309
370
|
[[package]]
|
|
310
371
|
name = "shell-words"
|
|
311
372
|
version = "1.1.1"
|
|
@@ -340,12 +401,24 @@ dependencies = [
|
|
|
340
401
|
"unicode-ident",
|
|
341
402
|
]
|
|
342
403
|
|
|
404
|
+
[[package]]
|
|
405
|
+
name = "typenum"
|
|
406
|
+
version = "1.20.1"
|
|
407
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
408
|
+
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
|
409
|
+
|
|
343
410
|
[[package]]
|
|
344
411
|
name = "unicode-ident"
|
|
345
412
|
version = "1.0.24"
|
|
346
413
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
347
414
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
348
415
|
|
|
416
|
+
[[package]]
|
|
417
|
+
name = "version_check"
|
|
418
|
+
version = "0.9.5"
|
|
419
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
420
|
+
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
|
421
|
+
|
|
349
422
|
[[package]]
|
|
350
423
|
name = "windows-link"
|
|
351
424
|
version = "0.2.1"
|
data/README.adoc
CHANGED
|
@@ -398,6 +398,29 @@ variable `KOTOSHU_MODEL_TIER`; one of `full`, `fluency`, `mini`):
|
|
|
398
398
|
KOTOSHU_MODEL_TIER=mini kotoshu setup en --model
|
|
399
399
|
----
|
|
400
400
|
|
|
401
|
+
[[typo-retrieval]]
|
|
402
|
+
=== Typo Retrieval (opt-in)
|
|
403
|
+
|
|
404
|
+
The typo-retrieval layer adds a 0.5 MB character bi-encoder to the
|
|
405
|
+
suggestion pipeline: it retrieves the twenty vocabulary entries nearest
|
|
406
|
+
a misspelling's character shape, and the language's FULL tier rescores
|
|
407
|
+
them exactly. On the frozen real-pair benchmarks it lifts top-5 recall
|
|
408
|
+
over the full tier alone (+6.3 points on English, +10 to +29 on de/es/ru).
|
|
409
|
+
|
|
410
|
+
Setup fetches everything with one flag:
|
|
411
|
+
|
|
412
|
+
[source,bash]
|
|
413
|
+
----
|
|
414
|
+
kotoshu setup en --typo
|
|
415
|
+
----
|
|
416
|
+
|
|
417
|
+
The layer arms only when you opt in with `KOTOSHU_TYPO_RETRIEVAL=1`;
|
|
418
|
+
without it, suggestions are byte-identical to the base pipeline. A
|
|
419
|
+
prebuilt 26 MB retrieval matrix is served by the model registry, so
|
|
420
|
+
arming is a download (about half a second) instead of a 25-second
|
|
421
|
+
derivation. The matrix rows pair with the language's full-tier
|
|
422
|
+
vocabulary, so keep the full tier cached (setup fetches it).
|
|
423
|
+
|
|
401
424
|
[[interactive-mode]]
|
|
402
425
|
[[interactive-mode]]
|
|
403
426
|
== Interactive Mode
|
data/Rakefile
CHANGED
|
@@ -31,6 +31,52 @@ report_divergences = lambda do |divergences|
|
|
|
31
31
|
puts "... and #{divergences.size - 10} more" if divergences.size > 10
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Plan 135: the ext tracks kotoshu-rs main via a git dependency pinned
|
|
35
|
+
# in ext/kotoshu_native/Cargo.lock; after every rs merge that pin
|
|
36
|
+
# goes stale and `rake compile` silently loads an old extension whose
|
|
37
|
+
# missing methods the typo layer's degrade rescue swallows. One task
|
|
38
|
+
# does the dance: update the pin (with the CLI git fetch the sandbox
|
|
39
|
+
# flake needs), print the resolved revision, recompile.
|
|
40
|
+
# The kotoshu-rs rev the ext's Cargo.lock pins (nil when unlocked).
|
|
41
|
+
# Shared by ext:update (prints it) and ext:pin_check (compares it to
|
|
42
|
+
# main) so the extraction can never diverge between the two.
|
|
43
|
+
def pinned_rs_rev
|
|
44
|
+
File.expand_path("Cargo.lock", __dir__).then { File.read(_1) }
|
|
45
|
+
.match(/^name = "kotoshu"$.*?^source = "git\+[^"]*#([0-9a-f]+)"/m)&.captures&.first
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
desc "Update the ext's kotoshu-rs pin to latest main and recompile"
|
|
49
|
+
task "ext:update" do
|
|
50
|
+
ext_dir = File.expand_path("ext/kotoshu_native", __dir__)
|
|
51
|
+
Dir.chdir(ext_dir) do
|
|
52
|
+
env = { "CARGO_NET_GIT_FETCH_WITH_CLI" => "true" }
|
|
53
|
+
ok = system(env, "cargo", "update", "-p", "kotoshu")
|
|
54
|
+
abort "ext:update: cargo update failed" unless ok
|
|
55
|
+
end
|
|
56
|
+
rev = pinned_rs_rev
|
|
57
|
+
puts "ext:update: kotoshu-rs pinned at #{rev ? rev[0, 10] : 'main (unpinned rev)'}"
|
|
58
|
+
Rake::Task["compile"].invoke
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
desc "Fail when the ext pins a kotoshu-rs rev behind main (KOTOSHU_RS_HEAD = main's sha)"
|
|
62
|
+
task "ext:pin_check" do
|
|
63
|
+
pinned = pinned_rs_rev
|
|
64
|
+
abort "ext:pin_check: Cargo.lock carries no kotoshu git pin" if pinned.nil?
|
|
65
|
+
head = ENV.fetch("KOTOSHU_RS_HEAD", "").strip
|
|
66
|
+
abort "ext:pin_check: pass kotoshu-rs main's sha via KOTOSHU_RS_HEAD" if head.empty?
|
|
67
|
+
if pinned == head
|
|
68
|
+
puts "ext:pin_check: kotoshu-rs pinned at main (#{pinned[0, 10]})"
|
|
69
|
+
else
|
|
70
|
+
abort <<~MESSAGE
|
|
71
|
+
ext:pin_check: ext pins #{pinned[0, 10]} but kotoshu-rs main is #{head[0, 10]}.
|
|
72
|
+
A stale pin builds without the newest rs bindings - features degrade
|
|
73
|
+
silently (e.g. a missing TypoEngine.matrix falls back to deriving).
|
|
74
|
+
Run `bundle exec rake ext:update`, then send the refreshed Cargo.lock
|
|
75
|
+
through a PR.
|
|
76
|
+
MESSAGE
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
34
80
|
namespace :kotoshu do
|
|
35
81
|
namespace :staged_languages do
|
|
36
82
|
desc <<~DESC
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Plan 135 — Post-train improvements: the two gotchas the release train exposed
|
|
2
|
+
|
|
3
|
+
Status: executed (2026-09-14). (1) `rake ext:update` — cargo update
|
|
4
|
+
with the CLI git fetch, prints the resolved rs revision (verified
|
|
5
|
+
live: c810ced3b4), recompiles; spec freezes the wiring. (2)
|
|
6
|
+
miss-driven registry refresh — `registry_entry_for` retries once with
|
|
7
|
+
a forced registry fetch when the wanted entry is absent from the
|
|
8
|
+
cached copy; offline refresh failure keeps the original miss
|
|
9
|
+
(broadened rescue after the Windows Errno lesson). Proven with a real
|
|
10
|
+
local HTTP server: stale v1.4.0 cache + served v1.7.0 registry
|
|
11
|
+
downloads the fresh pair end to end. Suite 4064/0, rubocop clean.
|
|
12
|
+
|
|
13
|
+
## 1. `rake ext:update` — the rs-main pin's manual dance
|
|
14
|
+
|
|
15
|
+
The gem ext tracks kotoshu-rs main via a git dependency pinned in
|
|
16
|
+
`ext/kotoshu_native/Cargo.lock`. After every rs merge, the developer
|
|
17
|
+
must remember `cargo update -p kotoshu && rake compile` — forgetting
|
|
18
|
+
it loads a stale extension whose missing methods the typo layer's
|
|
19
|
+
silent-degrade rescue swallows (bit twice now). One task does the
|
|
20
|
+
dance: update the pin, print the resolved revision, recompile.
|
|
21
|
+
|
|
22
|
+
## 2. Miss-driven registry refresh — the stale-cache silent degrade
|
|
23
|
+
|
|
24
|
+
A cached registry within its TTL shadows the live one: during the
|
|
25
|
+
train, a v1.4.0 cache made `setup --typo` report `:unavailable`
|
|
26
|
+
against a v1.7.0 registry because the wanted entry was absent from
|
|
27
|
+
the STALE copy. The fix is miss-driven invalidation: when a wanted
|
|
28
|
+
resource is absent from the cached registry, refresh the registry
|
|
29
|
+
once and look again before falling back (legacy CDN for tiers, the
|
|
30
|
+
honest error for the typo pair). Offline refresh failure keeps the
|
|
31
|
+
original miss — existing fallbacks untouched.
|
|
32
|
+
|
|
33
|
+
## Gates
|
|
34
|
+
|
|
35
|
+
- The stale-cache spec: a seeded old registry + a served fresh one
|
|
36
|
+
resolves the entry (fixture-driven, no network); the offline
|
|
37
|
+
variant keeps today's behavior exactly.
|
|
38
|
+
- Suite green; no behavior change for cache hits.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Plan 136 — Prebuilt typo matrices: arming as a download
|
|
2
|
+
|
|
3
|
+
Status: executed (2026-09-14; models PR #39, rs PR, gem PR — see below)
|
|
4
|
+
|
|
5
|
+
The v1.7.0 train's improvement backlog named first-arming time
|
|
6
|
+
(~25 s derivation) as the last user-visible cost of the typo layer.
|
|
7
|
+
The matrix ships as a KTM1 artifact (magic, version, count, dims,
|
|
8
|
+
int8 rows, f32 scales — index-parallel to the language's full-tier
|
|
9
|
+
vocabulary), derived once by `kotoshu-rs examples/matrix_export`.
|
|
10
|
+
|
|
11
|
+
## Shipped
|
|
12
|
+
|
|
13
|
+
- models: `models/en/typo.matrix.en.ktm1` (100k x 256 = 26.0 MB) +
|
|
14
|
+
descriptor + `kotoshu://models/en/typo-matrix` (mirror-only, the
|
|
15
|
+
additive template; validator branch + schema widened; the
|
|
16
|
+
typo-biencoder pair now pins to its own semver tag).
|
|
17
|
+
- rs: `TypoIndex::write_rows` / `parse_ktm1` (fully validated),
|
|
18
|
+
`TypoEngine::from_matrix`, the `matrix_export` example, the Ruby
|
|
19
|
+
binding `Kotoshu::Native::TypoEngine.matrix(model, tier, path)`.
|
|
20
|
+
Round-trip test: artifact slates identical to the derived index.
|
|
21
|
+
- gem: `ModelCache#{load_cached,download}_typo_matrix` (sha-verified
|
|
22
|
+
resolve; setup fetch is best-effort — a language without a matrix
|
|
23
|
+
arms by deriving), `Engine.for` prefers the cached matrix.
|
|
24
|
+
|
|
25
|
+
## Measured
|
|
26
|
+
|
|
27
|
+
- Arming: ~25 s derivation -> 25 ms artifact load (1000x).
|
|
28
|
+
- Suggest: unchanged (2 ms); `recieve -> receive` in the slate.
|
|
29
|
+
- The artifact round-trip is frozen by tests on both sides of the
|
|
30
|
+
boundary; derive-at-load remains the universal fallback.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Plan 137: typo-matrix backfill on satisfied setups
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
Two gaps left plan 136's instant arming unreachable through the CLI:
|
|
8
|
+
|
|
9
|
+
1. **LFS (models repo, fixed by PR #40):** the KTM1 matrix landed as a
|
|
10
|
+
plain git blob — `.gitattributes` had no `*.ktm1` rule — so the
|
|
11
|
+
registry's media-host mirror URL 404'd and `kotoshu setup en --typo`
|
|
12
|
+
could not download the matrix. The raw host served it; the media host
|
|
13
|
+
(LFS-only) did not.
|
|
14
|
+
2. **Backfill (this repo):** `ResourceManager#setup_typo_remote` returned
|
|
15
|
+
`:cached` early when the typo pair + full tier were cached, never
|
|
16
|
+
touching the matrix — and its download path didn't fetch the matrix
|
|
17
|
+
either. Only the library API (`Kotoshu::Typo::Engine.setup_typo`) did.
|
|
18
|
+
An install set up before the matrix existed (or a fresh one, before
|
|
19
|
+
this fix) would derive at arming time (~25-30s) forever.
|
|
20
|
+
|
|
21
|
+
## Changes
|
|
22
|
+
|
|
23
|
+
- `setup_typo_remote`: both paths now call `ensure_typo_matrix(cache, lang)`
|
|
24
|
+
— cache-only stat via `load_cached_typo_matrix`; best-effort fetch
|
|
25
|
+
otherwise (`:downloaded` on success, `nil` → degrade to `:cached` on
|
|
26
|
+
absent entry or network failure — the fallback remains deriving).
|
|
27
|
+
- Specs (`resource_manager_typo_spec.rb`): backfill-on-satisfied-setup
|
|
28
|
+
against a real local HTTP server; stays-`:cached` when the matrix is
|
|
29
|
+
already cached; degrades-`:cached` with no registry entry.
|
|
30
|
+
|
|
31
|
+
## Sister work
|
|
32
|
+
|
|
33
|
+
- models PR #40 (merged): `*.ktm1` LFS rule + renormalization — content
|
|
34
|
+
sha unchanged, registry untouched.
|
|
35
|
+
- models plan 10 / PR #41: `validate_registry.py --check-urls` fetchability
|
|
36
|
+
gate in CI and the release flow, so a dead mirror can never land again.
|
|
37
|
+
|
|
38
|
+
## Evidence (live, 2026-09-14)
|
|
39
|
+
|
|
40
|
+
- Mirror: `media.githubusercontent.com/.../typo.matrix.en.ktm1` → 200,
|
|
41
|
+
KTM1 magic, 26000016 bytes.
|
|
42
|
+
- `kotoshu setup en --typo` (registry wiped, pair+tier cached):
|
|
43
|
+
`typo: downloaded`, matrix cached, 2.1s.
|
|
44
|
+
- Arming: 31.8s (derive, stale-ext baseline) → **588ms** via
|
|
45
|
+
`TypoEngine.matrix` (after `rake ext:update` — the binding ships in
|
|
46
|
+
kotoshu-rs 0.2.0).
|
|
47
|
+
- `engine.suggest("recieve")` → `["recieved", "recieving", "receive"]`
|
|
48
|
+
in 9ms.
|
|
49
|
+
- `bundle exec rspec spec/kotoshu/resource_manager_typo_spec.rb
|
|
50
|
+
spec/kotoshu/cache/model_cache_typo_spec.rb` → 25 examples, 0 failures.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Plan 138: ext pin drift guard + typo-retrieval docs
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
1. **Silent pin drift.** The ext tracks kotoshu-rs main via a git rev in
|
|
8
|
+
the root `Cargo.lock`. Every rs merge moves main ahead while the pin
|
|
9
|
+
stays — and nothing fails: `rake compile` happily builds the old
|
|
10
|
+
extension, and the typo layer's degrade rescue swallows the missing
|
|
11
|
+
surface. Today that cost a live incident: a stale local ext lacked
|
|
12
|
+
`TypoEngine.matrix`, `Engine.for` silently fell back to deriving, and
|
|
13
|
+
arming took **31.8 s instead of 588 ms**. Drift is invisible exactly
|
|
14
|
+
when it matters.
|
|
15
|
+
2. **Undocumented feature.** `kotoshu setup LANG --typo` and
|
|
16
|
+
`KOTOSHU_TYPO_RETRIEVAL` shipped in 1.0.3 with no README section —
|
|
17
|
+
the flag is the only entry point and nobody can find it.
|
|
18
|
+
|
|
19
|
+
## Changes
|
|
20
|
+
|
|
21
|
+
- `Rakefile`: `pinned_rs_rev` helper shared by `ext:update` (one
|
|
22
|
+
extraction, two consumers — the extraction can never diverge) and a
|
|
23
|
+
new `rake ext:pin_check` that compares the pin against
|
|
24
|
+
`KOTOSHU_RS_HEAD` and aborts with the `ext:update`-plus-PR recipe on
|
|
25
|
+
drift (network stays at the workflow boundary).
|
|
26
|
+
- `.github/workflows/ext-pin-drift.yml`: weekly Monday 04:00 UTC +
|
|
27
|
+
`workflow_dispatch`; fetches rs main HEAD via `gh api` and runs the
|
|
28
|
+
check. Failing scheduled runs email the repo owner.
|
|
29
|
+
- `spec/kotoshu/rake_ext_update_spec.rb`: three pin_check specs running
|
|
30
|
+
the REAL task in a real subprocess against the REAL Cargo.lock
|
|
31
|
+
(match passes; behind fails with recipe + pinned sha; absent
|
|
32
|
+
KOTOSHU_RS_HEAD fails honestly).
|
|
33
|
+
- `README.adoc`: `[[typo-retrieval]]` section — what the layer does,
|
|
34
|
+
the measured recall lifts, the `--typo` setup flag, the
|
|
35
|
+
`KOTOSHU_TYPO_RETRIEVAL` opt-in, the byte-identical passthrough when
|
|
36
|
+
off, and the prebuilt-matrix arming story.
|
|
37
|
+
|
|
38
|
+
## Evidence (2026-09-14)
|
|
39
|
+
|
|
40
|
+
- `bundle exec rspec spec/kotoshu/rake_ext_update_spec.rb` — 5
|
|
41
|
+
examples, 0 failures.
|
|
42
|
+
- Live: `rake ext:pin_check KOTOSHU_RS_HEAD=$(gh api …/commits/main
|
|
43
|
+
--jq .sha)` → `kotoshu-rs pinned at main (0001b50056)`.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Plan 139: typo observability — armed_via + setup --list detail
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
1. **Silent arming path.** `Engine.for` preferred the matrix when
|
|
8
|
+
present and derived otherwise, but nothing recorded which path won.
|
|
9
|
+
A stale ext missing `TypoEngine.matrix` (the plan-138 incident)
|
|
10
|
+
looked identical to a successful matrix arm from the outside —
|
|
11
|
+
31.8 s of deriving with no signal. `defined?(…matrix)` was also the
|
|
12
|
+
wrong existence check (use `respond_to?`).
|
|
13
|
+
2. **Opaque setup list.** `kotoshu setup --list` printed language codes
|
|
14
|
+
only. An install with the pair+tier but no matrix (derives at arming)
|
|
15
|
+
looked the same as one with the matrix (instant arming).
|
|
16
|
+
|
|
17
|
+
## Changes
|
|
18
|
+
|
|
19
|
+
- `Typo::Engine#armed_via` → `:matrix` | `:derived`. Set at construction
|
|
20
|
+
so the E2E / live path can assert which arm won. `respond_to?(:matrix)`
|
|
21
|
+
replaces `defined?(…matrix)`.
|
|
22
|
+
- `ResourceManager.setup?(lang, resource: :typo_matrix)` — sha-verified
|
|
23
|
+
matrix presence (same contract as `load_cached_typo_matrix`).
|
|
24
|
+
- `kotoshu setup --list` prints per-resource detail:
|
|
25
|
+
`en: spelling, model, typo, typo-matrix`.
|
|
26
|
+
|
|
27
|
+
## Specs
|
|
28
|
+
|
|
29
|
+
- `resource_manager_typo_spec`: setup? :typo_matrix false/true.
|
|
30
|
+
- `engine_for_spec`: armed path asserts `armed_via == :derived` without
|
|
31
|
+
a matrix; new matrix path (gated on `KOTOSHU_TYPO_E2E` +
|
|
32
|
+
`KOTOSHU_TYPO_MATRIX`) asserts `:matrix`.
|
|
33
|
+
- `cli/end_to_end_spec`: setup --list matches `/en:.*spelling/`.
|
|
34
|
+
|
|
35
|
+
## Evidence
|
|
36
|
+
|
|
37
|
+
- Live: `Engine.for("en").armed_via` → `:matrix`; `Engine.for` with
|
|
38
|
+
matrix moved aside → `:derived`.
|
|
39
|
+
- `kotoshu setup --list` after the plan-12 de setup →
|
|
40
|
+
`de: spelling, model, typo, typo-matrix`.
|
|
@@ -212,7 +212,30 @@ module Kotoshu
|
|
|
212
212
|
# no entry for the pair
|
|
213
213
|
# @raise [Kotoshu::Error] when offline with no cached registry
|
|
214
214
|
def registry_entry_for(language_code, tier, force: false)
|
|
215
|
-
registry(force:
|
|
215
|
+
return registry(force: true)&.find(language_code.to_s, tier.to_s) if force
|
|
216
|
+
|
|
217
|
+
registry&.find(language_code.to_s, tier.to_s) ||
|
|
218
|
+
find_after_registry_refresh(language_code, tier)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Plan 135: a cached registry within its TTL shadows the live
|
|
222
|
+
# one - during the v1.7.0 train, a v1.4.0 cache made a newly
|
|
223
|
+
# published resource invisible until manual deletion. When the
|
|
224
|
+
# wanted entry is absent from the cached copy, refresh the
|
|
225
|
+
# registry ONCE and look again. An offline refresh failure keeps
|
|
226
|
+
# the original miss: the legacy fallbacks (CDN for tiers, the
|
|
227
|
+
# honest no-entry error for the typo pair) apply exactly as
|
|
228
|
+
# before.
|
|
229
|
+
def find_after_registry_refresh(language_code, tier)
|
|
230
|
+
refreshed = begin
|
|
231
|
+
registry(force: true)
|
|
232
|
+
rescue StandardError
|
|
233
|
+
# Offline or transport failure: keep the original miss, the
|
|
234
|
+
# existing fallbacks apply (Windows surfaces raw Errno
|
|
235
|
+
# classes the wrapper does not always raise as Kotoshu::Error).
|
|
236
|
+
nil
|
|
237
|
+
end
|
|
238
|
+
refreshed&.find(language_code.to_s, tier.to_s)
|
|
216
239
|
end
|
|
217
240
|
|
|
218
241
|
# Download and verify a tiered model from the registry.
|
|
@@ -500,6 +523,64 @@ module Kotoshu
|
|
|
500
523
|
result
|
|
501
524
|
end
|
|
502
525
|
|
|
526
|
+
# ---- prebuilt typo matrix (plan 136) ----
|
|
527
|
+
|
|
528
|
+
# Resolve the language's prebuilt KTM1 matrix from the cache
|
|
529
|
+
# only (the resolve half; setup downloads).
|
|
530
|
+
#
|
|
531
|
+
# @param language_code [String]
|
|
532
|
+
# @return [String, nil] path to the cached .ktm1, nil when absent
|
|
533
|
+
def load_cached_typo_matrix(language_code)
|
|
534
|
+
entry = begin
|
|
535
|
+
registry&.find(language_code.to_s, "typo-matrix")
|
|
536
|
+
rescue StandardError
|
|
537
|
+
nil
|
|
538
|
+
end
|
|
539
|
+
return nil unless entry
|
|
540
|
+
|
|
541
|
+
dir = File.join(@cache_path, language_code.to_s, "models", "typo-matrix")
|
|
542
|
+
file = File.join(dir, filename_from_url(entry.urls.mirror))
|
|
543
|
+
return nil unless File.exist?(file) && File.size(file).positive?
|
|
544
|
+
return nil unless Digest::SHA256.file(file).hexdigest == entry.sha256
|
|
545
|
+
|
|
546
|
+
file
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Download the matrix (the setup half). Raises like the other
|
|
550
|
+
# registry downloads when the registry cannot serve it yet.
|
|
551
|
+
#
|
|
552
|
+
# @param language_code [String]
|
|
553
|
+
# @param force [Boolean]
|
|
554
|
+
# @return [String] path to the cached .ktm1
|
|
555
|
+
def download_typo_matrix(language_code, force: false)
|
|
556
|
+
lang = language_code.to_s
|
|
557
|
+
entry = registry(force: force)&.find(lang, "typo-matrix")
|
|
558
|
+
unless entry
|
|
559
|
+
raise Kotoshu::Error,
|
|
560
|
+
"no registry entry for kotoshu://models/#{lang}/typo-matrix (the matrix ships per language; derive-at-load remains the fallback)"
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
dir = File.join(@cache_path, lang, "models", "typo-matrix")
|
|
564
|
+
FileUtils.mkdir_p(dir)
|
|
565
|
+
file = File.join(dir, filename_from_url(entry.urls.mirror))
|
|
566
|
+
used_url = download_primary_or_mirror!(entry, file)
|
|
567
|
+
verify_registry_sha256!(entry, file, "kotoshu://models/#{lang}/typo-matrix", used_url,
|
|
568
|
+
remediation: "Run kotoshu setup #{lang} --typo --force to re-download.")
|
|
569
|
+
|
|
570
|
+
write_metadata(File.join(dir, "metadata.json"),
|
|
571
|
+
"version" => entry.version,
|
|
572
|
+
"url" => used_url,
|
|
573
|
+
"language" => lang,
|
|
574
|
+
"type" => "typo-matrix",
|
|
575
|
+
"file" => File.basename(file),
|
|
576
|
+
"checksum" => entry.sha256,
|
|
577
|
+
"registry_id" => "kotoshu://models/#{lang}/typo-matrix",
|
|
578
|
+
"size_bytes" => entry.size_bytes,
|
|
579
|
+
"cached_at" => Time.now.utc.iso8601,
|
|
580
|
+
"source" => "registry")
|
|
581
|
+
file
|
|
582
|
+
end
|
|
583
|
+
|
|
503
584
|
# Get available model types for a language.
|
|
504
585
|
#
|
|
505
586
|
# @param language_code [String] ISO 639-1 language code
|
data/lib/kotoshu/cli.rb
CHANGED
|
@@ -549,7 +549,20 @@ module Kotoshu
|
|
|
549
549
|
end
|
|
550
550
|
|
|
551
551
|
puts "Set up languages:"
|
|
552
|
-
langs.each
|
|
552
|
+
langs.each do |lang|
|
|
553
|
+
parts = []
|
|
554
|
+
parts << "spelling" if Kotoshu::ResourceManager.setup?(lang)
|
|
555
|
+
parts << "frequency" if Kotoshu::ResourceManager.setup?(lang, resource: :frequency)
|
|
556
|
+
parts << "model" if Kotoshu::ResourceManager.setup?(lang, resource: :model, tier: :any)
|
|
557
|
+
parts << "typo" if Kotoshu::ResourceManager.setup?(lang, resource: :typo)
|
|
558
|
+
# Plan 139: surface the prebuilt matrix so a silent derive
|
|
559
|
+
# path is visible from `kotoshu setup --list`.
|
|
560
|
+
parts << "typo-matrix" if Kotoshu::ResourceManager.setup?(lang, resource: :typo_matrix)
|
|
561
|
+
# languages_setup is looser than setup? (TTL/metadata edge
|
|
562
|
+
# cases); fall back to the bare code when nothing verifies so
|
|
563
|
+
# we never print an empty "lang: " line.
|
|
564
|
+
puts(parts.empty? ? " #{lang}" : " #{lang}: #{parts.join(', ')}")
|
|
565
|
+
end
|
|
553
566
|
end
|
|
554
567
|
|
|
555
568
|
def display_result(result, source, application: nil)
|
|
Binary file
|
|
@@ -264,7 +264,7 @@ module Kotoshu
|
|
|
264
264
|
#
|
|
265
265
|
# @param language [String, Symbol] Language code
|
|
266
266
|
# @param resource [Symbol, nil] :spelling (default), :frequency,
|
|
267
|
-
# :model, or nil for "spelling"
|
|
267
|
+
# :model, :typo, :typo_matrix, or nil for "spelling"
|
|
268
268
|
# @param tier [Symbol, nil] Model tier to check (:full, :fluency,
|
|
269
269
|
# :mini, or :any); only meaningful with resource: :model
|
|
270
270
|
# @return [Boolean]
|
|
@@ -273,6 +273,7 @@ module Kotoshu
|
|
|
273
273
|
# ResourceManager.setup?(:en) # spelling
|
|
274
274
|
# ResourceManager.setup?(:en, resource: :frequency) # Kelly list
|
|
275
275
|
# ResourceManager.setup?(:en, resource: :model, tier: :mini)
|
|
276
|
+
# ResourceManager.setup?(:en, resource: :typo_matrix)
|
|
276
277
|
def setup?(language, resource: nil, tier: nil)
|
|
277
278
|
lang = normalize_language(language)
|
|
278
279
|
case resource&.to_sym
|
|
@@ -286,6 +287,11 @@ module Kotoshu
|
|
|
286
287
|
# tier (the rescore's exactness contract).
|
|
287
288
|
cache = model_cache_for
|
|
288
289
|
cache.typo_biencoder_cached? && cache.available?(cache.tier_resource_id(lang, :full))
|
|
290
|
+
when :typo_matrix
|
|
291
|
+
# Plan 139: the prebuilt KTM1 is a separate, optional half of
|
|
292
|
+
# the typo setup. Absent means arming derives (~25 s); present
|
|
293
|
+
# means arming is a download (ms).
|
|
294
|
+
!model_cache_for.load_cached_typo_matrix(lang).nil?
|
|
289
295
|
when :model
|
|
290
296
|
cache = model_cache_for
|
|
291
297
|
if tier.nil?
|
|
@@ -379,7 +385,9 @@ module Kotoshu
|
|
|
379
385
|
cache = model_cache_for(config: config)
|
|
380
386
|
unless force
|
|
381
387
|
full_id = cache.tier_resource_id(lang, :full)
|
|
382
|
-
|
|
388
|
+
if cache.typo_biencoder_cached? && cache.available?(full_id)
|
|
389
|
+
return ensure_typo_matrix(cache, lang) || :cached
|
|
390
|
+
end
|
|
383
391
|
end
|
|
384
392
|
begin
|
|
385
393
|
cache.download_typo_biencoder(force: force)
|
|
@@ -397,7 +405,23 @@ module Kotoshu
|
|
|
397
405
|
|
|
398
406
|
return :unavailable
|
|
399
407
|
end
|
|
408
|
+
ensure_typo_matrix(cache, lang)
|
|
409
|
+
:downloaded
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# Plan 136: an already-satisfied (or just-satisfied) typo setup
|
|
413
|
+
# still owes the user the instant-arming matrix — without this an
|
|
414
|
+
# install set up before the matrix existed derives at arming time
|
|
415
|
+
# (~25s) forever. Cache-only when present; best-effort fetch
|
|
416
|
+
# otherwise. Returns :downloaded when fetched, nil when present or
|
|
417
|
+
# unservable — callers degrade, never fail, on nil.
|
|
418
|
+
def ensure_typo_matrix(cache, lang)
|
|
419
|
+
return nil if cache.load_cached_typo_matrix(lang)
|
|
420
|
+
|
|
421
|
+
cache.download_typo_matrix(lang)
|
|
400
422
|
:downloaded
|
|
423
|
+
rescue Kotoshu::Error, StandardError
|
|
424
|
+
nil
|
|
401
425
|
end
|
|
402
426
|
|
|
403
427
|
def setup_frequency_remote(lang, force:, strict:, config:)
|
data/lib/kotoshu/typo/engine.rb
CHANGED
|
@@ -27,6 +27,10 @@ module Kotoshu
|
|
|
27
27
|
# @return [String] the language the tier was loaded for
|
|
28
28
|
attr_reader :language
|
|
29
29
|
|
|
30
|
+
# @return [Symbol] :matrix when a prebuilt KTM1 armed the engine,
|
|
31
|
+
# :derived when the index was built at load (plan 139)
|
|
32
|
+
attr_reader :armed_via
|
|
33
|
+
|
|
30
34
|
# Load the engine for a language. Yielded block receives the
|
|
31
35
|
# cache to use (defaults to the configured model cache) — specs
|
|
32
36
|
# inject a stub cache with local artifact paths.
|
|
@@ -46,13 +50,29 @@ module Kotoshu
|
|
|
46
50
|
|
|
47
51
|
native = Kotoshu::Native::TypoModel.load(typo[:onnx_path], typo[:vocab_path])
|
|
48
52
|
native_tier = Kotoshu::Native::TypoTier.load(tier[:model_path], tier[:vocab_path])
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
|
|
53
|
+
|
|
54
|
+
# Plan 136: a cached prebuilt matrix arms in milliseconds
|
|
55
|
+
# (TypoEngine.matrix); without one, derive eagerly with the
|
|
56
|
+
# GVL released (plan 134). Both paths produce identical
|
|
57
|
+
# slates - the artifact round-trip is frozen by the engine's
|
|
58
|
+
# own tests. armed_via records which path won so a silent
|
|
59
|
+
# degrade (stale ext missing .matrix, corrupt artifact) is
|
|
60
|
+
# observable instead of looking like success (plan 139).
|
|
61
|
+
engine =
|
|
62
|
+
if (matrix_path = cache.load_cached_typo_matrix(language)) &&
|
|
63
|
+
Kotoshu::Native::TypoEngine.respond_to?(:matrix)
|
|
64
|
+
begin
|
|
65
|
+
new(Kotoshu::Native::TypoEngine.matrix(native, native_tier, matrix_path),
|
|
66
|
+
native_tier: native_tier, language: language, armed_via: :matrix)
|
|
67
|
+
rescue StandardError
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
unless engine
|
|
72
|
+
engine = new(Kotoshu::Native::TypoEngine.new(native, native_tier),
|
|
73
|
+
native_tier: native_tier, language: language, armed_via: :derived)
|
|
74
|
+
engine.build_index
|
|
75
|
+
end
|
|
56
76
|
engine
|
|
57
77
|
rescue StandardError
|
|
58
78
|
# The opt-in layer never breaks a check: any load failure
|
|
@@ -73,10 +93,12 @@ module Kotoshu
|
|
|
73
93
|
# @param native_tier [Kotoshu::Native::TypoTier] the rescore
|
|
74
94
|
# tier (kept for the eager index build)
|
|
75
95
|
# @param language [String]
|
|
76
|
-
|
|
96
|
+
# @param armed_via [Symbol] :matrix or :derived (plan 139)
|
|
97
|
+
def initialize(native_engine, language:, native_tier: nil, armed_via: :derived)
|
|
77
98
|
@native_engine = native_engine
|
|
78
99
|
@native_tier = native_tier
|
|
79
100
|
@language = language
|
|
101
|
+
@armed_via = armed_via
|
|
80
102
|
end
|
|
81
103
|
|
|
82
104
|
# The rescored slate for a word as a SuggestionSet, source
|
|
@@ -120,6 +142,14 @@ module Kotoshu
|
|
|
120
142
|
cache = Cache::ModelCache.new(cache_path: configuration.cache_path)
|
|
121
143
|
typo = cache.download_typo_biencoder(force: force)
|
|
122
144
|
tier = cache.download_tiered_model(language, tier: :full, force_download: force)
|
|
123
|
-
|
|
145
|
+
# Plan 136: the prebuilt matrix makes arming instant. Best-effort
|
|
146
|
+
# only - a language without one arms by deriving (the fallback),
|
|
147
|
+
# and the fetch must never fail the setup that already succeeded.
|
|
148
|
+
matrix = begin
|
|
149
|
+
cache.download_typo_matrix(language, force: force)
|
|
150
|
+
rescue Kotoshu::Error, StandardError
|
|
151
|
+
nil
|
|
152
|
+
end
|
|
153
|
+
{ typo: typo, tier: tier, matrix: matrix }
|
|
124
154
|
end
|
|
125
155
|
end
|
data/lib/kotoshu/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kotoshu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.5
|
|
5
5
|
platform: aarch64-linux
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -124,6 +124,11 @@ files:
|
|
|
124
124
|
- TODO.impl/132-techniques-page-and-prose.md
|
|
125
125
|
- TODO.impl/133-rust-first-distribution.md
|
|
126
126
|
- TODO.impl/134-post-131-improvements.md
|
|
127
|
+
- TODO.impl/135-post-train-improvements.md
|
|
128
|
+
- TODO.impl/136-prebuilt-typo-matrices.md
|
|
129
|
+
- TODO.impl/137-typo-matrix-backfill.md
|
|
130
|
+
- TODO.impl/138-ext-pin-drift-guard.md
|
|
131
|
+
- TODO.impl/139-typo-observability.md
|
|
127
132
|
- TODO.impl/20-encapsulation-cleanup.md
|
|
128
133
|
- TODO.impl/21-autoload-migration.md
|
|
129
134
|
- TODO.impl/22-url-pin-single-source-of-truth.md
|