kotoshu 1.0.2-aarch64-linux → 1.0.4-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +23 -0
- data/Rakefile +46 -0
- data/TODO.impl/124-owner-residue.md +15 -6
- data/TODO.impl/131-hybrid-productization.md +16 -1
- data/TODO.impl/133-rust-first-distribution.md +23 -1
- data/TODO.impl/134-post-131-improvements.md +67 -0
- data/TODO.impl/135-post-train-improvements.md +38 -0
- data/TODO.impl/136-prebuilt-typo-matrices.md +30 -0
- data/TODO.impl/137-typo-matrix-backfill.md +50 -0
- data/TODO.impl/138-ext-pin-drift-guard.md +43 -0
- data/TODO.impl/139-typo-observability.md +40 -0
- data/TODO.impl/54-cjk-support.md +5 -2
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +29 -0
- data/lib/kotoshu/cache/model_cache.rb +241 -11
- data/lib/kotoshu/cli.rb +27 -2
- data/lib/kotoshu/configuration.rb +17 -2
- data/lib/kotoshu/resource_manager.rb +72 -1
- data/lib/kotoshu/spellchecker.rb +32 -2
- data/lib/kotoshu/suggestions/strategies/typo_retrieval_strategy.rb +55 -0
- data/lib/kotoshu/suggestions/typo_merge.rb +26 -0
- data/lib/kotoshu/suggestions.rb +2 -0
- data/lib/kotoshu/typo/engine.rb +155 -0
- data/lib/kotoshu/typo.rb +9 -0
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +1 -0
- metadata +11 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2a2b34597a309dfb1aa257ae199f334a25de28d9003725d97626748d395afaab
|
|
4
|
+
data.tar.gz: 7aa32123f22526fb680557043791ad643af4c680d093b259c19d6907de628c65
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0ef97520e8d53c55a093fa5a6af96dc02f78a98a13312d9c0aa31afa32b81144dfd7134f6e68ae859d9bd99722bbcc05c055c0c2dca75ea9ece42ce0639a26f1
|
|
7
|
+
data.tar.gz: babb5621f0707019802bab1cadba735fbd31bb6231967978aa61fceaa28c670a9d6d12e2703feeed8eaac22c1b0b1c8bdce0439d9f9df814b7127237b7ff1005
|
data/README.adoc
CHANGED
|
@@ -398,6 +398,29 @@ variable `KOTOSHU_MODEL_TIER`; one of `full`, `fluency`, `mini`):
|
|
|
398
398
|
KOTOSHU_MODEL_TIER=mini kotoshu setup en --model
|
|
399
399
|
----
|
|
400
400
|
|
|
401
|
+
[[typo-retrieval]]
|
|
402
|
+
=== Typo Retrieval (opt-in)
|
|
403
|
+
|
|
404
|
+
The typo-retrieval layer adds a 0.5 MB character bi-encoder to the
|
|
405
|
+
suggestion pipeline: it retrieves the twenty vocabulary entries nearest
|
|
406
|
+
a misspelling's character shape, and the language's FULL tier rescores
|
|
407
|
+
them exactly. On the frozen real-pair benchmarks it lifts top-5 recall
|
|
408
|
+
over the full tier alone (+6.3 points on English, +10 to +29 on de/es/ru).
|
|
409
|
+
|
|
410
|
+
Setup fetches everything with one flag:
|
|
411
|
+
|
|
412
|
+
[source,bash]
|
|
413
|
+
----
|
|
414
|
+
kotoshu setup en --typo
|
|
415
|
+
----
|
|
416
|
+
|
|
417
|
+
The layer arms only when you opt in with `KOTOSHU_TYPO_RETRIEVAL=1`;
|
|
418
|
+
without it, suggestions are byte-identical to the base pipeline. A
|
|
419
|
+
prebuilt 26 MB retrieval matrix is served by the model registry, so
|
|
420
|
+
arming is a download (about half a second) instead of a 25-second
|
|
421
|
+
derivation. The matrix rows pair with the language's full-tier
|
|
422
|
+
vocabulary, so keep the full tier cached (setup fetches it).
|
|
423
|
+
|
|
401
424
|
[[interactive-mode]]
|
|
402
425
|
[[interactive-mode]]
|
|
403
426
|
== Interactive Mode
|
data/Rakefile
CHANGED
|
@@ -31,6 +31,52 @@ report_divergences = lambda do |divergences|
|
|
|
31
31
|
puts "... and #{divergences.size - 10} more" if divergences.size > 10
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Plan 135: the ext tracks kotoshu-rs main via a git dependency pinned
|
|
35
|
+
# in ext/kotoshu_native/Cargo.lock; after every rs merge that pin
|
|
36
|
+
# goes stale and `rake compile` silently loads an old extension whose
|
|
37
|
+
# missing methods the typo layer's degrade rescue swallows. One task
|
|
38
|
+
# does the dance: update the pin (with the CLI git fetch the sandbox
|
|
39
|
+
# flake needs), print the resolved revision, recompile.
|
|
40
|
+
# The kotoshu-rs rev the ext's Cargo.lock pins (nil when unlocked).
|
|
41
|
+
# Shared by ext:update (prints it) and ext:pin_check (compares it to
|
|
42
|
+
# main) so the extraction can never diverge between the two.
|
|
43
|
+
def pinned_rs_rev
|
|
44
|
+
File.expand_path("Cargo.lock", __dir__).then { File.read(_1) }
|
|
45
|
+
.match(/^name = "kotoshu"$.*?^source = "git\+[^"]*#([0-9a-f]+)"/m)&.captures&.first
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
desc "Update the ext's kotoshu-rs pin to latest main and recompile"
|
|
49
|
+
task "ext:update" do
|
|
50
|
+
ext_dir = File.expand_path("ext/kotoshu_native", __dir__)
|
|
51
|
+
Dir.chdir(ext_dir) do
|
|
52
|
+
env = { "CARGO_NET_GIT_FETCH_WITH_CLI" => "true" }
|
|
53
|
+
ok = system(env, "cargo", "update", "-p", "kotoshu")
|
|
54
|
+
abort "ext:update: cargo update failed" unless ok
|
|
55
|
+
end
|
|
56
|
+
rev = pinned_rs_rev
|
|
57
|
+
puts "ext:update: kotoshu-rs pinned at #{rev ? rev[0, 10] : 'main (unpinned rev)'}"
|
|
58
|
+
Rake::Task["compile"].invoke
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
desc "Fail when the ext pins a kotoshu-rs rev behind main (KOTOSHU_RS_HEAD = main's sha)"
|
|
62
|
+
task "ext:pin_check" do
|
|
63
|
+
pinned = pinned_rs_rev
|
|
64
|
+
abort "ext:pin_check: Cargo.lock carries no kotoshu git pin" if pinned.nil?
|
|
65
|
+
head = ENV.fetch("KOTOSHU_RS_HEAD", "").strip
|
|
66
|
+
abort "ext:pin_check: pass kotoshu-rs main's sha via KOTOSHU_RS_HEAD" if head.empty?
|
|
67
|
+
if pinned == head
|
|
68
|
+
puts "ext:pin_check: kotoshu-rs pinned at main (#{pinned[0, 10]})"
|
|
69
|
+
else
|
|
70
|
+
abort <<~MESSAGE
|
|
71
|
+
ext:pin_check: ext pins #{pinned[0, 10]} but kotoshu-rs main is #{head[0, 10]}.
|
|
72
|
+
A stale pin builds without the newest rs bindings - features degrade
|
|
73
|
+
silently (e.g. a missing TypoEngine.matrix falls back to deriving).
|
|
74
|
+
Run `bundle exec rake ext:update`, then send the refreshed Cargo.lock
|
|
75
|
+
through a PR.
|
|
76
|
+
MESSAGE
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
34
80
|
namespace :kotoshu do
|
|
35
81
|
namespace :staged_languages do
|
|
36
82
|
desc <<~DESC
|
|
@@ -4,12 +4,21 @@ Status: standing inventory (updated 2026-09-10)
|
|
|
4
4
|
|
|
5
5
|
- VS Code Marketplace + Open VSX PATs → publish kotoshu-vscode
|
|
6
6
|
(extension is publish-ready; owner: "later").
|
|
7
|
-
- Worktree cleanup —
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
7
|
+
- ~~Worktree cleanup~~ — DONE 2026-09-12 under the owner's "make the
|
|
8
|
+
needed cleanups now": every repo down to its single main worktree,
|
|
9
|
+
merged branches deleted local+remote (see the memory entry).
|
|
10
|
+
- PyPI trusted publisher for kotoshu-native — owner registration
|
|
11
|
+
(project kotoshu-native, owner kotoshu, repo kotoshu-rs, workflow
|
|
12
|
+
release-pypi.yml, env blank), then rerun run 34681280341: the full
|
|
13
|
+
16-wheel matrix + sdist are BUILT and preserved as artifacts; the
|
|
14
|
+
publish step failed only on the missing registration.
|
|
15
|
+
- Typo layer registry cut — owner's three mechanical steps, runbooked
|
|
16
|
+
in models-fasttext-onnx docs/RELEASING.md; the moment the pair
|
|
17
|
+
carries vocab_url, `kotoshu setup LANG --typo` arms the layer.
|
|
18
|
+
- Gem 1.0.3 cut decision — main is ahead of 1.0.2 (opt-in typo layer,
|
|
19
|
+
--typo setup, Windows strict-wrap fix, eager GVL-released build);
|
|
20
|
+
nothing changes for default installs, so the cut timing and number
|
|
21
|
+
are the owner's call.
|
|
13
22
|
- kotoshu-lsp: still 0.1.1 — joins a future train if the owner wants
|
|
14
23
|
LSP on the 1.x line (audit train deliberately excluded it).
|
|
15
24
|
- Docker/kotoshu-ci + kotoshu-go/kotoshu-python floors: verify their
|
|
@@ -1,6 +1,21 @@
|
|
|
1
1
|
# Plan 131 — Productize the hybrid: C-retrieve + fastText-rescore in the engines
|
|
2
2
|
|
|
3
|
-
Status:
|
|
3
|
+
Status: executed on the engine and gem sides (2026-09-12; kotoshu-rs
|
|
4
|
+
PR #38 merged — the pure-Rust bi-encoder at parity cosine 1.00000 vs
|
|
5
|
+
onnxruntime, the int8 retrieval scan at 1.81 ms per suggest, the
|
|
6
|
+
frozen C-benchmark numbers reproduced EXACTLY through the shipped
|
|
7
|
+
engine path: top1 0.118772, top5 0.269829 on en real n=2509; wasm
|
|
8
|
+
loadTypo/typoSuggest + KPK1 typo sections + fp32 full-tier RowStore +
|
|
9
|
+
ruby bindings with a 28-assertion smoke; gem PR #191 — Typo::Engine
|
|
10
|
+
loader, TypoRetrievalStrategy, TypoMerge, KOTOSHU_TYPO_RETRIEVAL
|
|
11
|
+
opt-in with a byte-identical disabled passthrough, 27 specs, full
|
|
12
|
+
suite 4030/0). Gates: all three pre-declared clauses pass (numbers
|
|
13
|
+
within noise = exact on the decision metric; latency 1.8 ms against
|
|
14
|
+
the 10 ms budget; zero default change by construction and by spec).
|
|
15
|
+
REMAINING (owner-coupled): the registry promotion — the typo entry's
|
|
16
|
+
primary URL and vocab_url stay null until the owner cuts a registry
|
|
17
|
+
release carrying them; the server/playground/worker flips follow that
|
|
18
|
+
cut on the semantic-tier pattern
|
|
4
19
|
Depends on: plan 128 (verdict + artifacts), plan 113 (packs — the
|
|
5
20
|
delivery vehicle), registry v1.6.0 (typo-biencoder assets already opt-in)
|
|
6
21
|
|
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
# Plan 133 — Rust-first distribution: precompiled native gems and complete wheels
|
|
2
2
|
|
|
3
|
-
Status:
|
|
3
|
+
Status: executed (2026-09-12) — PR #189 (7-iteration CI chain: the
|
|
4
|
+
MinGW Rust toolchain for the ABI match, bindgen pointed at the
|
|
5
|
+
RubyInstaller devkit's ucrt64 headers through BINDGEN_EXTRA_CLANG_ARGS
|
|
6
|
+
with the windows-gnu target, macos-13 -> macos-15-intel for the retired
|
|
7
|
+
x86_64 macOS runner, and the release race fix so the native legs build
|
|
8
|
+
from the v<version> tag). Released as kotoshu 1.0.2: six artifacts on
|
|
9
|
+
RubyGems (ruby + x86_64-linux, aarch64-linux, arm64-darwin, x86_64-darwin,
|
|
10
|
+
x64-mingw-ucrt), verified by a toolchain-free scratch-GEM_HOME install
|
|
11
|
+
that resolved kotoshu-1.0.2-arm64-darwin, engaged the native engine
|
|
12
|
+
under the auto default, and passed the frozen vectors. Docker: both
|
|
13
|
+
images (docker-kotoshu-ci PR #2, kotoshu-server PR #9) dropped their
|
|
14
|
+
build-tool layers / gained the native-backend build assertion, with the
|
|
15
|
+
guard proving the platform gem resolves inside the slim builder. Site:
|
|
16
|
+
engines section resolution paragraph + the 1.0.2 news entry (092fc03).
|
|
17
|
+
Design deviation: no x86_64-linux-musl platform gem — there is no
|
|
18
|
+
hosted musl Ruby runner and cross needs a musl Ruby; Alpine resolves
|
|
19
|
+
the pure-Ruby gem (source-build of the extension remains possible
|
|
20
|
+
there). Python: the 16-wheel + sdist kotoshu-native 0.1.1 matrix is
|
|
21
|
+
built and smoke-green (run 34681280341) but the PyPI publish is gated
|
|
22
|
+
on the owner registering the trusted publisher (pypi.org, project
|
|
23
|
+
kotoshu-native, repo kotoshu-rs, workflow release-pypi.yml, env blank);
|
|
24
|
+
rerun `gh run rerun 34681280341 --repo kotoshu/kotoshu-rs --failed`
|
|
25
|
+
afterward — the wheel artifacts persist, no rebuild needed.
|
|
4
26
|
Depends on: the 1.0 conformance contract (identical outputs make the
|
|
5
27
|
backend default flip correctness-free); plan 93 (the python wheel matrix)
|
|
6
28
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Plan 134 — Post-131 improvements: the audit's honest gaps
|
|
2
|
+
|
|
3
|
+
Status: executed (2026-09-13). (1) GVL release rs PR #39 — the
|
|
4
|
+
without_gvl helper runs the typo compute through
|
|
5
|
+
rb_thread_call_without_gvl, PROVEN LIVE: a 5 ms ticker thread logged
|
|
6
|
+
4,118 ticks during the 24.8 s / 100k-word index build (a held GVL
|
|
7
|
+
starves it to ~0), suggest answers 2.2 ms after the eager build; gem
|
|
8
|
+
PR #195 makes Engine.for build eagerly. (2) native-suite CI leg (gem
|
|
9
|
+
PR #195): the conformance workflow compiles the extension and runs
|
|
10
|
+
the full suite — NativeBackend can no longer regress matrix-green.
|
|
11
|
+
(3) release verify (gem PR #195): the release asserts all six
|
|
12
|
+
RubyGems artifacts before reporting success. (4) runbook models PR
|
|
13
|
+
#37: docs/RELEASING.md carries the standard cut and the typo
|
|
14
|
+
promotion steps. (5) perf-weekly (gem PR #195): the weekly slow-suite
|
|
15
|
+
run whose first execution caught the suites rotted — repaired with
|
|
16
|
+
dated 2x-headroom bounds. Found and fixed en route (gem PR #196,
|
|
17
|
+
merged): setup_typo_remote rescued only Kotoshu::Error, leaking raw
|
|
18
|
+
Errno::ECONNREFUSED on Windows (caught by merge-preview CI on main).
|
|
19
|
+
|
|
20
|
+
Depends on: plan 131 (the typo layer), plan 133 (the platform gems)
|
|
21
|
+
|
|
22
|
+
## The items
|
|
23
|
+
|
|
24
|
+
### 1. The typo layer's first-suggest stall (production defect class)
|
|
25
|
+
|
|
26
|
+
The 100k-word index build (~15 s) happens lazily inside the first
|
|
27
|
+
`typo_suggest`, and the Ruby binding holds the GVL for the whole Rust
|
|
28
|
+
call — on the server, one arming request blocks every other request
|
|
29
|
+
for the build's duration. Fixes, both:
|
|
30
|
+
|
|
31
|
+
- The binding releases the GVL around the pure-Rust compute (index
|
|
32
|
+
build and scan touch no Ruby objects; results re-enter GVL scope to
|
|
33
|
+
build hashes), so a build never stops the world.
|
|
34
|
+
- The gem resolves EAGERLY: `Engine.for` builds the index at load —
|
|
35
|
+
the cost moves to opt-in setup where it belongs, never mid-check.
|
|
36
|
+
|
|
37
|
+
### 2. The native path is invisible to the main rake matrix
|
|
38
|
+
|
|
39
|
+
The nine rake legs report the native specs PENDING ("native extension
|
|
40
|
+
not built"); only the separate compare job compiles the ext. A
|
|
41
|
+
`NativeBackend` regression could land matrix-green. One dedicated
|
|
42
|
+
matrix job compiles the extension and runs the suite un-pended.
|
|
43
|
+
|
|
44
|
+
### 3. The release workflow trusts its own pushes
|
|
45
|
+
|
|
46
|
+
Nothing asserts the release is COMPLETE — a silently failed platform
|
|
47
|
+
push ships a partial release green. A `verify` job after the native
|
|
48
|
+
legs queries the RubyGems API for version x platform coverage (ruby +
|
|
49
|
+
the five native platforms) and fails on anything missing.
|
|
50
|
+
|
|
51
|
+
### 4. The owner's cut-day procedure lives in session memory
|
|
52
|
+
|
|
53
|
+
The three-step typo release checklist is recorded in memory and PR
|
|
54
|
+
bodies. The models repo carries the committed runbook instead — the
|
|
55
|
+
owner cuts the release without needing this session.
|
|
56
|
+
|
|
57
|
+
### 5. The weekly SLOW_TESTS dashboard, owner-free form
|
|
58
|
+
|
|
59
|
+
The T4.2 remainder as a workflow: weekly run of the slow suites, the
|
|
60
|
+
delta summary posted on the run page (a job summary + artifact — no
|
|
61
|
+
external dashboard to own).
|
|
62
|
+
|
|
63
|
+
## Gates
|
|
64
|
+
|
|
65
|
+
- Every fix ships with its spec or CI proof; no behavior change to
|
|
66
|
+
any frozen output (the GVL release is invisible; eager build moves
|
|
67
|
+
WHEN the same index is built, not what it computes).
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Plan 135 — Post-train improvements: the two gotchas the release train exposed
|
|
2
|
+
|
|
3
|
+
Status: executed (2026-09-14). (1) `rake ext:update` — cargo update
|
|
4
|
+
with the CLI git fetch, prints the resolved rs revision (verified
|
|
5
|
+
live: c810ced3b4), recompiles; spec freezes the wiring. (2)
|
|
6
|
+
miss-driven registry refresh — `registry_entry_for` retries once with
|
|
7
|
+
a forced registry fetch when the wanted entry is absent from the
|
|
8
|
+
cached copy; offline refresh failure keeps the original miss
|
|
9
|
+
(broadened rescue after the Windows Errno lesson). Proven with a real
|
|
10
|
+
local HTTP server: stale v1.4.0 cache + served v1.7.0 registry
|
|
11
|
+
downloads the fresh pair end to end. Suite 4064/0, rubocop clean.
|
|
12
|
+
|
|
13
|
+
## 1. `rake ext:update` — the rs-main pin's manual dance
|
|
14
|
+
|
|
15
|
+
The gem ext tracks kotoshu-rs main via a git dependency pinned in
|
|
16
|
+
`ext/kotoshu_native/Cargo.lock`. After every rs merge, the developer
|
|
17
|
+
must remember `cargo update -p kotoshu && rake compile` — forgetting
|
|
18
|
+
it loads a stale extension whose missing methods the typo layer's
|
|
19
|
+
silent-degrade rescue swallows (bit twice now). One task does the
|
|
20
|
+
dance: update the pin, print the resolved revision, recompile.
|
|
21
|
+
|
|
22
|
+
## 2. Miss-driven registry refresh — the stale-cache silent degrade
|
|
23
|
+
|
|
24
|
+
A cached registry within its TTL shadows the live one: during the
|
|
25
|
+
train, a v1.4.0 cache made `setup --typo` report `:unavailable`
|
|
26
|
+
against a v1.7.0 registry because the wanted entry was absent from
|
|
27
|
+
the STALE copy. The fix is miss-driven invalidation: when a wanted
|
|
28
|
+
resource is absent from the cached registry, refresh the registry
|
|
29
|
+
once and look again before falling back (legacy CDN for tiers, the
|
|
30
|
+
honest error for the typo pair). Offline refresh failure keeps the
|
|
31
|
+
original miss — existing fallbacks untouched.
|
|
32
|
+
|
|
33
|
+
## Gates
|
|
34
|
+
|
|
35
|
+
- The stale-cache spec: a seeded old registry + a served fresh one
|
|
36
|
+
resolves the entry (fixture-driven, no network); the offline
|
|
37
|
+
variant keeps today's behavior exactly.
|
|
38
|
+
- Suite green; no behavior change for cache hits.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Plan 136 — Prebuilt typo matrices: arming as a download
|
|
2
|
+
|
|
3
|
+
Status: executed (2026-09-14; models PR #39, rs PR, gem PR — see below)
|
|
4
|
+
|
|
5
|
+
The v1.7.0 train's improvement backlog named first-arming time
|
|
6
|
+
(~25 s derivation) as the last user-visible cost of the typo layer.
|
|
7
|
+
The matrix ships as a KTM1 artifact (magic, version, count, dims,
|
|
8
|
+
int8 rows, f32 scales — index-parallel to the language's full-tier
|
|
9
|
+
vocabulary), derived once by `kotoshu-rs examples/matrix_export`.
|
|
10
|
+
|
|
11
|
+
## Shipped
|
|
12
|
+
|
|
13
|
+
- models: `models/en/typo.matrix.en.ktm1` (100k x 256 = 26.0 MB) +
|
|
14
|
+
descriptor + `kotoshu://models/en/typo-matrix` (mirror-only, the
|
|
15
|
+
additive template; validator branch + schema widened; the
|
|
16
|
+
typo-biencoder pair now pins to its own semver tag).
|
|
17
|
+
- rs: `TypoIndex::write_rows` / `parse_ktm1` (fully validated),
|
|
18
|
+
`TypoEngine::from_matrix`, the `matrix_export` example, the Ruby
|
|
19
|
+
binding `Kotoshu::Native::TypoEngine.matrix(model, tier, path)`.
|
|
20
|
+
Round-trip test: artifact slates identical to the derived index.
|
|
21
|
+
- gem: `ModelCache#{load_cached,download}_typo_matrix` (sha-verified
|
|
22
|
+
resolve; setup fetch is best-effort — a language without a matrix
|
|
23
|
+
arms by deriving), `Engine.for` prefers the cached matrix.
|
|
24
|
+
|
|
25
|
+
## Measured
|
|
26
|
+
|
|
27
|
+
- Arming: ~25 s derivation -> 25 ms artifact load (1000x).
|
|
28
|
+
- Suggest: unchanged (2 ms); `recieve -> receive` in the slate.
|
|
29
|
+
- The artifact round-trip is frozen by tests on both sides of the
|
|
30
|
+
boundary; derive-at-load remains the universal fallback.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Plan 137: typo-matrix backfill on satisfied setups
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
Two gaps left plan 136's instant arming unreachable through the CLI:
|
|
8
|
+
|
|
9
|
+
1. **LFS (models repo, fixed by PR #40):** the KTM1 matrix landed as a
|
|
10
|
+
plain git blob — `.gitattributes` had no `*.ktm1` rule — so the
|
|
11
|
+
registry's media-host mirror URL 404'd and `kotoshu setup en --typo`
|
|
12
|
+
could not download the matrix. The raw host served it; the media host
|
|
13
|
+
(LFS-only) did not.
|
|
14
|
+
2. **Backfill (this repo):** `ResourceManager#setup_typo_remote` returned
|
|
15
|
+
`:cached` early when the typo pair + full tier were cached, never
|
|
16
|
+
touching the matrix — and its download path didn't fetch the matrix
|
|
17
|
+
either. Only the library API (`Kotoshu::Typo::Engine.setup_typo`) did.
|
|
18
|
+
An install set up before the matrix existed (or a fresh one, before
|
|
19
|
+
this fix) would derive at arming time (~25-30s) forever.
|
|
20
|
+
|
|
21
|
+
## Changes
|
|
22
|
+
|
|
23
|
+
- `setup_typo_remote`: both paths now call `ensure_typo_matrix(cache, lang)`
|
|
24
|
+
— cache-only stat via `load_cached_typo_matrix`; best-effort fetch
|
|
25
|
+
otherwise (`:downloaded` on success, `nil` → degrade to `:cached` on
|
|
26
|
+
absent entry or network failure — the fallback remains deriving).
|
|
27
|
+
- Specs (`resource_manager_typo_spec.rb`): backfill-on-satisfied-setup
|
|
28
|
+
against a real local HTTP server; stays-`:cached` when the matrix is
|
|
29
|
+
already cached; degrades-`:cached` with no registry entry.
|
|
30
|
+
|
|
31
|
+
## Sister work
|
|
32
|
+
|
|
33
|
+
- models PR #40 (merged): `*.ktm1` LFS rule + renormalization — content
|
|
34
|
+
sha unchanged, registry untouched.
|
|
35
|
+
- models plan 10 / PR #41: `validate_registry.py --check-urls` fetchability
|
|
36
|
+
gate in CI and the release flow, so a dead mirror can never land again.
|
|
37
|
+
|
|
38
|
+
## Evidence (live, 2026-09-14)
|
|
39
|
+
|
|
40
|
+
- Mirror: `media.githubusercontent.com/.../typo.matrix.en.ktm1` → 200,
|
|
41
|
+
KTM1 magic, 26000016 bytes.
|
|
42
|
+
- `kotoshu setup en --typo` (registry wiped, pair+tier cached):
|
|
43
|
+
`typo: downloaded`, matrix cached, 2.1s.
|
|
44
|
+
- Arming: 31.8s (derive, stale-ext baseline) → **588ms** via
|
|
45
|
+
`TypoEngine.matrix` (after `rake ext:update` — the binding ships in
|
|
46
|
+
kotoshu-rs 0.2.0).
|
|
47
|
+
- `engine.suggest("recieve")` → `["recieved", "recieving", "receive"]`
|
|
48
|
+
in 9ms.
|
|
49
|
+
- `bundle exec rspec spec/kotoshu/resource_manager_typo_spec.rb
|
|
50
|
+
spec/kotoshu/cache/model_cache_typo_spec.rb` → 25 examples, 0 failures.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Plan 138: ext pin drift guard + typo-retrieval docs
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
1. **Silent pin drift.** The ext tracks kotoshu-rs main via a git rev in
|
|
8
|
+
the root `Cargo.lock`. Every rs merge moves main ahead while the pin
|
|
9
|
+
stays — and nothing fails: `rake compile` happily builds the old
|
|
10
|
+
extension, and the typo layer's degrade rescue swallows the missing
|
|
11
|
+
surface. Today that cost a live incident: a stale local ext lacked
|
|
12
|
+
`TypoEngine.matrix`, `Engine.for` silently fell back to deriving, and
|
|
13
|
+
arming took **31.8 s instead of 588 ms**. Drift is invisible exactly
|
|
14
|
+
when it matters.
|
|
15
|
+
2. **Undocumented feature.** `kotoshu setup LANG --typo` and
|
|
16
|
+
`KOTOSHU_TYPO_RETRIEVAL` shipped in 1.0.3 with no README section —
|
|
17
|
+
the flag is the only entry point and nobody can find it.
|
|
18
|
+
|
|
19
|
+
## Changes
|
|
20
|
+
|
|
21
|
+
- `Rakefile`: `pinned_rs_rev` helper shared by `ext:update` (one
|
|
22
|
+
extraction, two consumers — the extraction can never diverge) and a
|
|
23
|
+
new `rake ext:pin_check` that compares the pin against
|
|
24
|
+
`KOTOSHU_RS_HEAD` and aborts with the `ext:update`-plus-PR recipe on
|
|
25
|
+
drift (network stays at the workflow boundary).
|
|
26
|
+
- `.github/workflows/ext-pin-drift.yml`: weekly Monday 04:00 UTC +
|
|
27
|
+
`workflow_dispatch`; fetches rs main HEAD via `gh api` and runs the
|
|
28
|
+
check. Failing scheduled runs email the repo owner.
|
|
29
|
+
- `spec/kotoshu/rake_ext_update_spec.rb`: three pin_check specs running
|
|
30
|
+
the REAL task in a real subprocess against the REAL Cargo.lock
|
|
31
|
+
(match passes; behind fails with recipe + pinned sha; absent
|
|
32
|
+
KOTOSHU_RS_HEAD fails honestly).
|
|
33
|
+
- `README.adoc`: `[[typo-retrieval]]` section — what the layer does,
|
|
34
|
+
the measured recall lifts, the `--typo` setup flag, the
|
|
35
|
+
`KOTOSHU_TYPO_RETRIEVAL` opt-in, the byte-identical passthrough when
|
|
36
|
+
off, and the prebuilt-matrix arming story.
|
|
37
|
+
|
|
38
|
+
## Evidence (2026-09-14)
|
|
39
|
+
|
|
40
|
+
- `bundle exec rspec spec/kotoshu/rake_ext_update_spec.rb` — 5
|
|
41
|
+
examples, 0 failures.
|
|
42
|
+
- Live: `rake ext:pin_check KOTOSHU_RS_HEAD=$(gh api …/commits/main
|
|
43
|
+
--jq .sha)` → `kotoshu-rs pinned at main (0001b50056)`.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Plan 139: typo observability — armed_via + setup --list detail
|
|
2
|
+
|
|
3
|
+
## Status: executed
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
1. **Silent arming path.** `Engine.for` preferred the matrix when
|
|
8
|
+
present and derived otherwise, but nothing recorded which path won.
|
|
9
|
+
A stale ext missing `TypoEngine.matrix` (the plan-138 incident)
|
|
10
|
+
looked identical to a successful matrix arm from the outside —
|
|
11
|
+
31.8 s of deriving with no signal. `defined?(…matrix)` was also the
|
|
12
|
+
wrong existence check (use `respond_to?`).
|
|
13
|
+
2. **Opaque setup list.** `kotoshu setup --list` printed language codes
|
|
14
|
+
only. An install with the pair+tier but no matrix (derives at arming)
|
|
15
|
+
looked the same as one with the matrix (instant arming).
|
|
16
|
+
|
|
17
|
+
## Changes
|
|
18
|
+
|
|
19
|
+
- `Typo::Engine#armed_via` → `:matrix` | `:derived`. Set at construction
|
|
20
|
+
so the E2E / live path can assert which arm won. `respond_to?(:matrix)`
|
|
21
|
+
replaces `defined?(…matrix)`.
|
|
22
|
+
- `ResourceManager.setup?(lang, resource: :typo_matrix)` — sha-verified
|
|
23
|
+
matrix presence (same contract as `load_cached_typo_matrix`).
|
|
24
|
+
- `kotoshu setup --list` prints per-resource detail:
|
|
25
|
+
`en: spelling, model, typo, typo-matrix`.
|
|
26
|
+
|
|
27
|
+
## Specs
|
|
28
|
+
|
|
29
|
+
- `resource_manager_typo_spec`: setup? :typo_matrix false/true.
|
|
30
|
+
- `engine_for_spec`: armed path asserts `armed_via == :derived` without
|
|
31
|
+
a matrix; new matrix path (gated on `KOTOSHU_TYPO_E2E` +
|
|
32
|
+
`KOTOSHU_TYPO_MATRIX`) asserts `:matrix`.
|
|
33
|
+
- `cli/end_to_end_spec`: setup --list matches `/en:.*spelling/`.
|
|
34
|
+
|
|
35
|
+
## Evidence
|
|
36
|
+
|
|
37
|
+
- Live: `Engine.for("en").armed_via` → `:matrix`; `Engine.for` with
|
|
38
|
+
matrix moved aside → `:derived`.
|
|
39
|
+
- `kotoshu setup --list` after the plan-12 de setup →
|
|
40
|
+
`de: spelling, model, typo, typo-matrix`.
|
data/TODO.impl/54-cjk-support.md
CHANGED
|
@@ -7,8 +7,11 @@ This is the largest tier 3 item: CJK languages need word-segmentation
|
|
|
7
7
|
(no spaces in zh/ja), different dictionary formats, and
|
|
8
8
|
romanization-aware suggestion strategies.
|
|
9
9
|
|
|
10
|
-
Status: outline only
|
|
11
|
-
|
|
10
|
+
Status: outline only, partially superseded. Japanese tokenization
|
|
11
|
+
shipped (the ja module with the suika tokenizer, live since the
|
|
12
|
+
language-modules wave); real zh/ko spell checking remains design
|
|
13
|
+
space — this TODO documents it, and real work still needs upstream
|
|
14
|
+
segmentation library choices and per-language fixture data.
|
|
12
15
|
|
|
13
16
|
## Per-language needs
|
|
14
17
|
|
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
# 56 — T4+ Quality, Architecture, and Ecosystem backlog
|
|
2
2
|
|
|
3
|
+
Status: largely superseded — most items below were promoted and
|
|
4
|
+
executed by the numbered plans; the remaining open ones are marked.
|
|
5
|
+
Snapshot annotations (2026-09-13): T4.1 shipped as the plan-147/148
|
|
6
|
+
perf waves (WordIndex + SweepIndex: suggest 20-140 s -> sub-second,
|
|
7
|
+
sweep 1472 -> 661 ms en; the Rust engine then took the sweep 10-15x
|
|
8
|
+
further, and plan 131's hybrid retrieval added 1.8 ms retrieval) with
|
|
9
|
+
published error budgets on /performance. T4.4 shipped as plan 97 (YARD
|
|
10
|
+
100% public surface, docs CI, hosted reference) plus the repeated
|
|
11
|
+
README truth passes. T5.2 and T5.3 shipped (Configuration as data
|
|
12
|
+
with the Resolver; WordResult/DocumentResult/SuggestionSet/Suggestion
|
|
13
|
+
on lutaml-model, hand-rolled to_h banned by the global rule). T6.1
|
|
14
|
+
shipped (kotoshu-lsp, live on RubyGems) and T6.2 shipped (the wasm
|
|
15
|
+
playground, @kotoshu/wasm, language packs). Still open: T4.2's weekly
|
|
16
|
+
SLOW_TESTS dashboard, T5.4 (= plan 53), and T6.3 (desktop dictionary
|
|
17
|
+
editor — future). CLOSED SINCE: T4.3's named properties shipped
|
|
18
|
+
(spec/properties/data_structure_properties_spec.rb — Trie
|
|
19
|
+
order-independence and totality, prefix agreement, set recovery, and
|
|
20
|
+
the every-suggestion-is-a-dictionary-word invariant; the affix
|
|
21
|
+
round-trip property remains the one open sub-item, gated on a
|
|
22
|
+
morphological API surface). T5.1 audited clean (2026-09-13): the
|
|
23
|
+
dedupe-once merge CompositeStrategy already performs is exactly the
|
|
24
|
+
fix T5.1 proposed (the code cites it); strategy purity holds with one
|
|
25
|
+
documented exception — the semantic-cascade threshold falls back to
|
|
26
|
+
the process Configuration when the composite was not given an
|
|
27
|
+
explicit one, by design and overridable; and the symspell/edit-1
|
|
28
|
+
overlap is moot — neither is wired into any default flow (the frozen
|
|
29
|
+
Generator owns the real path; the strategy classes are the composable
|
|
30
|
+
API, and the single-set merge makes overlapping compositions safe).
|
|
31
|
+
|
|
3
32
|
This file rolls up the T4 (Quality), T5 (Architecture), and T6
|
|
4
33
|
(Ecosystem) items from TODO.impl/39-tier3-and-beyond.md into
|
|
5
34
|
promotable units with concrete acceptance criteria.
|