kotoshu 1.0.3-aarch64-linux → 1.0.4-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ee5adc450313e1bcb119db5e2241c83166cec21650e84fab745d2f118763983c
4
- data.tar.gz: b2439602b868c0ac5851d502ef08bdb01e1a78883beeceec27511fe4ad996609
3
+ metadata.gz: 2a2b34597a309dfb1aa257ae199f334a25de28d9003725d97626748d395afaab
4
+ data.tar.gz: 7aa32123f22526fb680557043791ad643af4c680d093b259c19d6907de628c65
5
5
  SHA512:
6
- metadata.gz: 82f9569d119b24e47516dcf5fd3e5e102c3742c77c9ecafced3908303e6e0b30ba3a1b95589a1a31411e1dec15aabb2248c28fb52912478038268e40b5d87194
7
- data.tar.gz: bb21d49ce30ae56f90309b4eb90c83aa6ccbc7f5e51d38d99181a12a9d00b363b735f91476aad454504d2505a15b052a909c2f3bf80feae96f10edc2a23f78c5
6
+ metadata.gz: 0ef97520e8d53c55a093fa5a6af96dc02f78a98a13312d9c0aa31afa32b81144dfd7134f6e68ae859d9bd99722bbcc05c055c0c2dca75ea9ece42ce0639a26f1
7
+ data.tar.gz: babb5621f0707019802bab1cadba735fbd31bb6231967978aa61fceaa28c670a9d6d12e2703feeed8eaac22c1b0b1c8bdce0439d9f9df814b7127237b7ff1005
data/README.adoc CHANGED
@@ -398,6 +398,29 @@ variable `KOTOSHU_MODEL_TIER`; one of `full`, `fluency`, `mini`):
398
398
  KOTOSHU_MODEL_TIER=mini kotoshu setup en --model
399
399
  ----
400
400
 
401
+ [[typo-retrieval]]
402
+ === Typo Retrieval (opt-in)
403
+
404
+ The typo-retrieval layer adds a 0.5 MB character bi-encoder to the
405
+ suggestion pipeline: it retrieves the twenty vocabulary entries nearest
406
+ a misspelling's character shape, and the language's FULL tier rescores
407
+ them exactly. On the frozen real-pair benchmarks it lifts top-5 recall
408
+ over the full tier alone (+6.3 points on English, +10 to +29 on de/es/ru).
409
+
410
+ Setup fetches everything with one flag:
411
+
412
+ [source,bash]
413
+ ----
414
+ kotoshu setup en --typo
415
+ ----
416
+
417
+ The layer arms only when you opt in with `KOTOSHU_TYPO_RETRIEVAL=1`;
418
+ without it, suggestions are byte-identical to the base pipeline. A
419
+ prebuilt 26 MB retrieval matrix is served by the model registry, so
420
+ arming is a download (about half a second) instead of a 25-second
421
+ derivation. The matrix rows pair with the language's full-tier
422
+ vocabulary, so keep the full tier cached (setup fetches it).
423
+
401
424
  [[interactive-mode]]
402
425
  [[interactive-mode]]
403
426
  == Interactive Mode
data/Rakefile CHANGED
@@ -31,6 +31,52 @@ report_divergences = lambda do |divergences|
31
31
  puts "... and #{divergences.size - 10} more" if divergences.size > 10
32
32
  end
33
33
 
34
+ # Plan 135: the ext tracks kotoshu-rs main via a git dependency pinned
35
+ # in ext/kotoshu_native/Cargo.lock; after every rs merge that pin
36
+ # goes stale and `rake compile` silently loads an old extension whose
37
+ # missing methods the typo layer's degrade rescue swallows. One task
38
+ # does the dance: update the pin (with the CLI git fetch the sandbox
39
+ # flake needs), print the resolved revision, recompile.
40
+ # The kotoshu-rs rev the ext's Cargo.lock pins (nil when unlocked).
41
+ # Shared by ext:update (prints it) and ext:pin_check (compares it to
42
+ # main) so the extraction can never diverge between the two.
43
+ def pinned_rs_rev
44
+ File.expand_path("Cargo.lock", __dir__).then { File.read(_1) }
45
+ .match(/^name = "kotoshu"$.*?^source = "git\+[^"]*#([0-9a-f]+)"/m)&.captures&.first
46
+ end
47
+
48
+ desc "Update the ext's kotoshu-rs pin to latest main and recompile"
49
+ task "ext:update" do
50
+ ext_dir = File.expand_path("ext/kotoshu_native", __dir__)
51
+ Dir.chdir(ext_dir) do
52
+ env = { "CARGO_NET_GIT_FETCH_WITH_CLI" => "true" }
53
+ ok = system(env, "cargo", "update", "-p", "kotoshu")
54
+ abort "ext:update: cargo update failed" unless ok
55
+ end
56
+ rev = pinned_rs_rev
57
+ puts "ext:update: kotoshu-rs pinned at #{rev ? rev[0, 10] : 'main (unpinned rev)'}"
58
+ Rake::Task["compile"].invoke
59
+ end
60
+
61
+ desc "Fail when the ext pins a kotoshu-rs rev behind main (KOTOSHU_RS_HEAD = main's sha)"
62
+ task "ext:pin_check" do
63
+ pinned = pinned_rs_rev
64
+ abort "ext:pin_check: Cargo.lock carries no kotoshu git pin" if pinned.nil?
65
+ head = ENV.fetch("KOTOSHU_RS_HEAD", "").strip
66
+ abort "ext:pin_check: pass kotoshu-rs main's sha via KOTOSHU_RS_HEAD" if head.empty?
67
+ if pinned == head
68
+ puts "ext:pin_check: kotoshu-rs pinned at main (#{pinned[0, 10]})"
69
+ else
70
+ abort <<~MESSAGE
71
+ ext:pin_check: ext pins #{pinned[0, 10]} but kotoshu-rs main is #{head[0, 10]}.
72
+ A stale pin builds without the newest rs bindings - features degrade
73
+ silently (e.g. a missing TypoEngine.matrix falls back to deriving).
74
+ Run `bundle exec rake ext:update`, then send the refreshed Cargo.lock
75
+ through a PR.
76
+ MESSAGE
77
+ end
78
+ end
79
+
34
80
  namespace :kotoshu do
35
81
  namespace :staged_languages do
36
82
  desc <<~DESC
@@ -0,0 +1,38 @@
1
+ # Plan 135 — Post-train improvements: the two gotchas the release train exposed
2
+
3
+ Status: executed (2026-09-14). (1) `rake ext:update` — cargo update
4
+ with the CLI git fetch, prints the resolved rs revision (verified
5
+ live: c810ced3b4), recompiles; spec freezes the wiring. (2)
6
+ miss-driven registry refresh — `registry_entry_for` retries once with
7
+ a forced registry fetch when the wanted entry is absent from the
8
+ cached copy; offline refresh failure keeps the original miss
9
+ (broadened rescue after the Windows Errno lesson). Proven with a real
10
+ local HTTP server: stale v1.4.0 cache + served v1.7.0 registry
11
+ downloads the fresh pair end to end. Suite 4064/0, rubocop clean.
12
+
13
+ ## 1. `rake ext:update` — the rs-main pin's manual dance
14
+
15
+ The gem ext tracks kotoshu-rs main via a git dependency pinned in
16
+ `ext/kotoshu_native/Cargo.lock`. After every rs merge, the developer
17
+ must remember `cargo update -p kotoshu && rake compile` — forgetting
18
+ it loads a stale extension whose missing methods the typo layer's
19
+ silent-degrade rescue swallows (bit twice now). One task does the
20
+ dance: update the pin, print the resolved revision, recompile.
21
+
22
+ ## 2. Miss-driven registry refresh — the stale-cache silent degrade
23
+
24
+ A cached registry within its TTL shadows the live one: during the
25
+ train, a v1.4.0 cache made `setup --typo` report `:unavailable`
26
+ against a v1.7.0 registry because the wanted entry was absent from
27
+ the STALE copy. The fix is miss-driven invalidation: when a wanted
28
+ resource is absent from the cached registry, refresh the registry
29
+ once and look again before falling back (legacy CDN for tiers, the
30
+ honest error for the typo pair). Offline refresh failure keeps the
31
+ original miss — existing fallbacks untouched.
32
+
33
+ ## Gates
34
+
35
+ - The stale-cache spec: a seeded old registry + a served fresh one
36
+ resolves the entry (fixture-driven, no network); the offline
37
+ variant keeps today's behavior exactly.
38
+ - Suite green; no behavior change for cache hits.
@@ -0,0 +1,30 @@
1
+ # Plan 136 — Prebuilt typo matrices: arming as a download
2
+
3
+ Status: executed (2026-09-14; models PR #39, rs PR, gem PR — see below)
4
+
5
+ The v1.7.0 train's improvement backlog named first-arming time
6
+ (~25 s derivation) as the last user-visible cost of the typo layer.
7
+ The matrix ships as a KTM1 artifact (magic, version, count, dims,
8
+ int8 rows, f32 scales — index-parallel to the language's full-tier
9
+ vocabulary), derived once by `kotoshu-rs examples/matrix_export`.
10
+
11
+ ## Shipped
12
+
13
+ - models: `models/en/typo.matrix.en.ktm1` (100k x 256 = 26.0 MB) +
14
+ descriptor + `kotoshu://models/en/typo-matrix` (mirror-only, the
15
+ additive template; validator branch + schema widened; the
16
+ typo-biencoder pair now pins to its own semver tag).
17
+ - rs: `TypoIndex::write_rows` / `parse_ktm1` (fully validated),
18
+ `TypoEngine::from_matrix`, the `matrix_export` example, the Ruby
19
+ binding `Kotoshu::Native::TypoEngine.matrix(model, tier, path)`.
20
+ Round-trip test: artifact slates identical to the derived index.
21
+ - gem: `ModelCache#{load_cached,download}_typo_matrix` (sha-verified
22
+ resolve; setup fetch is best-effort — a language without a matrix
23
+ arms by deriving), `Engine.for` prefers the cached matrix.
24
+
25
+ ## Measured
26
+
27
+ - Arming: ~25 s derivation -> 25 ms artifact load (1000x).
28
+ - Suggest: unchanged (2 ms); `recieve -> receive` in the slate.
29
+ - The artifact round-trip is frozen by tests on both sides of the
30
+ boundary; derive-at-load remains the universal fallback.
@@ -0,0 +1,50 @@
1
+ # Plan 137: typo-matrix backfill on satisfied setups
2
+
3
+ ## Status: executed
4
+
5
+ ## Problem
6
+
7
+ Two gaps left plan 136's instant arming unreachable through the CLI:
8
+
9
+ 1. **LFS (models repo, fixed by PR #40):** the KTM1 matrix landed as a
10
+ plain git blob — `.gitattributes` had no `*.ktm1` rule — so the
11
+ registry's media-host mirror URL 404'd and `kotoshu setup en --typo`
12
+ could not download the matrix. The raw host served it; the media host
13
+ (LFS-only) did not.
14
+ 2. **Backfill (this repo):** `ResourceManager#setup_typo_remote` returned
15
+ `:cached` early when the typo pair + full tier were cached, never
16
+ touching the matrix — and its download path didn't fetch the matrix
17
+ either. Only the library API (`Kotoshu::Typo::Engine.setup_typo`) did.
18
+ An install set up before the matrix existed (or a fresh one, before
19
+ this fix) would derive at arming time (~25-30s) forever.
20
+
21
+ ## Changes
22
+
23
+ - `setup_typo_remote`: both paths now call `ensure_typo_matrix(cache, lang)`
24
+ — cache-only stat via `load_cached_typo_matrix`; best-effort fetch
25
+ otherwise (`:downloaded` on success, `nil` → degrade to `:cached` on
26
+ absent entry or network failure — the fallback remains deriving).
27
+ - Specs (`resource_manager_typo_spec.rb`): backfill-on-satisfied-setup
28
+ against a real local HTTP server; stays-`:cached` when the matrix is
29
+ already cached; degrades-`:cached` with no registry entry.
30
+
31
+ ## Sister work
32
+
33
+ - models PR #40 (merged): `*.ktm1` LFS rule + renormalization — content
34
+ sha unchanged, registry untouched.
35
+ - models plan 10 / PR #41: `validate_registry.py --check-urls` fetchability
36
+ gate in CI and the release flow, so a dead mirror can never land again.
37
+
38
+ ## Evidence (live, 2026-09-14)
39
+
40
+ - Mirror: `media.githubusercontent.com/.../typo.matrix.en.ktm1` → 200,
41
+ KTM1 magic, 26000016 bytes.
42
+ - `kotoshu setup en --typo` (registry wiped, pair+tier cached):
43
+ `typo: downloaded`, matrix cached, 2.1s.
44
+ - Arming: 31.8s (derive, stale-ext baseline) → **588ms** via
45
+ `TypoEngine.matrix` (after `rake ext:update` — the binding ships in
46
+ kotoshu-rs 0.2.0).
47
+ - `engine.suggest("recieve")` → `["recieved", "recieving", "receive"]`
48
+ in 9ms.
49
+ - `bundle exec rspec spec/kotoshu/resource_manager_typo_spec.rb
50
+ spec/kotoshu/cache/model_cache_typo_spec.rb` → 25 examples, 0 failures.
@@ -0,0 +1,43 @@
1
+ # Plan 138: ext pin drift guard + typo-retrieval docs
2
+
3
+ ## Status: executed
4
+
5
+ ## Problem
6
+
7
+ 1. **Silent pin drift.** The ext tracks kotoshu-rs main via a git rev in
8
+ the root `Cargo.lock`. Every rs merge moves main ahead while the pin
9
+ stays — and nothing fails: `rake compile` happily builds the old
10
+ extension, and the typo layer's degrade rescue swallows the missing
11
+ surface. Today that cost a live incident: a stale local ext lacked
12
+ `TypoEngine.matrix`, `Engine.for` silently fell back to deriving, and
13
+ arming took **31.8 s instead of 588 ms**. Drift is invisible exactly
14
+ when it matters.
15
+ 2. **Undocumented feature.** `kotoshu setup LANG --typo` and
16
+ `KOTOSHU_TYPO_RETRIEVAL` shipped in 1.0.3 with no README section —
17
+ the flag is the only entry point and nobody can find it.
18
+
19
+ ## Changes
20
+
21
+ - `Rakefile`: `pinned_rs_rev` helper shared by `ext:update` (one
22
+ extraction, two consumers — the extraction can never diverge) and a
23
+ new `rake ext:pin_check` that compares the pin against
24
+ `KOTOSHU_RS_HEAD` and aborts with the `ext:update`-plus-PR recipe on
25
+ drift (network stays at the workflow boundary).
26
+ - `.github/workflows/ext-pin-drift.yml`: weekly Monday 04:00 UTC +
27
+ `workflow_dispatch`; fetches rs main HEAD via `gh api` and runs the
28
+ check. Failing scheduled runs email the repo owner.
29
+ - `spec/kotoshu/rake_ext_update_spec.rb`: three pin_check specs running
30
+ the REAL task in a real subprocess against the REAL Cargo.lock
31
+ (match passes; behind fails with recipe + pinned sha; absent
32
+ KOTOSHU_RS_HEAD fails honestly).
33
+ - `README.adoc`: `[[typo-retrieval]]` section — what the layer does,
34
+ the measured recall lifts, the `--typo` setup flag, the
35
+ `KOTOSHU_TYPO_RETRIEVAL` opt-in, the byte-identical passthrough when
36
+ off, and the prebuilt-matrix arming story.
37
+
38
+ ## Evidence (2026-09-14)
39
+
40
+ - `bundle exec rspec spec/kotoshu/rake_ext_update_spec.rb` — 5
41
+ examples, 0 failures.
42
+ - Live: `rake ext:pin_check KOTOSHU_RS_HEAD=$(gh api …/commits/main
43
+ --jq .sha)` → `kotoshu-rs pinned at main (0001b50056)`.
@@ -0,0 +1,40 @@
1
+ # Plan 139: typo observability — armed_via + setup --list detail
2
+
3
+ ## Status: executed
4
+
5
+ ## Problem
6
+
7
+ 1. **Silent arming path.** `Engine.for` preferred the matrix when
8
+ present and derived otherwise, but nothing recorded which path won.
9
+ A stale ext missing `TypoEngine.matrix` (the plan-138 incident)
10
+ looked identical to a successful matrix arm from the outside —
11
+ 31.8 s of deriving with no signal. `defined?(…matrix)` was also the
12
+ wrong existence check (use `respond_to?`).
13
+ 2. **Opaque setup list.** `kotoshu setup --list` printed language codes
14
+ only. An install with the pair+tier but no matrix (derives at arming)
15
+ looked the same as one with the matrix (instant arming).
16
+
17
+ ## Changes
18
+
19
+ - `Typo::Engine#armed_via` → `:matrix` | `:derived`. Set at construction
20
+ so the E2E / live path can assert which arm won. `respond_to?(:matrix)`
21
+ replaces `defined?(…matrix)`.
22
+ - `ResourceManager.setup?(lang, resource: :typo_matrix)` — sha-verified
23
+ matrix presence (same contract as `load_cached_typo_matrix`).
24
+ - `kotoshu setup --list` prints per-resource detail:
25
+ `en: spelling, model, typo, typo-matrix`.
26
+
27
+ ## Specs
28
+
29
+ - `resource_manager_typo_spec`: setup? :typo_matrix false/true.
30
+ - `engine_for_spec`: armed path asserts `armed_via == :derived` without
31
+ a matrix; new matrix path (gated on `KOTOSHU_TYPO_E2E` +
32
+ `KOTOSHU_TYPO_MATRIX`) asserts `:matrix`.
33
+ - `cli/end_to_end_spec`: setup --list matches `/en:.*spelling/`.
34
+
35
+ ## Evidence
36
+
37
+ - Live: `Engine.for("en").armed_via` → `:matrix`; `Engine.for` with
38
+ matrix moved aside → `:derived`.
39
+ - `kotoshu setup --list` after the plan-12 de setup →
40
+ `de: spelling, model, typo, typo-matrix`.
@@ -212,7 +212,30 @@ module Kotoshu
212
212
  # no entry for the pair
213
213
  # @raise [Kotoshu::Error] when offline with no cached registry
214
214
  def registry_entry_for(language_code, tier, force: false)
215
- registry(force: force)&.find(language_code.to_s, tier.to_s)
215
+ return registry(force: true)&.find(language_code.to_s, tier.to_s) if force
216
+
217
+ registry&.find(language_code.to_s, tier.to_s) ||
218
+ find_after_registry_refresh(language_code, tier)
219
+ end
220
+
221
+ # Plan 135: a cached registry within its TTL shadows the live
222
+ # one - during the v1.7.0 train, a v1.4.0 cache made a newly
223
+ # published resource invisible until manual deletion. When the
224
+ # wanted entry is absent from the cached copy, refresh the
225
+ # registry ONCE and look again. An offline refresh failure keeps
226
+ # the original miss: the legacy fallbacks (CDN for tiers, the
227
+ # honest no-entry error for the typo pair) apply exactly as
228
+ # before.
229
+ def find_after_registry_refresh(language_code, tier)
230
+ refreshed = begin
231
+ registry(force: true)
232
+ rescue StandardError
233
+ # Offline or transport failure: keep the original miss, the
234
+ # existing fallbacks apply (Windows surfaces raw Errno
235
+ # classes the wrapper does not always raise as Kotoshu::Error).
236
+ nil
237
+ end
238
+ refreshed&.find(language_code.to_s, tier.to_s)
216
239
  end
217
240
 
218
241
  # Download and verify a tiered model from the registry.
@@ -500,6 +523,64 @@ module Kotoshu
500
523
  result
501
524
  end
502
525
 
526
+ # ---- prebuilt typo matrix (plan 136) ----
527
+
528
+ # Resolve the language's prebuilt KTM1 matrix from the cache
529
+ # only (the resolve half; setup downloads).
530
+ #
531
+ # @param language_code [String]
532
+ # @return [String, nil] path to the cached .ktm1, nil when absent
533
+ def load_cached_typo_matrix(language_code)
534
+ entry = begin
535
+ registry&.find(language_code.to_s, "typo-matrix")
536
+ rescue StandardError
537
+ nil
538
+ end
539
+ return nil unless entry
540
+
541
+ dir = File.join(@cache_path, language_code.to_s, "models", "typo-matrix")
542
+ file = File.join(dir, filename_from_url(entry.urls.mirror))
543
+ return nil unless File.exist?(file) && File.size(file).positive?
544
+ return nil unless Digest::SHA256.file(file).hexdigest == entry.sha256
545
+
546
+ file
547
+ end
548
+
549
+ # Download the matrix (the setup half). Raises like the other
550
+ # registry downloads when the registry cannot serve it yet.
551
+ #
552
+ # @param language_code [String]
553
+ # @param force [Boolean]
554
+ # @return [String] path to the cached .ktm1
555
+ def download_typo_matrix(language_code, force: false)
556
+ lang = language_code.to_s
557
+ entry = registry(force: force)&.find(lang, "typo-matrix")
558
+ unless entry
559
+ raise Kotoshu::Error,
560
+ "no registry entry for kotoshu://models/#{lang}/typo-matrix (the matrix ships per language; derive-at-load remains the fallback)"
561
+ end
562
+
563
+ dir = File.join(@cache_path, lang, "models", "typo-matrix")
564
+ FileUtils.mkdir_p(dir)
565
+ file = File.join(dir, filename_from_url(entry.urls.mirror))
566
+ used_url = download_primary_or_mirror!(entry, file)
567
+ verify_registry_sha256!(entry, file, "kotoshu://models/#{lang}/typo-matrix", used_url,
568
+ remediation: "Run kotoshu setup #{lang} --typo --force to re-download.")
569
+
570
+ write_metadata(File.join(dir, "metadata.json"),
571
+ "version" => entry.version,
572
+ "url" => used_url,
573
+ "language" => lang,
574
+ "type" => "typo-matrix",
575
+ "file" => File.basename(file),
576
+ "checksum" => entry.sha256,
577
+ "registry_id" => "kotoshu://models/#{lang}/typo-matrix",
578
+ "size_bytes" => entry.size_bytes,
579
+ "cached_at" => Time.now.utc.iso8601,
580
+ "source" => "registry")
581
+ file
582
+ end
583
+
503
584
  # Get available model types for a language.
504
585
  #
505
586
  # @param language_code [String] ISO 639-1 language code
data/lib/kotoshu/cli.rb CHANGED
@@ -549,7 +549,20 @@ module Kotoshu
549
549
  end
550
550
 
551
551
  puts "Set up languages:"
552
- langs.each { |lang| puts " #{lang}" }
552
+ langs.each do |lang|
553
+ parts = []
554
+ parts << "spelling" if Kotoshu::ResourceManager.setup?(lang)
555
+ parts << "frequency" if Kotoshu::ResourceManager.setup?(lang, resource: :frequency)
556
+ parts << "model" if Kotoshu::ResourceManager.setup?(lang, resource: :model, tier: :any)
557
+ parts << "typo" if Kotoshu::ResourceManager.setup?(lang, resource: :typo)
558
+ # Plan 139: surface the prebuilt matrix so a silent derive
559
+ # path is visible from `kotoshu setup --list`.
560
+ parts << "typo-matrix" if Kotoshu::ResourceManager.setup?(lang, resource: :typo_matrix)
561
+ # languages_setup is looser than setup? (TTL/metadata edge
562
+ # cases); fall back to the bare code when nothing verifies so
563
+ # we never print an empty "lang: " line.
564
+ puts(parts.empty? ? " #{lang}" : " #{lang}: #{parts.join(', ')}")
565
+ end
553
566
  end
554
567
 
555
568
  def display_result(result, source, application: nil)
@@ -264,7 +264,7 @@ module Kotoshu
264
264
  #
265
265
  # @param language [String, Symbol] Language code
266
266
  # @param resource [Symbol, nil] :spelling (default), :frequency,
267
- # :model, or nil for "spelling"
267
+ # :model, :typo, :typo_matrix, or nil for "spelling"
268
268
  # @param tier [Symbol, nil] Model tier to check (:full, :fluency,
269
269
  # :mini, or :any); only meaningful with resource: :model
270
270
  # @return [Boolean]
@@ -273,6 +273,7 @@ module Kotoshu
273
273
  # ResourceManager.setup?(:en) # spelling
274
274
  # ResourceManager.setup?(:en, resource: :frequency) # Kelly list
275
275
  # ResourceManager.setup?(:en, resource: :model, tier: :mini)
276
+ # ResourceManager.setup?(:en, resource: :typo_matrix)
276
277
  def setup?(language, resource: nil, tier: nil)
277
278
  lang = normalize_language(language)
278
279
  case resource&.to_sym
@@ -286,6 +287,11 @@ module Kotoshu
286
287
  # tier (the rescore's exactness contract).
287
288
  cache = model_cache_for
288
289
  cache.typo_biencoder_cached? && cache.available?(cache.tier_resource_id(lang, :full))
290
+ when :typo_matrix
291
+ # Plan 139: the prebuilt KTM1 is a separate, optional half of
292
+ # the typo setup. Absent means arming derives (~25 s); present
293
+ # means arming is a download (ms).
294
+ !model_cache_for.load_cached_typo_matrix(lang).nil?
289
295
  when :model
290
296
  cache = model_cache_for
291
297
  if tier.nil?
@@ -379,7 +385,9 @@ module Kotoshu
379
385
  cache = model_cache_for(config: config)
380
386
  unless force
381
387
  full_id = cache.tier_resource_id(lang, :full)
382
- return :cached if cache.typo_biencoder_cached? && cache.available?(full_id)
388
+ if cache.typo_biencoder_cached? && cache.available?(full_id)
389
+ return ensure_typo_matrix(cache, lang) || :cached
390
+ end
383
391
  end
384
392
  begin
385
393
  cache.download_typo_biencoder(force: force)
@@ -397,7 +405,23 @@ module Kotoshu
397
405
 
398
406
  return :unavailable
399
407
  end
408
+ ensure_typo_matrix(cache, lang)
409
+ :downloaded
410
+ end
411
+
412
+ # Plan 136: an already-satisfied (or just-satisfied) typo setup
413
+ # still owes the user the instant-arming matrix — without this an
414
+ # install set up before the matrix existed derives at arming time
415
+ # (~25s) forever. Cache-only when present; best-effort fetch
416
+ # otherwise. Returns :downloaded when fetched, nil when present or
417
+ # unservable — callers degrade, never fail, on nil.
418
+ def ensure_typo_matrix(cache, lang)
419
+ return nil if cache.load_cached_typo_matrix(lang)
420
+
421
+ cache.download_typo_matrix(lang)
400
422
  :downloaded
423
+ rescue Kotoshu::Error, StandardError
424
+ nil
401
425
  end
402
426
 
403
427
  def setup_frequency_remote(lang, force:, strict:, config:)
@@ -27,6 +27,10 @@ module Kotoshu
27
27
  # @return [String] the language the tier was loaded for
28
28
  attr_reader :language
29
29
 
30
+ # @return [Symbol] :matrix when a prebuilt KTM1 armed the engine,
31
+ # :derived when the index was built at load (plan 139)
32
+ attr_reader :armed_via
33
+
30
34
  # Load the engine for a language. Yielded block receives the
31
35
  # cache to use (defaults to the configured model cache) — specs
32
36
  # inject a stub cache with local artifact paths.
@@ -46,13 +50,29 @@ module Kotoshu
46
50
 
47
51
  native = Kotoshu::Native::TypoModel.load(typo[:onnx_path], typo[:vocab_path])
48
52
  native_tier = Kotoshu::Native::TypoTier.load(tier[:model_path], tier[:vocab_path])
49
- engine = new(Kotoshu::Native::TypoEngine.new(native, native_tier),
50
- native_tier: native_tier, language: language)
51
- # EAGER index build (plan 134): the multi-second derivation
52
- # runs HERE - with the GVL released by the binding - instead
53
- # of stalling the first suggest (and every Ruby thread under a
54
- # held GVL). The cost lands at opt-in setup where it belongs.
55
- engine.build_index
53
+
54
+ # Plan 136: a cached prebuilt matrix arms in milliseconds
55
+ # (TypoEngine.matrix); without one, derive eagerly with the
56
+ # GVL released (plan 134). Both paths produce identical
57
+ # slates - the artifact round-trip is frozen by the engine's
58
+ # own tests. armed_via records which path won so a silent
59
+ # degrade (stale ext missing .matrix, corrupt artifact) is
60
+ # observable instead of looking like success (plan 139).
61
+ engine =
62
+ if (matrix_path = cache.load_cached_typo_matrix(language)) &&
63
+ Kotoshu::Native::TypoEngine.respond_to?(:matrix)
64
+ begin
65
+ new(Kotoshu::Native::TypoEngine.matrix(native, native_tier, matrix_path),
66
+ native_tier: native_tier, language: language, armed_via: :matrix)
67
+ rescue StandardError
68
+ nil
69
+ end
70
+ end
71
+ unless engine
72
+ engine = new(Kotoshu::Native::TypoEngine.new(native, native_tier),
73
+ native_tier: native_tier, language: language, armed_via: :derived)
74
+ engine.build_index
75
+ end
56
76
  engine
57
77
  rescue StandardError
58
78
  # The opt-in layer never breaks a check: any load failure
@@ -73,10 +93,12 @@ module Kotoshu
73
93
  # @param native_tier [Kotoshu::Native::TypoTier] the rescore
74
94
  # tier (kept for the eager index build)
75
95
  # @param language [String]
76
- def initialize(native_engine, language:, native_tier: nil)
96
+ # @param armed_via [Symbol] :matrix or :derived (plan 139)
97
+ def initialize(native_engine, language:, native_tier: nil, armed_via: :derived)
77
98
  @native_engine = native_engine
78
99
  @native_tier = native_tier
79
100
  @language = language
101
+ @armed_via = armed_via
80
102
  end
81
103
 
82
104
  # The rescored slate for a word as a SuggestionSet, source
@@ -120,6 +142,14 @@ module Kotoshu
120
142
  cache = Cache::ModelCache.new(cache_path: configuration.cache_path)
121
143
  typo = cache.download_typo_biencoder(force: force)
122
144
  tier = cache.download_tiered_model(language, tier: :full, force_download: force)
123
- { typo: typo, tier: tier }
145
+ # Plan 136: the prebuilt matrix makes arming instant. Best-effort
146
+ # only - a language without one arms by deriving (the fallback),
147
+ # and the fetch must never fail the setup that already succeeded.
148
+ matrix = begin
149
+ cache.download_typo_matrix(language, force: force)
150
+ rescue Kotoshu::Error, StandardError
151
+ nil
152
+ end
153
+ { typo: typo, tier: tier, matrix: matrix }
124
154
  end
125
155
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kotoshu
4
- VERSION = '1.0.3'
4
+ VERSION = '1.0.4'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kotoshu
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.4
5
5
  platform: aarch64-linux
6
6
  authors:
7
7
  - Ribose Inc.
@@ -124,6 +124,11 @@ files:
124
124
  - TODO.impl/132-techniques-page-and-prose.md
125
125
  - TODO.impl/133-rust-first-distribution.md
126
126
  - TODO.impl/134-post-131-improvements.md
127
+ - TODO.impl/135-post-train-improvements.md
128
+ - TODO.impl/136-prebuilt-typo-matrices.md
129
+ - TODO.impl/137-typo-matrix-backfill.md
130
+ - TODO.impl/138-ext-pin-drift-guard.md
131
+ - TODO.impl/139-typo-observability.md
127
132
  - TODO.impl/20-encapsulation-cleanup.md
128
133
  - TODO.impl/21-autoload-migration.md
129
134
  - TODO.impl/22-url-pin-single-source-of-truth.md