kotoshu 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -88,6 +88,16 @@ module Kotoshu
88
88
  # Directory (under the cache root) holding the LID artifact pair.
89
89
  LID_DIRECTORY = "lid"
90
90
 
91
+ # The hybrid typo bi-encoder (plan 131): a language-less
92
+ # registry resource (kotoshu://models/typo/typo-biencoder) with
93
+ # a REQUIRED vocab sidecar, resolved exactly like the LID pair.
94
+ TYPO_LANGUAGE = "typo"
95
+ TYPO_TIER = "typo-biencoder"
96
+ TYPO_REGISTRY_ID = ModelRegistry::Resource.id_for(TYPO_LANGUAGE, TYPO_TIER)
97
+
98
+ # Cache directory (under the models root, beside the languages).
99
+ TYPO_DIRECTORY = File.join("models", "typo")
100
+
91
101
  # Normalize and validate a tier name.
92
102
  #
93
103
  # @param tier [String, Symbol] "full", "fluency", or "mini"
@@ -202,7 +212,30 @@ module Kotoshu
202
212
  # no entry for the pair
203
213
  # @raise [Kotoshu::Error] when offline with no cached registry
204
214
  def registry_entry_for(language_code, tier, force: false)
205
- registry(force: force)&.find(language_code.to_s, tier.to_s)
215
+ return registry(force: true)&.find(language_code.to_s, tier.to_s) if force
216
+
217
+ registry&.find(language_code.to_s, tier.to_s) ||
218
+ find_after_registry_refresh(language_code, tier)
219
+ end
220
+
221
+ # Plan 135: a cached registry within its TTL shadows the live
222
+ # one - during the v1.7.0 train, a v1.4.0 cache made a newly
223
+ # published resource invisible until manual deletion. When the
224
+ # wanted entry is absent from the cached copy, refresh the
225
+ # registry ONCE and look again. An offline refresh failure keeps
226
+ # the original miss: the legacy fallbacks (CDN for tiers, the
227
+ # honest no-entry error for the typo pair) apply exactly as
228
+ # before.
229
+ def find_after_registry_refresh(language_code, tier)
230
+ refreshed = begin
231
+ registry(force: true)
232
+ rescue StandardError
233
+ # Offline or transport failure: keep the original miss, the
234
+ # existing fallbacks apply (Windows surfaces raw Errno
235
+ # classes the wrapper does not always raise as Kotoshu::Error).
236
+ nil
237
+ end
238
+ refreshed&.find(language_code.to_s, tier.to_s)
206
239
  end
207
240
 
208
241
  # Download and verify a tiered model from the registry.
@@ -224,9 +257,17 @@ module Kotoshu
224
257
  def download_tiered_model(language_code, tier:, force_download: false)
225
258
  tier = self.class.normalize_tier(tier)
226
259
  lang = language_code.to_s
227
- return get("#{lang}:onnx", force_download: force_download) if tier == :full
228
260
 
261
+ # The full tier rides the registry when the cached registry
262
+ # knows it (primary URL, vocab sibling, sha256 — the path the
263
+ # typo layer's rescore requires, since the legacy download
264
+ # never fetched the vocab). Offline without a cached registry
265
+ # the entry is nil and the legacy CDN path serves, preserving
266
+ # the pre-registry behavior exactly.
229
267
  entry = registry_entry_for(lang, tier, force: force_download)
268
+ if tier == :full && entry.nil?
269
+ return get("#{lang}:onnx", force_download: force_download)
270
+ end
230
271
  unless entry
231
272
  raise Kotoshu::Error,
232
273
  "no registry entry for #{ModelRegistry::Resource.id_for(lang, tier)}"
@@ -257,6 +298,7 @@ module Kotoshu
257
298
  cached_at: Time.now.utc.iso8601,
258
299
  source: "registry"
259
300
  }
301
+ metadata[:vocab_file] = File.basename(vocab_path) if vocab_path
260
302
  write_metadata(File.join(dest_path, "metadata.json"), metadata)
261
303
 
262
304
  result = { model_path: model_file, metadata: metadata }
@@ -293,7 +335,7 @@ module Kotoshu
293
335
  return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
294
336
  return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
295
337
 
296
- verify_lid_integrity!(metadata, onnx_file, vocab_file)
338
+ verify_model_pair_integrity!(metadata, onnx_file, vocab_file)
297
339
 
298
340
  { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
299
341
  end
@@ -354,6 +396,191 @@ module Kotoshu
354
396
  { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
355
397
  end
356
398
 
399
+ # ---- typo bi-encoder pair (plan 131) ----
400
+
401
+ # Whether the typo bi-encoder pair is cached and complete
402
+ # (both files, checksum-verified).
403
+ #
404
+ # @return [Boolean]
405
+ def typo_biencoder_cached?
406
+ !load_cached_typo_biencoder.nil?
407
+ end
408
+
409
+ # Resolve the typo bi-encoder pair from the cache only — never
410
+ # touches the network (the suggest path resolves; setup
411
+ # downloads, the same two-stage split as the LID pair).
412
+ #
413
+ # @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
414
+ # cached, nil when not set up
415
+ # @raise [Kotoshu::IntegrityError] cached bytes fail their
416
+ # recorded checksum
417
+ def load_cached_typo_biencoder
418
+ dir = File.join(@cache_path, TYPO_DIRECTORY)
419
+ metadata_path = File.join(dir, "metadata.json")
420
+ return nil unless File.exist?(metadata_path)
421
+
422
+ metadata = read_metadata(metadata_path)
423
+ return nil unless metadata
424
+
425
+ onnx_file = File.join(dir, metadata["file"].to_s)
426
+ vocab_file = File.join(dir, metadata["vocab_file"].to_s)
427
+ return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
428
+ return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
429
+
430
+ verify_model_pair_integrity!(
431
+ metadata, onnx_file, vocab_file,
432
+ directory: dir,
433
+ registry_id: TYPO_REGISTRY_ID,
434
+ remediation: "Run Kotoshu.setup_typo(language, force: true) to re-download."
435
+ )
436
+
437
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
438
+ end
439
+
440
+ # Download the typo bi-encoder pair from the models registry,
441
+ # verifying the ONNX bytes against the registry sha256 (the
442
+ # setup half of the two-stage model). The vocab sidecar is
443
+ # required, not optional — the encoder cannot tokenize without
444
+ # its char vocabulary — so a registry entry without vocab_url
445
+ # (the pre-cut state) raises the honest error instead of
446
+ # caching a half-usable artifact.
447
+ #
448
+ # @param force [Boolean] re-fetch the registry first
449
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
450
+ # @raise [Kotoshu::Error] no entry, entry without vocab_url, or
451
+ # offline without a cached registry
452
+ # @raise [Kotoshu::IntegrityError] sha256 mismatch
453
+ def download_typo_biencoder(force: false)
454
+ entry = registry(force: force)&.find(TYPO_LANGUAGE, TYPO_TIER)
455
+ unless entry
456
+ raise Kotoshu::Error, "no registry entry for #{TYPO_REGISTRY_ID}"
457
+ end
458
+ if entry.vocab_url.nil? || entry.vocab_url.to_s.strip.empty?
459
+ raise Kotoshu::Error,
460
+ "#{TYPO_REGISTRY_ID} carries no vocab_url yet — the typo pair ships with a models registry release"
461
+ end
462
+
463
+ dir = File.join(@cache_path, TYPO_DIRECTORY)
464
+ FileUtils.mkdir_p(dir)
465
+
466
+ onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
467
+ used_url = download_primary_or_mirror!(entry, onnx_file)
468
+ verify_registry_sha256!(entry, onnx_file, TYPO_REGISTRY_ID, used_url,
469
+ remediation: "Run Kotoshu::Typo.setup(force: true) to re-download.")
470
+
471
+ vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
472
+ begin
473
+ download_file(entry.vocab_url, vocab_file)
474
+ rescue StandardError => e
475
+ File.delete(onnx_file) if File.exist?(onnx_file)
476
+ raise Kotoshu::Error, "typo vocab unavailable at #{entry.vocab_url}: #{e.message}"
477
+ end
478
+
479
+ metadata = {
480
+ version: entry.version,
481
+ url: used_url,
482
+ language: TYPO_LANGUAGE,
483
+ type: "typo-biencoder",
484
+ tier: TYPO_TIER,
485
+ file: File.basename(onnx_file),
486
+ vocab_file: File.basename(vocab_file),
487
+ checksum: entry.sha256,
488
+ registry_id: TYPO_REGISTRY_ID,
489
+ size_bytes: entry.size_bytes,
490
+ cached_at: Time.now.utc.iso8601,
491
+ source: "registry"
492
+ }
493
+ write_metadata(File.join(dir, "metadata.json"), metadata)
494
+
495
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
496
+ end
497
+
498
+ # Resolve a tiered model from the cache only — the cache half of
499
+ # the two-stage model for tiers, symmetric with
500
+ # {#load_cached_lid} / {#load_cached_typo_biencoder}. Returns
501
+ # the vocab sibling when the registry-served download landed it.
502
+ #
503
+ # @param language_code [String]
504
+ # @param tier [String, Symbol]
505
+ # @return [Hash, nil] { model_path:, vocab_path?, metadata } or nil
506
+ def load_cached_tier(language_code, tier)
507
+ tier = self.class.normalize_tier(tier)
508
+ resource_id = tier_resource_id(language_code.to_s, tier)
509
+ dir = resource_dir_for(resource_id)
510
+ metadata_path = File.join(dir, "metadata.json")
511
+ return nil unless File.exist?(metadata_path)
512
+
513
+ metadata = read_metadata(metadata_path)
514
+ return nil unless metadata
515
+
516
+ model_file = File.join(dir, metadata["file"].to_s)
517
+ return nil unless File.exist?(model_file) && File.size(model_file).positive?
518
+
519
+ result = { model_path: model_file, metadata: metadata }
520
+ vocab = metadata["vocab_file"].to_s
521
+ vocab_path = File.join(dir, vocab)
522
+ result[:vocab_path] = vocab_path if !vocab.empty? && File.exist?(vocab_path)
523
+ result
524
+ end
525
+
526
+ # ---- prebuilt typo matrix (plan 136) ----
527
+
528
+ # Resolve the language's prebuilt KTM1 matrix from the cache
529
+ # only (the resolve half; setup downloads).
530
+ #
531
+ # @param language_code [String]
532
+ # @return [String, nil] path to the cached .ktm1, nil when absent
533
+ def load_cached_typo_matrix(language_code)
534
+ entry = begin
535
+ registry&.find(language_code.to_s, "typo-matrix")
536
+ rescue StandardError
537
+ nil
538
+ end
539
+ return nil unless entry
540
+
541
+ dir = File.join(@cache_path, language_code.to_s, "models", "typo-matrix")
542
+ file = File.join(dir, filename_from_url(entry.urls.mirror))
543
+ return nil unless File.exist?(file) && File.size(file).positive?
544
+ return nil unless Digest::SHA256.file(file).hexdigest == entry.sha256
545
+
546
+ file
547
+ end
548
+
549
+ # Download the matrix (the setup half). Raises like the other
550
+ # registry downloads when the registry cannot serve it yet.
551
+ #
552
+ # @param language_code [String]
553
+ # @param force [Boolean]
554
+ # @return [String] path to the cached .ktm1
555
+ def download_typo_matrix(language_code, force: false)
556
+ lang = language_code.to_s
557
+ entry = registry(force: force)&.find(lang, "typo-matrix")
558
+ unless entry
559
+ raise Kotoshu::Error,
560
+ "no registry entry for kotoshu://models/#{lang}/typo-matrix (the matrix ships per language; derive-at-load remains the fallback)"
561
+ end
562
+
563
+ dir = File.join(@cache_path, lang, "models", "typo-matrix")
564
+ FileUtils.mkdir_p(dir)
565
+ file = File.join(dir, filename_from_url(entry.urls.mirror))
566
+ used_url = download_primary_or_mirror!(entry, file)
567
+ verify_registry_sha256!(entry, file, "kotoshu://models/#{lang}/typo-matrix", used_url,
568
+ remediation: "Run kotoshu setup #{lang} --typo --force to re-download.")
569
+
570
+ write_metadata(File.join(dir, "metadata.json"),
571
+ "version" => entry.version,
572
+ "url" => used_url,
573
+ "language" => lang,
574
+ "type" => "typo-matrix",
575
+ "file" => File.basename(file),
576
+ "checksum" => entry.sha256,
577
+ "registry_id" => "kotoshu://models/#{lang}/typo-matrix",
578
+ "size_bytes" => entry.size_bytes,
579
+ "cached_at" => Time.now.utc.iso8601,
580
+ "source" => "registry")
581
+ file
582
+ end
583
+
357
584
  # Get available model types for a language.
358
585
  #
359
586
  # @param language_code [String] ISO 639-1 language code
@@ -801,16 +1028,19 @@ module Kotoshu
801
1028
  # @return [void]
802
1029
  # @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
803
1030
  # corrupt bytes are removed so the next setup re-downloads
804
- def verify_lid_integrity!(metadata, onnx_file, vocab_file)
805
- dir = File.join(@cache_path, LID_DIRECTORY)
1031
+ def verify_model_pair_integrity!(metadata, onnx_file, vocab_file, directory: nil,
1032
+ registry_id: nil, remediation: nil)
1033
+ registry_id ||= LID_REGISTRY_ID
1034
+ directory ||= File.join(@cache_path, LID_DIRECTORY)
1035
+ remediation ||= "Run Kotoshu.setup_lid(force: true) to re-download."
806
1036
  if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
807
1037
  File.delete(onnx_file)
808
1038
  File.delete(vocab_file)
809
- FileUtils.rm_f(File.join(dir, "metadata.json"))
1039
+ FileUtils.rm_f(File.join(directory, "metadata.json"))
810
1040
  raise Kotoshu::Error,
811
- "Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
812
- "artifact is a Git LFS pointer stub, not model content. Run " \
813
- "Kotoshu.setup_lid(force: true) to re-download."
1041
+ "Integrity verification failed for #{registry_id}: cached " \
1042
+ "artifact is a Git LFS pointer stub, not model content. " \
1043
+ "#{remediation}"
814
1044
  end
815
1045
 
816
1046
  expected = metadata["checksum"]
@@ -820,11 +1050,11 @@ module Kotoshu
820
1050
  return if actual == expected
821
1051
 
822
1052
  raise Kotoshu::IntegrityError.new(
823
- LID_REGISTRY_ID,
1053
+ registry_id,
824
1054
  expected: expected,
825
1055
  actual: actual,
826
1056
  url: metadata["url"],
827
- remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
1057
+ remediation: remediation
828
1058
  )
829
1059
  end
830
1060
 
data/lib/kotoshu/cli.rb CHANGED
@@ -146,6 +146,7 @@ module Kotoshu
146
146
  kotoshu setup en de fr # download from GitHub
147
147
  kotoshu setup en --want spelling,frequency # also fetch Kelly list
148
148
  kotoshu setup en --model --tier mini # model at the mini tier
149
+ kotoshu setup en --typo # typo bi-encoder + full tier
149
150
  kotoshu setup en --aff /p/en.aff --dic /p/en.dic
150
151
  kotoshu setup en --from /usr/share/hunspell/
151
152
  kotoshu setup --force en # re-download
@@ -167,6 +168,10 @@ module Kotoshu
167
168
  type: :boolean,
168
169
  default: false,
169
170
  desc: "Fetch the ONNX model (shorthand for adding model to --want)"
171
+ method_option :typo,
172
+ type: :boolean,
173
+ default: false,
174
+ desc: "Fetch the typo-retrieval pair (bi-encoder + full tier; needs a registry release carrying it)"
170
175
  method_option :tier,
171
176
  type: :string,
172
177
  desc: "Model tier: full, fluency, or mini (default: full)"
@@ -192,6 +197,7 @@ module Kotoshu
192
197
 
193
198
  want = (options[:want] || "spelling").split(",").map(&:strip).map(&:to_sym)
194
199
  want << :model if options[:model] && !want.include?(:model)
200
+ want << :typo if options[:typo] && !want.include?(:typo)
195
201
  opts = setup_source_options(languages)
196
202
  opts[:want] = want
197
203
  opts[:tier] = options[:tier].to_sym if options[:tier]
@@ -234,6 +240,10 @@ module Kotoshu
234
240
  type: :boolean,
235
241
  default: false,
236
242
  desc: "Fetch the ONNX model (shorthand for adding model to --want)"
243
+ method_option :typo,
244
+ type: :boolean,
245
+ default: false,
246
+ desc: "Fetch the typo-retrieval pair (bi-encoder + full tier; needs a registry release carrying it)"
237
247
  method_option :tier,
238
248
  type: :string,
239
249
  desc: "Model tier: full, fluency, or mini (default: full)"
@@ -525,8 +535,10 @@ module Kotoshu
525
535
  frequency = result.frequency || "skipped"
526
536
  model = result.model || "skipped"
527
537
  model += " (#{result.model_tier} tier)" if result.model && result.model_tier
538
+ typo = result.typo || "skipped"
528
539
  source = result.source
529
- puts "OK (spelling: #{spelling}, frequency: #{frequency}, model: #{model}, source: #{source})"
540
+ puts "OK (spelling: #{spelling}, frequency: #{frequency}, model: #{model}, " \
541
+ "typo: #{typo}, source: #{source})"
530
542
  end
531
543
 
532
544
  def list_setup
@@ -537,7 +549,20 @@ module Kotoshu
537
549
  end
538
550
 
539
551
  puts "Set up languages:"
540
- langs.each { |lang| puts " #{lang}" }
552
+ langs.each do |lang|
553
+ parts = []
554
+ parts << "spelling" if Kotoshu::ResourceManager.setup?(lang)
555
+ parts << "frequency" if Kotoshu::ResourceManager.setup?(lang, resource: :frequency)
556
+ parts << "model" if Kotoshu::ResourceManager.setup?(lang, resource: :model, tier: :any)
557
+ parts << "typo" if Kotoshu::ResourceManager.setup?(lang, resource: :typo)
558
+ # Plan 139: surface the prebuilt matrix so a silent derive
559
+ # path is visible from `kotoshu setup --list`.
560
+ parts << "typo-matrix" if Kotoshu::ResourceManager.setup?(lang, resource: :typo_matrix)
561
+ # languages_setup is looser than setup? (TTL/metadata edge
562
+ # cases); fall back to the bare code when nothing verifies so
563
+ # we never print an empty "lang: " line.
564
+ puts(parts.empty? ? " #{lang}" : " #{lang}: #{parts.join(', ')}")
565
+ end
541
566
  end
542
567
 
543
568
  def display_result(result, source, application: nil)
@@ -206,10 +206,16 @@ module Kotoshu
206
206
  },
207
207
  backend: {
208
208
  env: "KOTOSHU_BACKEND",
209
- default: "ruby",
210
- description: "Engine backend for correct?/suggest (auto by default: native when the extension is available, ruby otherwise; or force either)",
209
+ default: "auto",
210
+ description: "Engine backend for correct?/suggest (auto: native when the extension is available, ruby otherwise; or force either)",
211
211
  type: String
212
212
  },
213
+ typo_retrieval: {
214
+ env: "KOTOSHU_TYPO_RETRIEVAL",
215
+ default: false,
216
+ description: "Opt-in hybrid typo retrieval (plan 131): the native bi-encoder slate merges ahead of frequency ranking; needs the native backend",
217
+ type: :boolean
218
+ },
213
219
  personal_dictionary: {
214
220
  env: "KOTOSHU_PERSONAL_DICTIONARY",
215
221
  default: true,
@@ -241,6 +247,7 @@ module Kotoshu
241
247
  # conformance contract), so the default is the faster engine
242
248
  # with the pure fallback, never a failure.
243
249
  backend: "auto",
250
+ typo_retrieval: false,
244
251
  personal_dictionary: true
245
252
  }.freeze
246
253
 
@@ -301,6 +308,12 @@ module Kotoshu
301
308
  # NativeBackend for the boundary — the semantic path stays Ruby.
302
309
  attr_accessor :backend
303
310
 
311
+ # @return [Boolean] Opt-in hybrid typo retrieval (plan 131): the
312
+ # native bi-encoder's rescored slate merges ahead of frequency
313
+ # ranking when the native backend serves the language. Env:
314
+ # KOTOSHU_TYPO_RETRIEVAL.
315
+ attr_accessor :typo_retrieval
316
+
304
317
  # @return [Boolean] Whether Spellchecker#check accepts words from
305
318
  # the user personal dictionary (~/.config/kotoshu/personal.dic,
306
319
  # KOTOSHU_PERSONAL_DIC override) instead of flagging them as
@@ -511,6 +524,7 @@ module Kotoshu
511
524
  max_cache_size: @max_cache_size,
512
525
  semantic_cascade_threshold: @semantic_cascade_threshold,
513
526
  backend: @backend,
527
+ typo_retrieval: @typo_retrieval,
514
528
  personal_dictionary: @personal_dictionary
515
529
  }
516
530
  end
@@ -629,6 +643,7 @@ module Kotoshu
629
643
  @resource_pin = DEFAULTS[:resource_pin]
630
644
  @semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
631
645
  @backend = DEFAULTS[:backend]
646
+ @typo_retrieval = DEFAULTS[:typo_retrieval]
632
647
  @personal_dictionary = DEFAULTS[:personal_dictionary]
633
648
  end
634
649
 
@@ -41,6 +41,7 @@ module Kotoshu
41
41
  :frequency, # :downloaded | :local | :cached | :unavailable | nil
42
42
  :model, # :downloaded | :cached | :unavailable | nil
43
43
  :model_tier, # :full | :fluency | :mini | nil (tier actually set up)
44
+ :typo, # :downloaded | :cached | :unavailable | nil (plan 131 pair)
44
45
  :source, # :kotoshu | :local
45
46
  keyword_init: true
46
47
  ) do
@@ -263,7 +264,7 @@ module Kotoshu
263
264
  #
264
265
  # @param language [String, Symbol] Language code
265
266
  # @param resource [Symbol, nil] :spelling (default), :frequency,
266
- # :model, or nil for "spelling"
267
+ # :model, :typo, :typo_matrix, or nil for "spelling"
267
268
  # @param tier [Symbol, nil] Model tier to check (:full, :fluency,
268
269
  # :mini, or :any); only meaningful with resource: :model
269
270
  # @return [Boolean]
@@ -272,6 +273,7 @@ module Kotoshu
272
273
  # ResourceManager.setup?(:en) # spelling
273
274
  # ResourceManager.setup?(:en, resource: :frequency) # Kelly list
274
275
  # ResourceManager.setup?(:en, resource: :model, tier: :mini)
276
+ # ResourceManager.setup?(:en, resource: :typo_matrix)
275
277
  def setup?(language, resource: nil, tier: nil)
276
278
  lang = normalize_language(language)
277
279
  case resource&.to_sym
@@ -280,6 +282,16 @@ module Kotoshu
280
282
  when :frequency
281
283
  fc = frequency_cache_for
282
284
  fc.supports_resource?(lang) && fc.available?(lang)
285
+ when :typo
286
+ # Both halves: the language-less pair and the language's full
287
+ # tier (the rescore's exactness contract).
288
+ cache = model_cache_for
289
+ cache.typo_biencoder_cached? && cache.available?(cache.tier_resource_id(lang, :full))
290
+ when :typo_matrix
291
+ # Plan 139: the prebuilt KTM1 is a separate, optional half of
292
+ # the typo setup. Absent means arming derives (~25 s); present
293
+ # means arming is a download (ms).
294
+ !model_cache_for.load_cached_typo_matrix(lang).nil?
283
295
  when :model
284
296
  cache = model_cache_for
285
297
  if tier.nil?
@@ -343,16 +355,75 @@ module Kotoshu
343
355
  end
344
356
  end
345
357
 
358
+ typo_status = if want.include?(:typo)
359
+ setup_typo_remote(lang, force: force, strict: strict, config: config,
360
+ tier: model_tier)
361
+ end
362
+
346
363
  SetupResult.new(
347
364
  language: lang,
348
365
  spelling: spelling_status,
349
366
  frequency: frequency_status,
350
367
  model: model_status,
351
368
  model_tier: model_tier,
369
+ typo: typo_status,
352
370
  source: :kotoshu
353
371
  )
354
372
  end
355
373
 
374
+ # The typo layer's setup half (plan 131): the language-less
375
+ # bi-encoder pair, plus the language's FULL tier when this run did
376
+ # not already fetch it (the hybrid's rescore is full-tier-exact by
377
+ # contract — a mini tier would rank wrong, so full is fetched
378
+ # explicitly, not defaulted).
379
+ #
380
+ # Returns :cached when both halves were already present,
381
+ # :downloaded when anything was fetched, :unavailable when the
382
+ # registry cannot serve the pair yet (strict: raises instead —
383
+ # the pre-cut state).
384
+ def setup_typo_remote(lang, force:, strict:, config:, tier: nil)
385
+ cache = model_cache_for(config: config)
386
+ unless force
387
+ full_id = cache.tier_resource_id(lang, :full)
388
+ if cache.typo_biencoder_cached? && cache.available?(full_id)
389
+ return ensure_typo_matrix(cache, lang) || :cached
390
+ end
391
+ end
392
+ begin
393
+ cache.download_typo_biencoder(force: force)
394
+ cache.download_tiered_model(lang, tier: :full, force_download: force) unless tier == :full
395
+ rescue Kotoshu::Error
396
+ raise if strict
397
+
398
+ return :unavailable
399
+ rescue StandardError => e
400
+ # Windows surfaces closed-port fetches as raw Errno classes the
401
+ # cache does not always wrap; the two-stage contract is
402
+ # setup-degrades unless strict, and strict reports through the
403
+ # typed error like every setup path.
404
+ raise Kotoshu::Error, "typo setup failed: #{e.class}: #{e.message}" if strict
405
+
406
+ return :unavailable
407
+ end
408
+ ensure_typo_matrix(cache, lang)
409
+ :downloaded
410
+ end
411
+
412
+ # Plan 136: an already-satisfied (or just-satisfied) typo setup
413
+ # still owes the user the instant-arming matrix — without this an
414
+ # install set up before the matrix existed derives at arming time
415
+ # (~25s) forever. Cache-only when present; best-effort fetch
416
+ # otherwise. Returns :downloaded when fetched, nil when present or
417
+ # unservable — callers degrade, never fail, on nil.
418
+ def ensure_typo_matrix(cache, lang)
419
+ return nil if cache.load_cached_typo_matrix(lang)
420
+
421
+ cache.download_typo_matrix(lang)
422
+ :downloaded
423
+ rescue Kotoshu::Error, StandardError
424
+ nil
425
+ end
426
+
356
427
  def setup_frequency_remote(lang, force:, strict:, config:)
357
428
  cache = frequency_cache_for(config: config)
358
429
  return :unavailable unless cache.supports_resource?(lang)
@@ -140,9 +140,39 @@ module Kotoshu
140
140
  def suggest(word, max_suggestions: nil)
141
141
  return Suggestions::SuggestionSet.empty if word.nil? || word.empty?
142
142
 
143
- return @native_backend.suggest(word, max_suggestions: max_suggestions) if @native_backend
143
+ base = if @native_backend
144
+ @native_backend.suggest(word, max_suggestions: max_suggestions)
145
+ else
146
+ @generator.generate(word, max_suggestions: max_suggestions)
147
+ end
148
+ merge_typo_retrieval(base, word, max_suggestions: max_suggestions)
149
+ end
150
+
151
+ # Merge the opt-in typo-retrieval slate (plan 131) AHEAD of the
152
+ # base suggestions: the bi-encoder surfaces corrections a
153
+ # frequency-ranked sweep misses, so its rows lead. Inert unless
154
+ # typo_retrieval is configured AND the native engine loaded for
155
+ # this language — a disabled layer is a passthrough, never an
156
+ # error, never a different order.
157
+ def merge_typo_retrieval(base, word, max_suggestions: nil)
158
+ return base unless @config.typo_retrieval
159
+
160
+ engine = typo_retrieval_engine
161
+ return base unless engine
162
+
163
+ typo = engine.suggest(word, max_suggestions: max_suggestions)
164
+ limit = max_suggestions || @config.max_suggestions
165
+ Suggestions::TypoMerge.call(base: base, typo: typo, limit: limit)
166
+ end
167
+
168
+ # The typo engine for this language, resolved once. A nil result
169
+ # is memoized too — an unavailable layer must not retry its
170
+ # artifact resolution on every suggestion.
171
+ def typo_retrieval_engine
172
+ return @typo_retrieval_engine if defined?(@typo_retrieval_engine)
144
173
 
145
- @generator.generate(word, max_suggestions: max_suggestions)
174
+ code = (@resource_bundle&.language || @config.language).to_s.split("-").first
175
+ @typo_retrieval_engine = Kotoshu::Typo::Engine.for(code, configuration: @config)
146
176
  end
147
177
 
148
178
  # Check a word and return a result object.
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ module Strategies
6
+ # The hybrid typo-retrieval strategy (plan 131): a frozen char
7
+ # bi-encoder retrieves dictionary candidates a frequency-ranked
8
+ # sweep misses, the fastText full tier rescores them, and the
9
+ # rows join the suggestion pipeline AHEAD of frequency ranking
10
+ # — the layer measured at +6.3 pp top-5 on the frozen
11
+ # benchmark. Opt-in and native-only; the engine is injected so
12
+ # the strategy itself stays pure orchestration.
13
+ class TypoRetrievalStrategy < BaseStrategy
14
+ # @return [#suggest] the typo engine ({Typo::Engine} in
15
+ # production; anything answering #suggest(word) in specs)
16
+ attr_reader :engine
17
+
18
+ # @param engine [#suggest] the typo engine
19
+ # @param config [Hash] additional configuration
20
+ def initialize(engine:, **config)
21
+ super(name: :typo_retrieval, **config)
22
+ @engine = engine
23
+ end
24
+
25
+ # Generate the typo-retrieval slate for the context's word.
26
+ #
27
+ # @param context [Context] the suggestion context
28
+ # @return [SuggestionSet]
29
+ def generate(context)
30
+ return SuggestionSet.empty unless enabled? && @engine
31
+
32
+ @engine.suggest(context.word, max_suggestions: max_results)
33
+ end
34
+
35
+ # This strategy is a candidate generator for misspellings; it
36
+ # has no real-word-error story, which the semantic strategy
37
+ # owns.
38
+ #
39
+ # @param context [Context]
40
+ # @return [Boolean]
41
+ def handles?(context)
42
+ enabled? && @engine && !context.word.empty?
43
+ end
44
+
45
+ # The strategy's rows already carry their own order (the
46
+ # rescored ranking); do not let a confident base set skip it.
47
+ #
48
+ # @return [Boolean]
49
+ def skip_when_confident?
50
+ false
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Suggestions
5
+ # The typo-retrieval merge (plan 131): the bi-encoder's rescored
6
+ # slate joins AHEAD of the base suggestions — it surfaces
7
+ # corrections a frequency-ranked sweep misses — with duplicates
8
+ # (same word) dropped from the base side and the set capped at the
9
+ # caller's limit. A pure function over real SuggestionSets; the
10
+ # spellchecker calls it only when the opt-in layer is armed.
11
+ module TypoMerge
12
+ # @param base [SuggestionSet] the engine's own suggestions
13
+ # @param typo [SuggestionSet] the typo-retrieval slate
14
+ # @param limit [Integer] final cap
15
+ # @return [SuggestionSet] merged set, order adopted verbatim
16
+ def self.call(base:, typo:, limit:)
17
+ return base if typo.empty?
18
+
19
+ typo_words = typo.map(&:word)
20
+ merged = typo.suggestions +
21
+ base.suggestions.reject { |suggestion| typo_words.include?(suggestion.word) }
22
+ SuggestionSet.new(merged.first(limit), max_size: limit, ranked: true)
23
+ end
24
+ end
25
+ end
26
+ end