kotoshu 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +23 -0
- data/Rakefile +46 -0
- data/TODO.impl/124-owner-residue.md +15 -6
- data/TODO.impl/131-hybrid-productization.md +16 -1
- data/TODO.impl/133-rust-first-distribution.md +23 -1
- data/TODO.impl/134-post-131-improvements.md +67 -0
- data/TODO.impl/135-post-train-improvements.md +38 -0
- data/TODO.impl/136-prebuilt-typo-matrices.md +30 -0
- data/TODO.impl/137-typo-matrix-backfill.md +50 -0
- data/TODO.impl/138-ext-pin-drift-guard.md +43 -0
- data/TODO.impl/139-typo-observability.md +40 -0
- data/TODO.impl/54-cjk-support.md +5 -2
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +29 -0
- data/lib/kotoshu/cache/model_cache.rb +241 -11
- data/lib/kotoshu/cli.rb +27 -2
- data/lib/kotoshu/configuration.rb +17 -2
- data/lib/kotoshu/resource_manager.rb +72 -1
- data/lib/kotoshu/spellchecker.rb +32 -2
- data/lib/kotoshu/suggestions/strategies/typo_retrieval_strategy.rb +55 -0
- data/lib/kotoshu/suggestions/typo_merge.rb +26 -0
- data/lib/kotoshu/suggestions.rb +2 -0
- data/lib/kotoshu/typo/engine.rb +155 -0
- data/lib/kotoshu/typo.rb +9 -0
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +1 -0
- metadata +12 -2
|
@@ -88,6 +88,16 @@ module Kotoshu
|
|
|
88
88
|
# Directory (under the cache root) holding the LID artifact pair.
|
|
89
89
|
LID_DIRECTORY = "lid"
|
|
90
90
|
|
|
91
|
+
# The hybrid typo bi-encoder (plan 131): a language-less
|
|
92
|
+
# registry resource (kotoshu://models/typo/typo-biencoder) with
|
|
93
|
+
# a REQUIRED vocab sidecar, resolved exactly like the LID pair.
|
|
94
|
+
TYPO_LANGUAGE = "typo"
|
|
95
|
+
TYPO_TIER = "typo-biencoder"
|
|
96
|
+
TYPO_REGISTRY_ID = ModelRegistry::Resource.id_for(TYPO_LANGUAGE, TYPO_TIER)
|
|
97
|
+
|
|
98
|
+
# Cache directory (under the models root, beside the languages).
|
|
99
|
+
TYPO_DIRECTORY = File.join("models", "typo")
|
|
100
|
+
|
|
91
101
|
# Normalize and validate a tier name.
|
|
92
102
|
#
|
|
93
103
|
# @param tier [String, Symbol] "full", "fluency", or "mini"
|
|
@@ -202,7 +212,30 @@ module Kotoshu
|
|
|
202
212
|
# no entry for the pair
|
|
203
213
|
# @raise [Kotoshu::Error] when offline with no cached registry
|
|
204
214
|
def registry_entry_for(language_code, tier, force: false)
|
|
205
|
-
registry(force:
|
|
215
|
+
return registry(force: true)&.find(language_code.to_s, tier.to_s) if force
|
|
216
|
+
|
|
217
|
+
registry&.find(language_code.to_s, tier.to_s) ||
|
|
218
|
+
find_after_registry_refresh(language_code, tier)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Plan 135: a cached registry within its TTL shadows the live
|
|
222
|
+
# one - during the v1.7.0 train, a v1.4.0 cache made a newly
|
|
223
|
+
# published resource invisible until manual deletion. When the
|
|
224
|
+
# wanted entry is absent from the cached copy, refresh the
|
|
225
|
+
# registry ONCE and look again. An offline refresh failure keeps
|
|
226
|
+
# the original miss: the legacy fallbacks (CDN for tiers, the
|
|
227
|
+
# honest no-entry error for the typo pair) apply exactly as
|
|
228
|
+
# before.
|
|
229
|
+
def find_after_registry_refresh(language_code, tier)
|
|
230
|
+
refreshed = begin
|
|
231
|
+
registry(force: true)
|
|
232
|
+
rescue StandardError
|
|
233
|
+
# Offline or transport failure: keep the original miss, the
|
|
234
|
+
# existing fallbacks apply (Windows surfaces raw Errno
|
|
235
|
+
# classes the wrapper does not always raise as Kotoshu::Error).
|
|
236
|
+
nil
|
|
237
|
+
end
|
|
238
|
+
refreshed&.find(language_code.to_s, tier.to_s)
|
|
206
239
|
end
|
|
207
240
|
|
|
208
241
|
# Download and verify a tiered model from the registry.
|
|
@@ -224,9 +257,17 @@ module Kotoshu
|
|
|
224
257
|
def download_tiered_model(language_code, tier:, force_download: false)
|
|
225
258
|
tier = self.class.normalize_tier(tier)
|
|
226
259
|
lang = language_code.to_s
|
|
227
|
-
return get("#{lang}:onnx", force_download: force_download) if tier == :full
|
|
228
260
|
|
|
261
|
+
# The full tier rides the registry when the cached registry
|
|
262
|
+
# knows it (primary URL, vocab sibling, sha256 — the path the
|
|
263
|
+
# typo layer's rescore requires, since the legacy download
|
|
264
|
+
# never fetched the vocab). Offline without a cached registry
|
|
265
|
+
# the entry is nil and the legacy CDN path serves, preserving
|
|
266
|
+
# the pre-registry behavior exactly.
|
|
229
267
|
entry = registry_entry_for(lang, tier, force: force_download)
|
|
268
|
+
if tier == :full && entry.nil?
|
|
269
|
+
return get("#{lang}:onnx", force_download: force_download)
|
|
270
|
+
end
|
|
230
271
|
unless entry
|
|
231
272
|
raise Kotoshu::Error,
|
|
232
273
|
"no registry entry for #{ModelRegistry::Resource.id_for(lang, tier)}"
|
|
@@ -257,6 +298,7 @@ module Kotoshu
|
|
|
257
298
|
cached_at: Time.now.utc.iso8601,
|
|
258
299
|
source: "registry"
|
|
259
300
|
}
|
|
301
|
+
metadata[:vocab_file] = File.basename(vocab_path) if vocab_path
|
|
260
302
|
write_metadata(File.join(dest_path, "metadata.json"), metadata)
|
|
261
303
|
|
|
262
304
|
result = { model_path: model_file, metadata: metadata }
|
|
@@ -293,7 +335,7 @@ module Kotoshu
|
|
|
293
335
|
return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
294
336
|
return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
|
|
295
337
|
|
|
296
|
-
|
|
338
|
+
verify_model_pair_integrity!(metadata, onnx_file, vocab_file)
|
|
297
339
|
|
|
298
340
|
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
299
341
|
end
|
|
@@ -354,6 +396,191 @@ module Kotoshu
|
|
|
354
396
|
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
355
397
|
end
|
|
356
398
|
|
|
399
|
+
# ---- typo bi-encoder pair (plan 131) ----
|
|
400
|
+
|
|
401
|
+
# Whether the typo bi-encoder pair is cached and complete
|
|
402
|
+
# (both files, checksum-verified).
|
|
403
|
+
#
|
|
404
|
+
# @return [Boolean]
|
|
405
|
+
def typo_biencoder_cached?
|
|
406
|
+
!load_cached_typo_biencoder.nil?
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
# Resolve the typo bi-encoder pair from the cache only — never
|
|
410
|
+
# touches the network (the suggest path resolves; setup
|
|
411
|
+
# downloads, the same two-stage split as the LID pair).
|
|
412
|
+
#
|
|
413
|
+
# @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
|
|
414
|
+
# cached, nil when not set up
|
|
415
|
+
# @raise [Kotoshu::IntegrityError] cached bytes fail their
|
|
416
|
+
# recorded checksum
|
|
417
|
+
def load_cached_typo_biencoder
|
|
418
|
+
dir = File.join(@cache_path, TYPO_DIRECTORY)
|
|
419
|
+
metadata_path = File.join(dir, "metadata.json")
|
|
420
|
+
return nil unless File.exist?(metadata_path)
|
|
421
|
+
|
|
422
|
+
metadata = read_metadata(metadata_path)
|
|
423
|
+
return nil unless metadata
|
|
424
|
+
|
|
425
|
+
onnx_file = File.join(dir, metadata["file"].to_s)
|
|
426
|
+
vocab_file = File.join(dir, metadata["vocab_file"].to_s)
|
|
427
|
+
return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
428
|
+
return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
|
|
429
|
+
|
|
430
|
+
verify_model_pair_integrity!(
|
|
431
|
+
metadata, onnx_file, vocab_file,
|
|
432
|
+
directory: dir,
|
|
433
|
+
registry_id: TYPO_REGISTRY_ID,
|
|
434
|
+
remediation: "Run Kotoshu.setup_typo(language, force: true) to re-download."
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# Download the typo bi-encoder pair from the models registry,
|
|
441
|
+
# verifying the ONNX bytes against the registry sha256 (the
|
|
442
|
+
# setup half of the two-stage model). The vocab sidecar is
|
|
443
|
+
# required, not optional — the encoder cannot tokenize without
|
|
444
|
+
# its char vocabulary — so a registry entry without vocab_url
|
|
445
|
+
# (the pre-cut state) raises the honest error instead of
|
|
446
|
+
# caching a half-usable artifact.
|
|
447
|
+
#
|
|
448
|
+
# @param force [Boolean] re-fetch the registry first
|
|
449
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
450
|
+
# @raise [Kotoshu::Error] no entry, entry without vocab_url, or
|
|
451
|
+
# offline without a cached registry
|
|
452
|
+
# @raise [Kotoshu::IntegrityError] sha256 mismatch
|
|
453
|
+
def download_typo_biencoder(force: false)
|
|
454
|
+
entry = registry(force: force)&.find(TYPO_LANGUAGE, TYPO_TIER)
|
|
455
|
+
unless entry
|
|
456
|
+
raise Kotoshu::Error, "no registry entry for #{TYPO_REGISTRY_ID}"
|
|
457
|
+
end
|
|
458
|
+
if entry.vocab_url.nil? || entry.vocab_url.to_s.strip.empty?
|
|
459
|
+
raise Kotoshu::Error,
|
|
460
|
+
"#{TYPO_REGISTRY_ID} carries no vocab_url yet — the typo pair ships with a models registry release"
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
dir = File.join(@cache_path, TYPO_DIRECTORY)
|
|
464
|
+
FileUtils.mkdir_p(dir)
|
|
465
|
+
|
|
466
|
+
onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
|
|
467
|
+
used_url = download_primary_or_mirror!(entry, onnx_file)
|
|
468
|
+
verify_registry_sha256!(entry, onnx_file, TYPO_REGISTRY_ID, used_url,
|
|
469
|
+
remediation: "Run Kotoshu::Typo.setup(force: true) to re-download.")
|
|
470
|
+
|
|
471
|
+
vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
|
|
472
|
+
begin
|
|
473
|
+
download_file(entry.vocab_url, vocab_file)
|
|
474
|
+
rescue StandardError => e
|
|
475
|
+
File.delete(onnx_file) if File.exist?(onnx_file)
|
|
476
|
+
raise Kotoshu::Error, "typo vocab unavailable at #{entry.vocab_url}: #{e.message}"
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
metadata = {
|
|
480
|
+
version: entry.version,
|
|
481
|
+
url: used_url,
|
|
482
|
+
language: TYPO_LANGUAGE,
|
|
483
|
+
type: "typo-biencoder",
|
|
484
|
+
tier: TYPO_TIER,
|
|
485
|
+
file: File.basename(onnx_file),
|
|
486
|
+
vocab_file: File.basename(vocab_file),
|
|
487
|
+
checksum: entry.sha256,
|
|
488
|
+
registry_id: TYPO_REGISTRY_ID,
|
|
489
|
+
size_bytes: entry.size_bytes,
|
|
490
|
+
cached_at: Time.now.utc.iso8601,
|
|
491
|
+
source: "registry"
|
|
492
|
+
}
|
|
493
|
+
write_metadata(File.join(dir, "metadata.json"), metadata)
|
|
494
|
+
|
|
495
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
# Resolve a tiered model from the cache only — the cache half of
|
|
499
|
+
# the two-stage model for tiers, symmetric with
|
|
500
|
+
# {#load_cached_lid} / {#load_cached_typo_biencoder}. Returns
|
|
501
|
+
# the vocab sibling when the registry-served download landed it.
|
|
502
|
+
#
|
|
503
|
+
# @param language_code [String]
|
|
504
|
+
# @param tier [String, Symbol]
|
|
505
|
+
# @return [Hash, nil] { model_path:, vocab_path?, metadata } or nil
|
|
506
|
+
def load_cached_tier(language_code, tier)
|
|
507
|
+
tier = self.class.normalize_tier(tier)
|
|
508
|
+
resource_id = tier_resource_id(language_code.to_s, tier)
|
|
509
|
+
dir = resource_dir_for(resource_id)
|
|
510
|
+
metadata_path = File.join(dir, "metadata.json")
|
|
511
|
+
return nil unless File.exist?(metadata_path)
|
|
512
|
+
|
|
513
|
+
metadata = read_metadata(metadata_path)
|
|
514
|
+
return nil unless metadata
|
|
515
|
+
|
|
516
|
+
model_file = File.join(dir, metadata["file"].to_s)
|
|
517
|
+
return nil unless File.exist?(model_file) && File.size(model_file).positive?
|
|
518
|
+
|
|
519
|
+
result = { model_path: model_file, metadata: metadata }
|
|
520
|
+
vocab = metadata["vocab_file"].to_s
|
|
521
|
+
vocab_path = File.join(dir, vocab)
|
|
522
|
+
result[:vocab_path] = vocab_path if !vocab.empty? && File.exist?(vocab_path)
|
|
523
|
+
result
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
# ---- prebuilt typo matrix (plan 136) ----
|
|
527
|
+
|
|
528
|
+
# Resolve the language's prebuilt KTM1 matrix from the cache
|
|
529
|
+
# only (the resolve half; setup downloads).
|
|
530
|
+
#
|
|
531
|
+
# @param language_code [String]
|
|
532
|
+
# @return [String, nil] path to the cached .ktm1, nil when absent
|
|
533
|
+
def load_cached_typo_matrix(language_code)
|
|
534
|
+
entry = begin
|
|
535
|
+
registry&.find(language_code.to_s, "typo-matrix")
|
|
536
|
+
rescue StandardError
|
|
537
|
+
nil
|
|
538
|
+
end
|
|
539
|
+
return nil unless entry
|
|
540
|
+
|
|
541
|
+
dir = File.join(@cache_path, language_code.to_s, "models", "typo-matrix")
|
|
542
|
+
file = File.join(dir, filename_from_url(entry.urls.mirror))
|
|
543
|
+
return nil unless File.exist?(file) && File.size(file).positive?
|
|
544
|
+
return nil unless Digest::SHA256.file(file).hexdigest == entry.sha256
|
|
545
|
+
|
|
546
|
+
file
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Download the matrix (the setup half). Raises like the other
|
|
550
|
+
# registry downloads when the registry cannot serve it yet.
|
|
551
|
+
#
|
|
552
|
+
# @param language_code [String]
|
|
553
|
+
# @param force [Boolean]
|
|
554
|
+
# @return [String] path to the cached .ktm1
|
|
555
|
+
def download_typo_matrix(language_code, force: false)
|
|
556
|
+
lang = language_code.to_s
|
|
557
|
+
entry = registry(force: force)&.find(lang, "typo-matrix")
|
|
558
|
+
unless entry
|
|
559
|
+
raise Kotoshu::Error,
|
|
560
|
+
"no registry entry for kotoshu://models/#{lang}/typo-matrix (the matrix ships per language; derive-at-load remains the fallback)"
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
dir = File.join(@cache_path, lang, "models", "typo-matrix")
|
|
564
|
+
FileUtils.mkdir_p(dir)
|
|
565
|
+
file = File.join(dir, filename_from_url(entry.urls.mirror))
|
|
566
|
+
used_url = download_primary_or_mirror!(entry, file)
|
|
567
|
+
verify_registry_sha256!(entry, file, "kotoshu://models/#{lang}/typo-matrix", used_url,
|
|
568
|
+
remediation: "Run kotoshu setup #{lang} --typo --force to re-download.")
|
|
569
|
+
|
|
570
|
+
write_metadata(File.join(dir, "metadata.json"),
|
|
571
|
+
"version" => entry.version,
|
|
572
|
+
"url" => used_url,
|
|
573
|
+
"language" => lang,
|
|
574
|
+
"type" => "typo-matrix",
|
|
575
|
+
"file" => File.basename(file),
|
|
576
|
+
"checksum" => entry.sha256,
|
|
577
|
+
"registry_id" => "kotoshu://models/#{lang}/typo-matrix",
|
|
578
|
+
"size_bytes" => entry.size_bytes,
|
|
579
|
+
"cached_at" => Time.now.utc.iso8601,
|
|
580
|
+
"source" => "registry")
|
|
581
|
+
file
|
|
582
|
+
end
|
|
583
|
+
|
|
357
584
|
# Get available model types for a language.
|
|
358
585
|
#
|
|
359
586
|
# @param language_code [String] ISO 639-1 language code
|
|
@@ -801,16 +1028,19 @@ module Kotoshu
|
|
|
801
1028
|
# @return [void]
|
|
802
1029
|
# @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
|
|
803
1030
|
# corrupt bytes are removed so the next setup re-downloads
|
|
804
|
-
def
|
|
805
|
-
|
|
1031
|
+
def verify_model_pair_integrity!(metadata, onnx_file, vocab_file, directory: nil,
|
|
1032
|
+
registry_id: nil, remediation: nil)
|
|
1033
|
+
registry_id ||= LID_REGISTRY_ID
|
|
1034
|
+
directory ||= File.join(@cache_path, LID_DIRECTORY)
|
|
1035
|
+
remediation ||= "Run Kotoshu.setup_lid(force: true) to re-download."
|
|
806
1036
|
if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
|
|
807
1037
|
File.delete(onnx_file)
|
|
808
1038
|
File.delete(vocab_file)
|
|
809
|
-
FileUtils.rm_f(File.join(
|
|
1039
|
+
FileUtils.rm_f(File.join(directory, "metadata.json"))
|
|
810
1040
|
raise Kotoshu::Error,
|
|
811
|
-
"Integrity verification failed for #{
|
|
812
|
-
"artifact is a Git LFS pointer stub, not model content.
|
|
813
|
-
"
|
|
1041
|
+
"Integrity verification failed for #{registry_id}: cached " \
|
|
1042
|
+
"artifact is a Git LFS pointer stub, not model content. " \
|
|
1043
|
+
"#{remediation}"
|
|
814
1044
|
end
|
|
815
1045
|
|
|
816
1046
|
expected = metadata["checksum"]
|
|
@@ -820,11 +1050,11 @@ module Kotoshu
|
|
|
820
1050
|
return if actual == expected
|
|
821
1051
|
|
|
822
1052
|
raise Kotoshu::IntegrityError.new(
|
|
823
|
-
|
|
1053
|
+
registry_id,
|
|
824
1054
|
expected: expected,
|
|
825
1055
|
actual: actual,
|
|
826
1056
|
url: metadata["url"],
|
|
827
|
-
remediation:
|
|
1057
|
+
remediation: remediation
|
|
828
1058
|
)
|
|
829
1059
|
end
|
|
830
1060
|
|
data/lib/kotoshu/cli.rb
CHANGED
|
@@ -146,6 +146,7 @@ module Kotoshu
|
|
|
146
146
|
kotoshu setup en de fr # download from GitHub
|
|
147
147
|
kotoshu setup en --want spelling,frequency # also fetch Kelly list
|
|
148
148
|
kotoshu setup en --model --tier mini # model at the mini tier
|
|
149
|
+
kotoshu setup en --typo # typo bi-encoder + full tier
|
|
149
150
|
kotoshu setup en --aff /p/en.aff --dic /p/en.dic
|
|
150
151
|
kotoshu setup en --from /usr/share/hunspell/
|
|
151
152
|
kotoshu setup --force en # re-download
|
|
@@ -167,6 +168,10 @@ module Kotoshu
|
|
|
167
168
|
type: :boolean,
|
|
168
169
|
default: false,
|
|
169
170
|
desc: "Fetch the ONNX model (shorthand for adding model to --want)"
|
|
171
|
+
method_option :typo,
|
|
172
|
+
type: :boolean,
|
|
173
|
+
default: false,
|
|
174
|
+
desc: "Fetch the typo-retrieval pair (bi-encoder + full tier; needs a registry release carrying it)"
|
|
170
175
|
method_option :tier,
|
|
171
176
|
type: :string,
|
|
172
177
|
desc: "Model tier: full, fluency, or mini (default: full)"
|
|
@@ -192,6 +197,7 @@ module Kotoshu
|
|
|
192
197
|
|
|
193
198
|
want = (options[:want] || "spelling").split(",").map(&:strip).map(&:to_sym)
|
|
194
199
|
want << :model if options[:model] && !want.include?(:model)
|
|
200
|
+
want << :typo if options[:typo] && !want.include?(:typo)
|
|
195
201
|
opts = setup_source_options(languages)
|
|
196
202
|
opts[:want] = want
|
|
197
203
|
opts[:tier] = options[:tier].to_sym if options[:tier]
|
|
@@ -234,6 +240,10 @@ module Kotoshu
|
|
|
234
240
|
type: :boolean,
|
|
235
241
|
default: false,
|
|
236
242
|
desc: "Fetch the ONNX model (shorthand for adding model to --want)"
|
|
243
|
+
method_option :typo,
|
|
244
|
+
type: :boolean,
|
|
245
|
+
default: false,
|
|
246
|
+
desc: "Fetch the typo-retrieval pair (bi-encoder + full tier; needs a registry release carrying it)"
|
|
237
247
|
method_option :tier,
|
|
238
248
|
type: :string,
|
|
239
249
|
desc: "Model tier: full, fluency, or mini (default: full)"
|
|
@@ -525,8 +535,10 @@ module Kotoshu
|
|
|
525
535
|
frequency = result.frequency || "skipped"
|
|
526
536
|
model = result.model || "skipped"
|
|
527
537
|
model += " (#{result.model_tier} tier)" if result.model && result.model_tier
|
|
538
|
+
typo = result.typo || "skipped"
|
|
528
539
|
source = result.source
|
|
529
|
-
puts "OK (spelling: #{spelling}, frequency: #{frequency}, model: #{model},
|
|
540
|
+
puts "OK (spelling: #{spelling}, frequency: #{frequency}, model: #{model}, " \
|
|
541
|
+
"typo: #{typo}, source: #{source})"
|
|
530
542
|
end
|
|
531
543
|
|
|
532
544
|
def list_setup
|
|
@@ -537,7 +549,20 @@ module Kotoshu
|
|
|
537
549
|
end
|
|
538
550
|
|
|
539
551
|
puts "Set up languages:"
|
|
540
|
-
langs.each
|
|
552
|
+
langs.each do |lang|
|
|
553
|
+
parts = []
|
|
554
|
+
parts << "spelling" if Kotoshu::ResourceManager.setup?(lang)
|
|
555
|
+
parts << "frequency" if Kotoshu::ResourceManager.setup?(lang, resource: :frequency)
|
|
556
|
+
parts << "model" if Kotoshu::ResourceManager.setup?(lang, resource: :model, tier: :any)
|
|
557
|
+
parts << "typo" if Kotoshu::ResourceManager.setup?(lang, resource: :typo)
|
|
558
|
+
# Plan 139: surface the prebuilt matrix so a silent derive
|
|
559
|
+
# path is visible from `kotoshu setup --list`.
|
|
560
|
+
parts << "typo-matrix" if Kotoshu::ResourceManager.setup?(lang, resource: :typo_matrix)
|
|
561
|
+
# languages_setup is looser than setup? (TTL/metadata edge
|
|
562
|
+
# cases); fall back to the bare code when nothing verifies so
|
|
563
|
+
# we never print an empty "lang: " line.
|
|
564
|
+
puts(parts.empty? ? " #{lang}" : " #{lang}: #{parts.join(', ')}")
|
|
565
|
+
end
|
|
541
566
|
end
|
|
542
567
|
|
|
543
568
|
def display_result(result, source, application: nil)
|
|
@@ -206,10 +206,16 @@ module Kotoshu
|
|
|
206
206
|
},
|
|
207
207
|
backend: {
|
|
208
208
|
env: "KOTOSHU_BACKEND",
|
|
209
|
-
default: "
|
|
210
|
-
description: "Engine backend for correct?/suggest (auto
|
|
209
|
+
default: "auto",
|
|
210
|
+
description: "Engine backend for correct?/suggest (auto: native when the extension is available, ruby otherwise; or force either)",
|
|
211
211
|
type: String
|
|
212
212
|
},
|
|
213
|
+
typo_retrieval: {
|
|
214
|
+
env: "KOTOSHU_TYPO_RETRIEVAL",
|
|
215
|
+
default: false,
|
|
216
|
+
description: "Opt-in hybrid typo retrieval (plan 131): the native bi-encoder slate merges ahead of frequency ranking; needs the native backend",
|
|
217
|
+
type: :boolean
|
|
218
|
+
},
|
|
213
219
|
personal_dictionary: {
|
|
214
220
|
env: "KOTOSHU_PERSONAL_DICTIONARY",
|
|
215
221
|
default: true,
|
|
@@ -241,6 +247,7 @@ module Kotoshu
|
|
|
241
247
|
# conformance contract), so the default is the faster engine
|
|
242
248
|
# with the pure fallback, never a failure.
|
|
243
249
|
backend: "auto",
|
|
250
|
+
typo_retrieval: false,
|
|
244
251
|
personal_dictionary: true
|
|
245
252
|
}.freeze
|
|
246
253
|
|
|
@@ -301,6 +308,12 @@ module Kotoshu
|
|
|
301
308
|
# NativeBackend for the boundary — the semantic path stays Ruby.
|
|
302
309
|
attr_accessor :backend
|
|
303
310
|
|
|
311
|
+
# @return [Boolean] Opt-in hybrid typo retrieval (plan 131): the
|
|
312
|
+
# native bi-encoder's rescored slate merges ahead of frequency
|
|
313
|
+
# ranking when the native backend serves the language. Env:
|
|
314
|
+
# KOTOSHU_TYPO_RETRIEVAL.
|
|
315
|
+
attr_accessor :typo_retrieval
|
|
316
|
+
|
|
304
317
|
# @return [Boolean] Whether Spellchecker#check accepts words from
|
|
305
318
|
# the user personal dictionary (~/.config/kotoshu/personal.dic,
|
|
306
319
|
# KOTOSHU_PERSONAL_DIC override) instead of flagging them as
|
|
@@ -511,6 +524,7 @@ module Kotoshu
|
|
|
511
524
|
max_cache_size: @max_cache_size,
|
|
512
525
|
semantic_cascade_threshold: @semantic_cascade_threshold,
|
|
513
526
|
backend: @backend,
|
|
527
|
+
typo_retrieval: @typo_retrieval,
|
|
514
528
|
personal_dictionary: @personal_dictionary
|
|
515
529
|
}
|
|
516
530
|
end
|
|
@@ -629,6 +643,7 @@ module Kotoshu
|
|
|
629
643
|
@resource_pin = DEFAULTS[:resource_pin]
|
|
630
644
|
@semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
|
|
631
645
|
@backend = DEFAULTS[:backend]
|
|
646
|
+
@typo_retrieval = DEFAULTS[:typo_retrieval]
|
|
632
647
|
@personal_dictionary = DEFAULTS[:personal_dictionary]
|
|
633
648
|
end
|
|
634
649
|
|
|
@@ -41,6 +41,7 @@ module Kotoshu
|
|
|
41
41
|
:frequency, # :downloaded | :local | :cached | :unavailable | nil
|
|
42
42
|
:model, # :downloaded | :cached | :unavailable | nil
|
|
43
43
|
:model_tier, # :full | :fluency | :mini | nil (tier actually set up)
|
|
44
|
+
:typo, # :downloaded | :cached | :unavailable | nil (plan 131 pair)
|
|
44
45
|
:source, # :kotoshu | :local
|
|
45
46
|
keyword_init: true
|
|
46
47
|
) do
|
|
@@ -263,7 +264,7 @@ module Kotoshu
|
|
|
263
264
|
#
|
|
264
265
|
# @param language [String, Symbol] Language code
|
|
265
266
|
# @param resource [Symbol, nil] :spelling (default), :frequency,
|
|
266
|
-
# :model, or nil for "spelling"
|
|
267
|
+
# :model, :typo, :typo_matrix, or nil for "spelling"
|
|
267
268
|
# @param tier [Symbol, nil] Model tier to check (:full, :fluency,
|
|
268
269
|
# :mini, or :any); only meaningful with resource: :model
|
|
269
270
|
# @return [Boolean]
|
|
@@ -272,6 +273,7 @@ module Kotoshu
|
|
|
272
273
|
# ResourceManager.setup?(:en) # spelling
|
|
273
274
|
# ResourceManager.setup?(:en, resource: :frequency) # Kelly list
|
|
274
275
|
# ResourceManager.setup?(:en, resource: :model, tier: :mini)
|
|
276
|
+
# ResourceManager.setup?(:en, resource: :typo_matrix)
|
|
275
277
|
def setup?(language, resource: nil, tier: nil)
|
|
276
278
|
lang = normalize_language(language)
|
|
277
279
|
case resource&.to_sym
|
|
@@ -280,6 +282,16 @@ module Kotoshu
|
|
|
280
282
|
when :frequency
|
|
281
283
|
fc = frequency_cache_for
|
|
282
284
|
fc.supports_resource?(lang) && fc.available?(lang)
|
|
285
|
+
when :typo
|
|
286
|
+
# Both halves: the language-less pair and the language's full
|
|
287
|
+
# tier (the rescore's exactness contract).
|
|
288
|
+
cache = model_cache_for
|
|
289
|
+
cache.typo_biencoder_cached? && cache.available?(cache.tier_resource_id(lang, :full))
|
|
290
|
+
when :typo_matrix
|
|
291
|
+
# Plan 139: the prebuilt KTM1 is a separate, optional half of
|
|
292
|
+
# the typo setup. Absent means arming derives (~25 s); present
|
|
293
|
+
# means arming is a download (ms).
|
|
294
|
+
!model_cache_for.load_cached_typo_matrix(lang).nil?
|
|
283
295
|
when :model
|
|
284
296
|
cache = model_cache_for
|
|
285
297
|
if tier.nil?
|
|
@@ -343,16 +355,75 @@ module Kotoshu
|
|
|
343
355
|
end
|
|
344
356
|
end
|
|
345
357
|
|
|
358
|
+
typo_status = if want.include?(:typo)
|
|
359
|
+
setup_typo_remote(lang, force: force, strict: strict, config: config,
|
|
360
|
+
tier: model_tier)
|
|
361
|
+
end
|
|
362
|
+
|
|
346
363
|
SetupResult.new(
|
|
347
364
|
language: lang,
|
|
348
365
|
spelling: spelling_status,
|
|
349
366
|
frequency: frequency_status,
|
|
350
367
|
model: model_status,
|
|
351
368
|
model_tier: model_tier,
|
|
369
|
+
typo: typo_status,
|
|
352
370
|
source: :kotoshu
|
|
353
371
|
)
|
|
354
372
|
end
|
|
355
373
|
|
|
374
|
+
# The typo layer's setup half (plan 131): the language-less
|
|
375
|
+
# bi-encoder pair, plus the language's FULL tier when this run did
|
|
376
|
+
# not already fetch it (the hybrid's rescore is full-tier-exact by
|
|
377
|
+
# contract — a mini tier would rank wrong, so full is fetched
|
|
378
|
+
# explicitly, not defaulted).
|
|
379
|
+
#
|
|
380
|
+
# Returns :cached when both halves were already present,
|
|
381
|
+
# :downloaded when anything was fetched, :unavailable when the
|
|
382
|
+
# registry cannot serve the pair yet (strict: raises instead —
|
|
383
|
+
# the pre-cut state).
|
|
384
|
+
def setup_typo_remote(lang, force:, strict:, config:, tier: nil)
|
|
385
|
+
cache = model_cache_for(config: config)
|
|
386
|
+
unless force
|
|
387
|
+
full_id = cache.tier_resource_id(lang, :full)
|
|
388
|
+
if cache.typo_biencoder_cached? && cache.available?(full_id)
|
|
389
|
+
return ensure_typo_matrix(cache, lang) || :cached
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
begin
|
|
393
|
+
cache.download_typo_biencoder(force: force)
|
|
394
|
+
cache.download_tiered_model(lang, tier: :full, force_download: force) unless tier == :full
|
|
395
|
+
rescue Kotoshu::Error
|
|
396
|
+
raise if strict
|
|
397
|
+
|
|
398
|
+
return :unavailable
|
|
399
|
+
rescue StandardError => e
|
|
400
|
+
# Windows surfaces closed-port fetches as raw Errno classes the
|
|
401
|
+
# cache does not always wrap; the two-stage contract is
|
|
402
|
+
# setup-degrades unless strict, and strict reports through the
|
|
403
|
+
# typed error like every setup path.
|
|
404
|
+
raise Kotoshu::Error, "typo setup failed: #{e.class}: #{e.message}" if strict
|
|
405
|
+
|
|
406
|
+
return :unavailable
|
|
407
|
+
end
|
|
408
|
+
ensure_typo_matrix(cache, lang)
|
|
409
|
+
:downloaded
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# Plan 136: an already-satisfied (or just-satisfied) typo setup
|
|
413
|
+
# still owes the user the instant-arming matrix — without this an
|
|
414
|
+
# install set up before the matrix existed derives at arming time
|
|
415
|
+
# (~25s) forever. Cache-only when present; best-effort fetch
|
|
416
|
+
# otherwise. Returns :downloaded when fetched, nil when present or
|
|
417
|
+
# unservable — callers degrade, never fail, on nil.
|
|
418
|
+
def ensure_typo_matrix(cache, lang)
|
|
419
|
+
return nil if cache.load_cached_typo_matrix(lang)
|
|
420
|
+
|
|
421
|
+
cache.download_typo_matrix(lang)
|
|
422
|
+
:downloaded
|
|
423
|
+
rescue Kotoshu::Error, StandardError
|
|
424
|
+
nil
|
|
425
|
+
end
|
|
426
|
+
|
|
356
427
|
def setup_frequency_remote(lang, force:, strict:, config:)
|
|
357
428
|
cache = frequency_cache_for(config: config)
|
|
358
429
|
return :unavailable unless cache.supports_resource?(lang)
|
data/lib/kotoshu/spellchecker.rb
CHANGED
|
@@ -140,9 +140,39 @@ module Kotoshu
|
|
|
140
140
|
def suggest(word, max_suggestions: nil)
|
|
141
141
|
return Suggestions::SuggestionSet.empty if word.nil? || word.empty?
|
|
142
142
|
|
|
143
|
-
|
|
143
|
+
base = if @native_backend
|
|
144
|
+
@native_backend.suggest(word, max_suggestions: max_suggestions)
|
|
145
|
+
else
|
|
146
|
+
@generator.generate(word, max_suggestions: max_suggestions)
|
|
147
|
+
end
|
|
148
|
+
merge_typo_retrieval(base, word, max_suggestions: max_suggestions)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Merge the opt-in typo-retrieval slate (plan 131) AHEAD of the
|
|
152
|
+
# base suggestions: the bi-encoder surfaces corrections a
|
|
153
|
+
# frequency-ranked sweep misses, so its rows lead. Inert unless
|
|
154
|
+
# typo_retrieval is configured AND the native engine loaded for
|
|
155
|
+
# this language — a disabled layer is a passthrough, never an
|
|
156
|
+
# error, never a different order.
|
|
157
|
+
def merge_typo_retrieval(base, word, max_suggestions: nil)
|
|
158
|
+
return base unless @config.typo_retrieval
|
|
159
|
+
|
|
160
|
+
engine = typo_retrieval_engine
|
|
161
|
+
return base unless engine
|
|
162
|
+
|
|
163
|
+
typo = engine.suggest(word, max_suggestions: max_suggestions)
|
|
164
|
+
limit = max_suggestions || @config.max_suggestions
|
|
165
|
+
Suggestions::TypoMerge.call(base: base, typo: typo, limit: limit)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# The typo engine for this language, resolved once. A nil result
|
|
169
|
+
# is memoized too — an unavailable layer must not retry its
|
|
170
|
+
# artifact resolution on every suggestion.
|
|
171
|
+
def typo_retrieval_engine
|
|
172
|
+
return @typo_retrieval_engine if defined?(@typo_retrieval_engine)
|
|
144
173
|
|
|
145
|
-
@
|
|
174
|
+
code = (@resource_bundle&.language || @config.language).to_s.split("-").first
|
|
175
|
+
@typo_retrieval_engine = Kotoshu::Typo::Engine.for(code, configuration: @config)
|
|
146
176
|
end
|
|
147
177
|
|
|
148
178
|
# Check a word and return a result object.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# The hybrid typo-retrieval strategy (plan 131): a frozen char
|
|
7
|
+
# bi-encoder retrieves dictionary candidates a frequency-ranked
|
|
8
|
+
# sweep misses, the fastText full tier rescores them, and the
|
|
9
|
+
# rows join the suggestion pipeline AHEAD of frequency ranking
|
|
10
|
+
# — the layer measured at +6.3 pp top-5 on the frozen
|
|
11
|
+
# benchmark. Opt-in and native-only; the engine is injected so
|
|
12
|
+
# the strategy itself stays pure orchestration.
|
|
13
|
+
class TypoRetrievalStrategy < BaseStrategy
|
|
14
|
+
# @return [#suggest] the typo engine ({Typo::Engine} in
|
|
15
|
+
# production; anything answering #suggest(word) in specs)
|
|
16
|
+
attr_reader :engine
|
|
17
|
+
|
|
18
|
+
# @param engine [#suggest] the typo engine
|
|
19
|
+
# @param config [Hash] additional configuration
|
|
20
|
+
def initialize(engine:, **config)
|
|
21
|
+
super(name: :typo_retrieval, **config)
|
|
22
|
+
@engine = engine
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Generate the typo-retrieval slate for the context's word.
|
|
26
|
+
#
|
|
27
|
+
# @param context [Context] the suggestion context
|
|
28
|
+
# @return [SuggestionSet]
|
|
29
|
+
def generate(context)
|
|
30
|
+
return SuggestionSet.empty unless enabled? && @engine
|
|
31
|
+
|
|
32
|
+
@engine.suggest(context.word, max_suggestions: max_results)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# This strategy is a candidate generator for misspellings; it
|
|
36
|
+
# has no real-word-error story, which the semantic strategy
|
|
37
|
+
# owns.
|
|
38
|
+
#
|
|
39
|
+
# @param context [Context]
|
|
40
|
+
# @return [Boolean]
|
|
41
|
+
def handles?(context)
|
|
42
|
+
enabled? && @engine && !context.word.empty?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The strategy's rows already carry their own order (the
|
|
46
|
+
# rescored ranking); do not let a confident base set skip it.
|
|
47
|
+
#
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
def skip_when_confident?
|
|
50
|
+
false
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# The typo-retrieval merge (plan 131): the bi-encoder's rescored
|
|
6
|
+
# slate joins AHEAD of the base suggestions — it surfaces
|
|
7
|
+
# corrections a frequency-ranked sweep misses — with duplicates
|
|
8
|
+
# (same word) dropped from the base side and the set capped at the
|
|
9
|
+
# caller's limit. A pure function over real SuggestionSets; the
|
|
10
|
+
# spellchecker calls it only when the opt-in layer is armed.
|
|
11
|
+
module TypoMerge
|
|
12
|
+
# @param base [SuggestionSet] the engine's own suggestions
|
|
13
|
+
# @param typo [SuggestionSet] the typo-retrieval slate
|
|
14
|
+
# @param limit [Integer] final cap
|
|
15
|
+
# @return [SuggestionSet] merged set, order adopted verbatim
|
|
16
|
+
def self.call(base:, typo:, limit:)
|
|
17
|
+
return base if typo.empty?
|
|
18
|
+
|
|
19
|
+
typo_words = typo.map(&:word)
|
|
20
|
+
merged = typo.suggestions +
|
|
21
|
+
base.suggestions.reject { |suggestion| typo_words.include?(suggestion.word) }
|
|
22
|
+
SuggestionSet.new(merged.first(limit), max_size: limit, ranked: true)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|