kotoshu 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +53 -0
- data/Cargo.lock +72 -4
- data/TODO.impl/00-vision.md +37 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/ext/kotoshu_native/Cargo.toml +5 -2
- data/ext/kotoshu_native/src/lib.rs +134 -6
- data/lib/kotoshu/cache/language_cache.rb +4 -1
- data/lib/kotoshu/cache/model_cache.rb +150 -2
- data/lib/kotoshu/cli.rb +8 -0
- data/lib/kotoshu/configuration.rb +20 -2
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +78 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +47 -1
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/registry.rb +20 -4
- data/lib/kotoshu/keyboard.rb +9 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language.rb +4 -0
- data/lib/kotoshu/languages/ar/language.rb +14 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +16 -0
- data/lib/kotoshu/languages/he/language.rb +15 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +38 -0
- data/lib/kotoshu/languages.rb +12 -0
- data/lib/kotoshu/spellchecker.rb +35 -0
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +45 -5
- data/scripts/extract_eval_grids.rb +4 -1
- data/sig/kotoshu.rbs +45 -0
- metadata +25 -1
|
@@ -1,15 +1,143 @@
|
|
|
1
1
|
//! Kotoshu native extension (plan 66 / TODO.impl 05, P4b).
|
|
2
2
|
//!
|
|
3
|
-
//! The
|
|
4
|
-
//!
|
|
5
|
-
//!
|
|
6
|
-
//!
|
|
3
|
+
//! The extension forwards its `#[magnus::init]` to the engine core's
|
|
4
|
+
//! Ruby bindings. `kotoshu::ffi::ruby::init` defines the `Kotoshu::Native`
|
|
5
|
+
//! module (`VERSION`, `available?`, `Dictionary.load` with `correct?` /
|
|
6
|
+
//! `suggest`); this crate contains no engine logic of its own,
|
|
7
7
|
//! exactly like kotoshu-rs' `tests/ruby_ext` reference shim and
|
|
8
8
|
//! parsanol-ruby's `ext/parsanol_native`.
|
|
9
|
+
//!
|
|
10
|
+
//! One binding lives here instead of the core crate: the language
|
|
11
|
+
//! identification surface (plan 106). `LidModel.load` +
|
|
12
|
+
//! `LidModel#detect` wrap `kotoshu::lid::LidModel` — the pure-Rust
|
|
13
|
+
//! reader for the `kotoshu://models/lid/lid-176` artifact pair —
|
|
14
|
+
//! behind the same `model` feature the core crate ships it under
|
|
15
|
+
//! (see the features table in Cargo.toml). Only the magnus glue is
|
|
16
|
+
//! gem-side; every scoring decision stays in the core, where the
|
|
17
|
+
//! parity suite freezes it against the shared frozen corpus.
|
|
18
|
+
|
|
19
|
+
use std::path::Path;
|
|
20
|
+
|
|
21
|
+
use magnus::typed_data::Obj;
|
|
22
|
+
use magnus::typed_data::TypedData;
|
|
23
|
+
use magnus::value::Lazy;
|
|
24
|
+
use magnus::{
|
|
25
|
+
Class, DataType, DataTypeFunctions, Error, ExceptionClass, Module, Object, RClass, RHash,
|
|
26
|
+
RModule, Ruby, data_type_builder, method,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/// The `Kotoshu` module, defined (idempotently) on first access — the
|
|
30
|
+
/// same helper shape as the core crate's `ffi::ruby`.
|
|
31
|
+
fn kotoshu_module(ruby: &Ruby) -> RModule {
|
|
32
|
+
static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
|
|
33
|
+
ruby.define_module("Kotoshu")
|
|
34
|
+
.expect("cannot define Kotoshu module")
|
|
35
|
+
});
|
|
36
|
+
ruby.get_inner(&MODULE)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// The `Kotoshu::Native` module hosting the engine bindings.
|
|
40
|
+
fn native_module(ruby: &Ruby) -> RModule {
|
|
41
|
+
static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
|
|
42
|
+
kotoshu_module(ruby)
|
|
43
|
+
.define_module("Native")
|
|
44
|
+
.expect("cannot define Kotoshu::Native module")
|
|
45
|
+
});
|
|
46
|
+
ruby.get_inner(&MODULE)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/// `Kotoshu::Native::Error`, defined by the core crate's init — every
|
|
50
|
+
/// LID failure crosses the boundary as the same exception class the
|
|
51
|
+
/// dictionary surface raises.
|
|
52
|
+
fn error_class(ruby: &Ruby) -> ExceptionClass {
|
|
53
|
+
static CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
|
|
54
|
+
native_module(ruby)
|
|
55
|
+
.const_get::<_, ExceptionClass>("Error")
|
|
56
|
+
.expect("Kotoshu::Native::Error not defined after ffi::ruby::init")
|
|
57
|
+
});
|
|
58
|
+
ruby.get_inner(&CLASS)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// A loaded LID model wrapped as a Ruby object: the twin of the core
|
|
62
|
+
/// crate's `RubyDictionary`. Ruby's GC owns it once wrapped; dropping
|
|
63
|
+
/// the Ruby object drops the engine model (~1 MB of int8 rows plus
|
|
64
|
+
/// ~1.8 MB of sidecar tables, roughly the artifact pair's own size).
|
|
65
|
+
#[derive(Debug)]
|
|
66
|
+
pub struct RubyLidModel {
|
|
67
|
+
inner: kotoshu::lid::LidModel,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Plain owned data; Ruby objects move between threads only with the
|
|
71
|
+
// GVL held, which TypedData's Send bound models.
|
|
72
|
+
impl DataTypeFunctions for RubyLidModel {}
|
|
73
|
+
|
|
74
|
+
unsafe impl TypedData for RubyLidModel {
|
|
75
|
+
fn class(ruby: &Ruby) -> RClass {
|
|
76
|
+
static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
|
|
77
|
+
let class = native_module(ruby)
|
|
78
|
+
.define_class("LidModel", ruby.class_object())
|
|
79
|
+
.expect("cannot define Kotoshu::Native::LidModel");
|
|
80
|
+
// Instances exist only through `LidModel.load`; `new`/
|
|
81
|
+
// `allocate` would produce a model-less object.
|
|
82
|
+
class.undef_default_alloc_func();
|
|
83
|
+
class
|
|
84
|
+
});
|
|
85
|
+
ruby.get_inner(&CLASS)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
fn data_type() -> &'static DataType {
|
|
89
|
+
static DATA_TYPE: DataType =
|
|
90
|
+
data_type_builder!(RubyLidModel, "Kotoshu/Native/LidModel").build();
|
|
91
|
+
&DATA_TYPE
|
|
92
|
+
}
|
|
93
|
+
}
|
|
9
94
|
|
|
10
|
-
|
|
95
|
+
/// `Kotoshu::Native::LidModel.load(onnx_path, vocab_path)` — reads the
|
|
96
|
+
/// registry artifact pair (the `lid.176.onnx` container and its
|
|
97
|
+
/// `lid.176.vocab.json` sidecar) and wraps the parsed model. Failures
|
|
98
|
+
/// raise `Kotoshu::Native::Error` carrying the Rust error message,
|
|
99
|
+
/// mirroring `Dictionary.load`.
|
|
100
|
+
fn lid_model_load(
|
|
101
|
+
ruby: &Ruby,
|
|
102
|
+
_class: RClass,
|
|
103
|
+
onnx_path: String,
|
|
104
|
+
vocab_path: String,
|
|
105
|
+
) -> Result<Obj<RubyLidModel>, Error> {
|
|
106
|
+
let fail = |message: String| {
|
|
107
|
+
Error::new(
|
|
108
|
+
error_class(ruby),
|
|
109
|
+
format!("failed to load LID model ({onnx_path}, {vocab_path}): {message}"),
|
|
110
|
+
)
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
let onnx_bytes =
|
|
114
|
+
std::fs::read(Path::new(&onnx_path)).map_err(|error| fail(error.to_string()))?;
|
|
115
|
+
let vocab_bytes =
|
|
116
|
+
std::fs::read(Path::new(&vocab_path)).map_err(|error| fail(error.to_string()))?;
|
|
117
|
+
let model = kotoshu::lid::LidModel::parse(&onnx_bytes, &vocab_bytes)
|
|
118
|
+
.map_err(|error| fail(error.to_string()))?;
|
|
119
|
+
Ok(ruby.obj_wrap(RubyLidModel { inner: model }))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/// `Kotoshu::Native::LidModel#detect(text)` — the top label and its
|
|
123
|
+
/// probability as `{ "code" => String, "score" => Float }`, the gem's
|
|
124
|
+
/// `LanguageIdentifier` detection pair (labels equal and scores within
|
|
125
|
+
/// 5e-4 of the fastText bindings; frozen by the kotoshu-rs parity
|
|
126
|
+
/// suite over the shared corpus fixture).
|
|
127
|
+
fn lid_model_detect(ruby: &Ruby, rb_self: &RubyLidModel, text: String) -> Result<RHash, Error> {
|
|
128
|
+
let detection = rb_self.inner.detect(&text);
|
|
129
|
+
let hash = ruby.hash_new();
|
|
130
|
+
hash.aset("code", detection.code.as_str())?;
|
|
131
|
+
hash.aset("score", detection.score)?;
|
|
132
|
+
Ok(hash)
|
|
133
|
+
}
|
|
11
134
|
|
|
12
135
|
#[magnus::init]
|
|
13
136
|
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
14
|
-
kotoshu::ffi::ruby::init(ruby)
|
|
137
|
+
kotoshu::ffi::ruby::init(ruby)?;
|
|
138
|
+
|
|
139
|
+
let class = RubyLidModel::class(ruby);
|
|
140
|
+
class.define_singleton_method("load", method!(lid_model_load, 2))?;
|
|
141
|
+
class.define_method("detect", method!(lid_model_detect, 1))?;
|
|
142
|
+
Ok(())
|
|
15
143
|
}
|
|
@@ -31,8 +31,11 @@ module Kotoshu
|
|
|
31
31
|
# modules (lib/kotoshu/languages/). Staged-only dictionaries in
|
|
32
32
|
# the dictionaries repo stay outside this list until their
|
|
33
33
|
# module lands; local files come via setup --aff/--dic instead.
|
|
34
|
+
# Batch 3 (plan 100) promotes the RTL wiring ar/fa/he and adds
|
|
35
|
+
# bg et hr id lt lv sk sl sr.
|
|
34
36
|
AVAILABLE_LANGUAGES = %w[
|
|
35
|
-
ca cs da de el en es
|
|
37
|
+
ar bg ca cs da de el en es et fa fr he hr hu id it lt lv nb nl
|
|
38
|
+
pl pt ro ru sk sl sr sv tr uk vi
|
|
36
39
|
].freeze
|
|
37
40
|
|
|
38
41
|
# Get or download spelling dictionary for a language.
|
|
@@ -72,6 +72,22 @@ module Kotoshu
|
|
|
72
72
|
# output is stable across machines and runs.
|
|
73
73
|
TIER_PREFERENCE = %i[mini fluency full].freeze
|
|
74
74
|
|
|
75
|
+
# The language-identification artifact pair (plan 106): registry
|
|
76
|
+
# resource `kotoshu://models/lid/lid-176` — the ONNX-converted
|
|
77
|
+
# lid.176 model (int8-per-row, dim 16) plus its vocab.json
|
|
78
|
+
# sidecar. Lives in the same registry as the embedding tiers but
|
|
79
|
+
# is NOT a language tier: it has no language of its own and the
|
|
80
|
+
# tier machinery (resource ids, TIERS) does not apply. It is
|
|
81
|
+
# stored under a top-level `lid/` directory (not
|
|
82
|
+
# `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
|
|
83
|
+
# reporting per-language ids only.
|
|
84
|
+
LID_LANGUAGE = "lid"
|
|
85
|
+
LID_TIER = "lid-176"
|
|
86
|
+
LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
|
|
87
|
+
|
|
88
|
+
# Directory (under the cache root) holding the LID artifact pair.
|
|
89
|
+
LID_DIRECTORY = "lid"
|
|
90
|
+
|
|
75
91
|
# Normalize and validate a tier name.
|
|
76
92
|
#
|
|
77
93
|
# @param tier [String, Symbol] "full", "fluency", or "mini"
|
|
@@ -248,6 +264,96 @@ module Kotoshu
|
|
|
248
264
|
result
|
|
249
265
|
end
|
|
250
266
|
|
|
267
|
+
# ---- LID artifact pair (plan 106) ----
|
|
268
|
+
|
|
269
|
+
# Whether the LID artifact pair is cached and complete.
|
|
270
|
+
#
|
|
271
|
+
# @return [Boolean]
|
|
272
|
+
def lid_cached?
|
|
273
|
+
!load_cached_lid.nil?
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Resolve the LID artifact pair from the cache only — never
|
|
277
|
+
# touches the network (the resolve half of the two-stage model).
|
|
278
|
+
#
|
|
279
|
+
# @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
|
|
280
|
+
# cached, nil when not set up
|
|
281
|
+
# @raise [Kotoshu::IntegrityError] cached bytes fail their
|
|
282
|
+
# recorded checksum
|
|
283
|
+
def load_cached_lid
|
|
284
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
285
|
+
metadata_path = File.join(dir, "metadata.json")
|
|
286
|
+
return nil unless File.exist?(metadata_path)
|
|
287
|
+
|
|
288
|
+
metadata = read_metadata(metadata_path)
|
|
289
|
+
return nil unless metadata
|
|
290
|
+
|
|
291
|
+
onnx_file = File.join(dir, metadata["file"].to_s)
|
|
292
|
+
vocab_file = File.join(dir, metadata["vocab_file"].to_s)
|
|
293
|
+
return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
294
|
+
return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
|
|
295
|
+
|
|
296
|
+
verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
297
|
+
|
|
298
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Download the LID artifact pair from the models registry
|
|
302
|
+
# (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
|
|
303
|
+
# against the registry sha256 before accepting them (the setup
|
|
304
|
+
# half of the two-stage model; same contract as
|
|
305
|
+
# {#download_tiered_model}).
|
|
306
|
+
#
|
|
307
|
+
# @param force [Boolean] re-fetch the registry first
|
|
308
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
309
|
+
# @raise [Kotoshu::Error] no registry entry, or offline without
|
|
310
|
+
# a cached registry (the registry is fetched once per cache
|
|
311
|
+
# instance; see {#registry})
|
|
312
|
+
# @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
|
|
313
|
+
# bytes are removed before raising)
|
|
314
|
+
def download_lid(force: false)
|
|
315
|
+
entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
|
|
316
|
+
unless entry
|
|
317
|
+
raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
321
|
+
FileUtils.mkdir_p(dir)
|
|
322
|
+
|
|
323
|
+
onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
|
|
324
|
+
used_url = download_primary_or_mirror!(entry, onnx_file)
|
|
325
|
+
verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
|
|
326
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
|
|
327
|
+
|
|
328
|
+
# The vocab sidecar is required, not optional: the reader
|
|
329
|
+
# cannot score without its labels and Huffman counts.
|
|
330
|
+
vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
|
|
331
|
+
begin
|
|
332
|
+
download_file(entry.vocab_url, vocab_file)
|
|
333
|
+
rescue StandardError => e
|
|
334
|
+
File.delete(onnx_file) if File.exist?(onnx_file)
|
|
335
|
+
raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
metadata = {
|
|
339
|
+
version: entry.version,
|
|
340
|
+
url: used_url,
|
|
341
|
+
language: LID_LANGUAGE,
|
|
342
|
+
type: "lid",
|
|
343
|
+
tier: LID_TIER,
|
|
344
|
+
file: File.basename(onnx_file),
|
|
345
|
+
vocab_file: File.basename(vocab_file),
|
|
346
|
+
checksum: entry.sha256,
|
|
347
|
+
registry_id: LID_REGISTRY_ID,
|
|
348
|
+
size_bytes: entry.size_bytes,
|
|
349
|
+
cached_at: Time.now.utc.iso8601,
|
|
350
|
+
source: "registry"
|
|
351
|
+
}
|
|
352
|
+
write_metadata(File.join(dir, "metadata.json"), metadata)
|
|
353
|
+
|
|
354
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
355
|
+
end
|
|
356
|
+
|
|
251
357
|
# Get available model types for a language.
|
|
252
358
|
#
|
|
253
359
|
# @param language_code [String] ISO 639-1 language code
|
|
@@ -629,7 +735,9 @@ module Kotoshu
|
|
|
629
735
|
# Verify downloaded model bytes against the registry's sha256.
|
|
630
736
|
# Corrupt bytes are removed from disk before raising so the next
|
|
631
737
|
# attempt re-downloads (same contract as BaseCache#verify_and_audit).
|
|
632
|
-
|
|
738
|
+
# +remediation+ overrides the default re-download hint (the LID
|
|
739
|
+
# pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
|
|
740
|
+
def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
|
|
633
741
|
actual = Digest::SHA256.file(model_file).hexdigest
|
|
634
742
|
if actual == entry.sha256
|
|
635
743
|
@audit_log.record(
|
|
@@ -649,7 +757,8 @@ module Kotoshu
|
|
|
649
757
|
expected: entry.sha256,
|
|
650
758
|
actual: actual,
|
|
651
759
|
url: url,
|
|
652
|
-
remediation:
|
|
760
|
+
remediation: remediation ||
|
|
761
|
+
"Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
|
|
653
762
|
)
|
|
654
763
|
end
|
|
655
764
|
|
|
@@ -680,6 +789,45 @@ module Kotoshu
|
|
|
680
789
|
# Caches written without a checksum field are accepted silently
|
|
681
790
|
# to preserve backward compatibility with pre-verification caches.
|
|
682
791
|
#
|
|
792
|
+
# Verify the cached LID pair against the checksum recorded at
|
|
793
|
+
# download time (which, for registry downloads, is the registry's
|
|
794
|
+
# sha256). A Git LFS pointer stub is refused by content first —
|
|
795
|
+
# same contract as {#verify_cached_integrity!}, against the
|
|
796
|
+
# fixed LID paths instead of a resource-id-derived layout.
|
|
797
|
+
#
|
|
798
|
+
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
799
|
+
# @param onnx_file [String] Path to the cached lid.176.onnx
|
|
800
|
+
# @param vocab_file [String] Path to the cached vocab sidecar
|
|
801
|
+
# @return [void]
|
|
802
|
+
# @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
|
|
803
|
+
# corrupt bytes are removed so the next setup re-downloads
|
|
804
|
+
def verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
805
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
806
|
+
if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
|
|
807
|
+
File.delete(onnx_file)
|
|
808
|
+
File.delete(vocab_file)
|
|
809
|
+
FileUtils.rm_f(File.join(dir, "metadata.json"))
|
|
810
|
+
raise Kotoshu::Error,
|
|
811
|
+
"Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
|
|
812
|
+
"artifact is a Git LFS pointer stub, not model content. Run " \
|
|
813
|
+
"Kotoshu.setup_lid(force: true) to re-download."
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
expected = metadata["checksum"]
|
|
817
|
+
return unless expected
|
|
818
|
+
|
|
819
|
+
actual = Digest::SHA256.file(onnx_file).hexdigest
|
|
820
|
+
return if actual == expected
|
|
821
|
+
|
|
822
|
+
raise Kotoshu::IntegrityError.new(
|
|
823
|
+
LID_REGISTRY_ID,
|
|
824
|
+
expected: expected,
|
|
825
|
+
actual: actual,
|
|
826
|
+
url: metadata["url"],
|
|
827
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
|
|
828
|
+
)
|
|
829
|
+
end
|
|
830
|
+
|
|
683
831
|
# @param resource_id [String] The resource identifier (e.g. "en:onnx")
|
|
684
832
|
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
685
833
|
# @param model_file [String] Path to the cached model file
|
data/lib/kotoshu/cli.rb
CHANGED
|
@@ -162,6 +162,10 @@ module Kotoshu
|
|
|
162
162
|
method_option :exclude,
|
|
163
163
|
type: :array,
|
|
164
164
|
desc: "Skip files matching these globs (always wins over --include and the extension default)"
|
|
165
|
+
method_option :personal,
|
|
166
|
+
type: :boolean,
|
|
167
|
+
default: true,
|
|
168
|
+
desc: "Accept words from the personal dictionary (personal.dic); --no-personal flags them again"
|
|
165
169
|
# Check spelling in files, directories, or stdin. Cache-only —
|
|
166
170
|
# never downloads; run `kotoshu setup LANG` first. See the
|
|
167
171
|
# command long description (kotoshu help check) for flags,
|
|
@@ -395,6 +399,10 @@ module Kotoshu
|
|
|
395
399
|
Kotoshu::Configuration.reset
|
|
396
400
|
cfg = Kotoshu::Configuration.instance
|
|
397
401
|
cfg.default_language = options[:language] if options[:language] && options[:language] != "auto"
|
|
402
|
+
# `kotoshu check --no-personal` opts out of the personal
|
|
403
|
+
# dictionary for this run (plan 105). Other commands do not
|
|
404
|
+
# declare the flag, so options[:personal] is nil there.
|
|
405
|
+
cfg.personal_dictionary = options[:personal] unless options[:personal].nil?
|
|
398
406
|
end
|
|
399
407
|
|
|
400
408
|
# Install a ProgressReporter on Configuration.download_reporter
|
|
@@ -209,6 +209,12 @@ module Kotoshu
|
|
|
209
209
|
default: "ruby",
|
|
210
210
|
description: "Engine backend for correct?/suggest (ruby, native, or auto)",
|
|
211
211
|
type: String
|
|
212
|
+
},
|
|
213
|
+
personal_dictionary: {
|
|
214
|
+
env: "KOTOSHU_PERSONAL_DICTIONARY",
|
|
215
|
+
default: true,
|
|
216
|
+
description: "Words in the personal dictionary (personal.dic) pass the check path",
|
|
217
|
+
type: :boolean
|
|
212
218
|
}
|
|
213
219
|
}.freeze
|
|
214
220
|
|
|
@@ -229,7 +235,8 @@ module Kotoshu
|
|
|
229
235
|
default_language: "en",
|
|
230
236
|
resource_pin: "main",
|
|
231
237
|
semantic_cascade_threshold: 1.0,
|
|
232
|
-
backend: "ruby"
|
|
238
|
+
backend: "ruby",
|
|
239
|
+
personal_dictionary: true
|
|
233
240
|
}.freeze
|
|
234
241
|
|
|
235
242
|
# @return [String, nil] Path to the dictionary file
|
|
@@ -289,6 +296,15 @@ module Kotoshu
|
|
|
289
296
|
# NativeBackend for the boundary — the semantic path stays Ruby.
|
|
290
297
|
attr_accessor :backend
|
|
291
298
|
|
|
299
|
+
# @return [Boolean] Whether Spellchecker#check accepts words from
|
|
300
|
+
# the user personal dictionary (~/.config/kotoshu/personal.dic,
|
|
301
|
+
# KOTOSHU_PERSONAL_DIC override) instead of flagging them as
|
|
302
|
+
# errors — the README story and the LSP diagnostics behavior
|
|
303
|
+
# (plan 105). Opt out with KOTOSHU_PERSONAL_DICTIONARY=false or
|
|
304
|
+
# `kotoshu check --no-personal`; the suggest path is unaffected,
|
|
305
|
+
# so personal words stay spellcheckable.
|
|
306
|
+
attr_accessor :personal_dictionary
|
|
307
|
+
|
|
292
308
|
# @return [String, nil] Path to cache directory
|
|
293
309
|
attr_accessor :cache_path
|
|
294
310
|
|
|
@@ -489,7 +505,8 @@ module Kotoshu
|
|
|
489
505
|
cache_ttl: @cache_ttl,
|
|
490
506
|
max_cache_size: @max_cache_size,
|
|
491
507
|
semantic_cascade_threshold: @semantic_cascade_threshold,
|
|
492
|
-
backend: @backend
|
|
508
|
+
backend: @backend,
|
|
509
|
+
personal_dictionary: @personal_dictionary
|
|
493
510
|
}
|
|
494
511
|
end
|
|
495
512
|
|
|
@@ -607,6 +624,7 @@ module Kotoshu
|
|
|
607
624
|
@resource_pin = DEFAULTS[:resource_pin]
|
|
608
625
|
@semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
|
|
609
626
|
@backend = DEFAULTS[:backend]
|
|
627
|
+
@personal_dictionary = DEFAULTS[:personal_dictionary]
|
|
610
628
|
end
|
|
611
629
|
|
|
612
630
|
# Apply resolved values from the resolver (ENV, defaults).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# Arabic 101 keyboard layout (standard IBM PC Arabic layout).
|
|
7
|
+
#
|
|
8
|
+
# The Arabic letters sit on the QWERTY key faces: the number row
|
|
9
|
+
# is shared with ASCII, and ض ص ث ق ف غ ع ه خ ح ج د / ش س ي ب ل ا
|
|
10
|
+
# ت ن م ك ط / ئ ء ؤ ر لا ى ة و ز ظ fill the three letter rows.
|
|
11
|
+
# Diacritics (tashkeel) are dead keys and are not in the grid;
|
|
12
|
+
# the hamza carriers أ إ آ and the ه/ة ي/ى spelling slips are
|
|
13
|
+
# handled at the normalizer level, not as physical keys.
|
|
14
|
+
#
|
|
15
|
+
# The key grid is mirrored from the models repo eval harness
|
|
16
|
+
# (kotoshu/models-fasttext-onnx eval/noise.py _AR_101, plan 83
|
|
17
|
+
# batch 2 of the model coverage expansion, verified there
|
|
18
|
+
# against the kbdlayout.info Arabic 101 driver table) so
|
|
19
|
+
# typo-adjacency used for suggestion ranking matches the
|
|
20
|
+
# adjacency used to evaluate the models.
|
|
21
|
+
# spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
|
|
22
|
+
# the sync.
|
|
23
|
+
#
|
|
24
|
+
# Languages: ar, ar-SA
|
|
25
|
+
class Arabic101 < Layout
|
|
26
|
+
# Key positions for the Arabic 101 layout.
|
|
27
|
+
# Each key maps to [row, column] coordinates.
|
|
28
|
+
KEY_POSITIONS = {
|
|
29
|
+
# Number row
|
|
30
|
+
'ذ' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
|
|
31
|
+
'4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
|
|
32
|
+
'8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
|
|
33
|
+
'=' => [0, 12],
|
|
34
|
+
# Top row (ض ص ث ق ف غ ع ه خ ح ج د)
|
|
35
|
+
'ض' => [1, 0], 'ص' => [1, 1], 'ث' => [1, 2], 'ق' => [1, 3],
|
|
36
|
+
'ف' => [1, 4], 'غ' => [1, 5], 'ع' => [1, 6], 'ه' => [1, 7],
|
|
37
|
+
'خ' => [1, 8], 'ح' => [1, 9], 'ج' => [1, 10], 'د' => [1, 11],
|
|
38
|
+
# Home row (ش س ي ب ل ا ت ن م ك ط)
|
|
39
|
+
'ش' => [2, 0], 'س' => [2, 1], 'ي' => [2, 2], 'ب' => [2, 3],
|
|
40
|
+
'ل' => [2, 4], 'ا' => [2, 5], 'ت' => [2, 6], 'ن' => [2, 7],
|
|
41
|
+
'م' => [2, 8], 'ك' => [2, 9], 'ط' => [2, 10],
|
|
42
|
+
# Bottom row (ئ ء ؤ ر لا ى ة و ز ظ)
|
|
43
|
+
'ئ' => [3, 0], 'ء' => [3, 1], 'ؤ' => [3, 2], 'ر' => [3, 3],
|
|
44
|
+
'لا' => [3, 4], 'ى' => [3, 5], 'ة' => [3, 6], 'و' => [3, 7],
|
|
45
|
+
'ز' => [3, 8], 'ظ' => [3, 9]
|
|
46
|
+
}.freeze
|
|
47
|
+
|
|
48
|
+
# Initialize the Arabic 101 layout.
|
|
49
|
+
def initialize
|
|
50
|
+
super(
|
|
51
|
+
name: 'Arabic-101',
|
|
52
|
+
language_codes: %w[ar ar-SA],
|
|
53
|
+
key_positions: KEY_POSITIONS
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# Bulgarian BDS keyboard layout (БДС 5237:1978, KBDBUL.DLL).
|
|
7
|
+
#
|
|
8
|
+
# The national Bulgarian phonetic-adjacent ordering: ы у е и ш щ
|
|
9
|
+
# к с д з ц / ь я а о ж г т н в м ч / ю й ъ э ф х п р л б. The
|
|
10
|
+
# Bulgarian-only letters ъ ь are real keys (unlike JCUKEN, where
|
|
11
|
+
# ъ sits top-right and ы is the home-row opener).
|
|
12
|
+
#
|
|
13
|
+
# The key grid is mirrored from the models repo eval harness
|
|
14
|
+
# (kotoshu/models-fasttext-onnx eval/noise.py _BG_BDS, plan 83
|
|
15
|
+
# batch 2 of the model coverage expansion, verified there
|
|
16
|
+
# against the kbdlayout.info KBDBUL driver table) so
|
|
17
|
+
# typo-adjacency used for suggestion ranking matches the
|
|
18
|
+
# adjacency used to evaluate the models.
|
|
19
|
+
# spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
|
|
20
|
+
# the sync.
|
|
21
|
+
#
|
|
22
|
+
# Languages: bg, bg-BG
|
|
23
|
+
class BulgarianBds < Layout
|
|
24
|
+
# Key positions for the Bulgarian BDS layout.
|
|
25
|
+
# Each key maps to [row, column] coordinates.
|
|
26
|
+
KEY_POSITIONS = {
|
|
27
|
+
# Number row
|
|
28
|
+
')' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
|
|
29
|
+
'4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
|
|
30
|
+
'8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '$' => [0, 11],
|
|
31
|
+
'€' => [0, 12],
|
|
32
|
+
# Top row (ы у е и ш щ к с д з ц)
|
|
33
|
+
'ы' => [1, 0], 'у' => [1, 1], 'е' => [1, 2], 'и' => [1, 3],
|
|
34
|
+
'ш' => [1, 4], 'щ' => [1, 5], 'к' => [1, 6], 'с' => [1, 7],
|
|
35
|
+
'д' => [1, 8], 'з' => [1, 9], 'ц' => [1, 10],
|
|
36
|
+
# Home row (ь я а о ж г т н в м ч)
|
|
37
|
+
'ь' => [2, 0], 'я' => [2, 1], 'а' => [2, 2], 'о' => [2, 3],
|
|
38
|
+
'ж' => [2, 4], 'г' => [2, 5], 'т' => [2, 6], 'н' => [2, 7],
|
|
39
|
+
'в' => [2, 8], 'м' => [2, 9], 'ч' => [2, 10],
|
|
40
|
+
# Bottom row (ю й ъ э ф х п р л б)
|
|
41
|
+
'ю' => [3, 0], 'й' => [3, 1], 'ъ' => [3, 2], 'э' => [3, 3],
|
|
42
|
+
'ф' => [3, 4], 'х' => [3, 5], 'п' => [3, 6], 'р' => [3, 7],
|
|
43
|
+
'л' => [3, 8], 'б' => [3, 9]
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
# Initialize the Bulgarian BDS layout.
|
|
47
|
+
def initialize
|
|
48
|
+
super(
|
|
49
|
+
name: 'Bulgarian-BDS',
|
|
50
|
+
language_codes: %w[bg bg-BG],
|
|
51
|
+
key_positions: KEY_POSITIONS
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# Hebrew standard keyboard layout (SI-1452).
|
|
7
|
+
#
|
|
8
|
+
# The 27 Hebrew letters all sit on key faces in the standard
|
|
9
|
+
# ordering, including the five final-letter forms ם ן ך ף ץ as
|
|
10
|
+
# real keys. Niqqud (vowel points) are dead keys and are not in
|
|
11
|
+
# the grid; geresh/gershayim are shift states, not base keys.
|
|
12
|
+
#
|
|
13
|
+
# The key grid is mirrored from the models repo eval harness
|
|
14
|
+
# (kotoshu/models-fasttext-onnx eval/noise.py _HE_SI1452, plan 83
|
|
15
|
+
# batch 2 of the model coverage expansion, verified there
|
|
16
|
+
# against the Culmus SI-1452 layout table) so typo-adjacency
|
|
17
|
+
# used for suggestion ranking matches the adjacency used to
|
|
18
|
+
# evaluate the models.
|
|
19
|
+
# spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
|
|
20
|
+
# the sync.
|
|
21
|
+
#
|
|
22
|
+
# Languages: he, he-IL
|
|
23
|
+
class HebrewSI1452 < Layout
|
|
24
|
+
# Key positions for the Hebrew SI-1452 layout.
|
|
25
|
+
# Each key maps to [row, column] coordinates.
|
|
26
|
+
KEY_POSITIONS = {
|
|
27
|
+
# Number row
|
|
28
|
+
'`' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
|
|
29
|
+
'4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
|
|
30
|
+
'8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
|
|
31
|
+
'=' => [0, 12],
|
|
32
|
+
# Top row (ק ר א ט ו ן ם פ)
|
|
33
|
+
'/' => [1, 0], "'" => [1, 1], 'ק' => [1, 2], 'ר' => [1, 3],
|
|
34
|
+
'א' => [1, 4], 'ט' => [1, 5], 'ו' => [1, 6], 'ן' => [1, 7],
|
|
35
|
+
'ם' => [1, 8], 'פ' => [1, 9], '[' => [1, 10], ']' => [1, 11],
|
|
36
|
+
# Home row (ש ד ג כ ע י ח ל ך ף)
|
|
37
|
+
'ש' => [2, 0], 'ד' => [2, 1], 'ג' => [2, 2], 'כ' => [2, 3],
|
|
38
|
+
'ע' => [2, 4], 'י' => [2, 5], 'ח' => [2, 6], 'ל' => [2, 7],
|
|
39
|
+
'ך' => [2, 8], 'ף' => [2, 9], ',' => [2, 10],
|
|
40
|
+
# Bottom row (ז ס ב ה נ מ צ ת ץ)
|
|
41
|
+
'ז' => [3, 0], 'ס' => [3, 1], 'ב' => [3, 2], 'ה' => [3, 3],
|
|
42
|
+
'נ' => [3, 4], 'מ' => [3, 5], 'צ' => [3, 6], 'ת' => [3, 7],
|
|
43
|
+
'ץ' => [3, 8], '.' => [3, 9]
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
# Initialize the Hebrew SI-1452 layout.
|
|
47
|
+
def initialize
|
|
48
|
+
super(
|
|
49
|
+
name: 'Hebrew-SI-1452',
|
|
50
|
+
language_codes: %w[he he-IL],
|
|
51
|
+
key_positions: KEY_POSITIONS
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|