kotoshu 0.9.2 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +53 -0
  3. data/Cargo.lock +72 -4
  4. data/TODO.impl/00-vision.md +37 -0
  5. data/TODO.impl/100-full-feature-batch3.md +28 -0
  6. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  7. data/TODO.impl/102-lid-chain.md +28 -0
  8. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  9. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  10. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  11. data/TODO.impl/106-native-lid-detection.md +36 -0
  12. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  13. data/ext/kotoshu_native/Cargo.toml +5 -2
  14. data/ext/kotoshu_native/src/lib.rs +134 -6
  15. data/lib/kotoshu/cache/language_cache.rb +4 -1
  16. data/lib/kotoshu/cache/model_cache.rb +150 -2
  17. data/lib/kotoshu/cli.rb +8 -0
  18. data/lib/kotoshu/configuration.rb +20 -2
  19. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  20. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  21. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  22. data/lib/kotoshu/keyboard/layouts/hrsl.rb +78 -0
  23. data/lib/kotoshu/keyboard/layouts/latin.rb +47 -1
  24. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  25. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  26. data/lib/kotoshu/keyboard/registry.rb +20 -4
  27. data/lib/kotoshu/keyboard.rb +9 -0
  28. data/lib/kotoshu/language/lid_detector.rb +188 -0
  29. data/lib/kotoshu/language.rb +4 -0
  30. data/lib/kotoshu/languages/ar/language.rb +14 -0
  31. data/lib/kotoshu/languages/bg/language.rb +37 -0
  32. data/lib/kotoshu/languages/et/language.rb +24 -0
  33. data/lib/kotoshu/languages/fa/language.rb +16 -0
  34. data/lib/kotoshu/languages/he/language.rb +15 -0
  35. data/lib/kotoshu/languages/hr/language.rb +24 -0
  36. data/lib/kotoshu/languages/id/language.rb +25 -0
  37. data/lib/kotoshu/languages/lt/language.rb +24 -0
  38. data/lib/kotoshu/languages/lv/language.rb +24 -0
  39. data/lib/kotoshu/languages/sk/language.rb +25 -0
  40. data/lib/kotoshu/languages/sl/language.rb +25 -0
  41. data/lib/kotoshu/languages/sr/language.rb +38 -0
  42. data/lib/kotoshu/languages.rb +12 -0
  43. data/lib/kotoshu/spellchecker.rb +35 -0
  44. data/lib/kotoshu/version.rb +1 -1
  45. data/lib/kotoshu.rb +45 -5
  46. data/scripts/extract_eval_grids.rb +4 -1
  47. data/sig/kotoshu.rbs +45 -0
  48. metadata +25 -1
@@ -1,15 +1,143 @@
1
1
  //! Kotoshu native extension (plan 66 / TODO.impl 05, P4b).
2
2
  //!
3
- //! The whole extension: a `#[magnus::init]` that forwards to the engine
4
- //! core's Ruby bindings. `kotoshu::ffi::ruby::init` defines the
5
- //! `Kotoshu::Native` module (`VERSION`, `available?`, `Dictionary.load`
6
- //! with `correct?` / `suggest`); this crate contains no logic of its own,
3
+ //! The extension forwards its `#[magnus::init]` to the engine core's
4
+ //! Ruby bindings. `kotoshu::ffi::ruby::init` defines the `Kotoshu::Native`
5
+ //! module (`VERSION`, `available?`, `Dictionary.load` with `correct?` /
6
+ //! `suggest`); this crate contains no engine logic of its own,
7
7
  //! exactly like kotoshu-rs' `tests/ruby_ext` reference shim and
8
8
  //! parsanol-ruby's `ext/parsanol_native`.
9
+ //!
10
+ //! One binding lives here instead of the core crate: the language
11
+ //! identification surface (plan 106). `LidModel.load` +
12
+ //! `LidModel#detect` wrap `kotoshu::lid::LidModel` — the pure-Rust
13
+ //! reader for the `kotoshu://models/lid/lid-176` artifact pair —
14
+ //! behind the same `model` feature the core crate ships it under
15
+ //! (see the features table in Cargo.toml). Only the magnus glue is
16
+ //! gem-side; every scoring decision stays in the core, where the
17
+ //! parity suite freezes it against the shared frozen corpus.
18
+
19
+ use std::path::Path;
20
+
21
+ use magnus::typed_data::Obj;
22
+ use magnus::typed_data::TypedData;
23
+ use magnus::value::Lazy;
24
+ use magnus::{
25
+ Class, DataType, DataTypeFunctions, Error, ExceptionClass, Module, Object, RClass, RHash,
26
+ RModule, Ruby, data_type_builder, method,
27
+ };
28
+
29
+ /// The `Kotoshu` module, defined (idempotently) on first access — the
30
+ /// same helper shape as the core crate's `ffi::ruby`.
31
+ fn kotoshu_module(ruby: &Ruby) -> RModule {
32
+ static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
33
+ ruby.define_module("Kotoshu")
34
+ .expect("cannot define Kotoshu module")
35
+ });
36
+ ruby.get_inner(&MODULE)
37
+ }
38
+
39
+ /// The `Kotoshu::Native` module hosting the engine bindings.
40
+ fn native_module(ruby: &Ruby) -> RModule {
41
+ static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
42
+ kotoshu_module(ruby)
43
+ .define_module("Native")
44
+ .expect("cannot define Kotoshu::Native module")
45
+ });
46
+ ruby.get_inner(&MODULE)
47
+ }
48
+
49
+ /// `Kotoshu::Native::Error`, defined by the core crate's init — every
50
+ /// LID failure crosses the boundary as the same exception class the
51
+ /// dictionary surface raises.
52
+ fn error_class(ruby: &Ruby) -> ExceptionClass {
53
+ static CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
54
+ native_module(ruby)
55
+ .const_get::<_, ExceptionClass>("Error")
56
+ .expect("Kotoshu::Native::Error not defined after ffi::ruby::init")
57
+ });
58
+ ruby.get_inner(&CLASS)
59
+ }
60
+
61
+ /// A loaded LID model wrapped as a Ruby object: the twin of the core
62
+ /// crate's `RubyDictionary`. Ruby's GC owns it once wrapped; dropping
63
+ /// the Ruby object drops the engine model (~1 MB of int8 rows plus
64
+ /// ~1.8 MB of sidecar tables, roughly the artifact pair's own size).
65
+ #[derive(Debug)]
66
+ pub struct RubyLidModel {
67
+ inner: kotoshu::lid::LidModel,
68
+ }
69
+
70
+ // Plain owned data; Ruby objects move between threads only with the
71
+ // GVL held, which TypedData's Send bound models.
72
+ impl DataTypeFunctions for RubyLidModel {}
73
+
74
+ unsafe impl TypedData for RubyLidModel {
75
+ fn class(ruby: &Ruby) -> RClass {
76
+ static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
77
+ let class = native_module(ruby)
78
+ .define_class("LidModel", ruby.class_object())
79
+ .expect("cannot define Kotoshu::Native::LidModel");
80
+ // Instances exist only through `LidModel.load`; `new`/
81
+ // `allocate` would produce a model-less object.
82
+ class.undef_default_alloc_func();
83
+ class
84
+ });
85
+ ruby.get_inner(&CLASS)
86
+ }
87
+
88
+ fn data_type() -> &'static DataType {
89
+ static DATA_TYPE: DataType =
90
+ data_type_builder!(RubyLidModel, "Kotoshu/Native/LidModel").build();
91
+ &DATA_TYPE
92
+ }
93
+ }
9
94
 
10
- use magnus::{Error, Ruby};
95
+ /// `Kotoshu::Native::LidModel.load(onnx_path, vocab_path)` — reads the
96
+ /// registry artifact pair (the `lid.176.onnx` container and its
97
+ /// `lid.176.vocab.json` sidecar) and wraps the parsed model. Failures
98
+ /// raise `Kotoshu::Native::Error` carrying the Rust error message,
99
+ /// mirroring `Dictionary.load`.
100
+ fn lid_model_load(
101
+ ruby: &Ruby,
102
+ _class: RClass,
103
+ onnx_path: String,
104
+ vocab_path: String,
105
+ ) -> Result<Obj<RubyLidModel>, Error> {
106
+ let fail = |message: String| {
107
+ Error::new(
108
+ error_class(ruby),
109
+ format!("failed to load LID model ({onnx_path}, {vocab_path}): {message}"),
110
+ )
111
+ };
112
+
113
+ let onnx_bytes =
114
+ std::fs::read(Path::new(&onnx_path)).map_err(|error| fail(error.to_string()))?;
115
+ let vocab_bytes =
116
+ std::fs::read(Path::new(&vocab_path)).map_err(|error| fail(error.to_string()))?;
117
+ let model = kotoshu::lid::LidModel::parse(&onnx_bytes, &vocab_bytes)
118
+ .map_err(|error| fail(error.to_string()))?;
119
+ Ok(ruby.obj_wrap(RubyLidModel { inner: model }))
120
+ }
121
+
122
+ /// `Kotoshu::Native::LidModel#detect(text)` — the top label and its
123
+ /// probability as `{ "code" => String, "score" => Float }`, the gem's
124
+ /// `LanguageIdentifier` detection pair (labels equal and scores within
125
+ /// 5e-4 of the fastText bindings; frozen by the kotoshu-rs parity
126
+ /// suite over the shared corpus fixture).
127
+ fn lid_model_detect(ruby: &Ruby, rb_self: &RubyLidModel, text: String) -> Result<RHash, Error> {
128
+ let detection = rb_self.inner.detect(&text);
129
+ let hash = ruby.hash_new();
130
+ hash.aset("code", detection.code.as_str())?;
131
+ hash.aset("score", detection.score)?;
132
+ Ok(hash)
133
+ }
11
134
 
12
135
  #[magnus::init]
13
136
  fn init(ruby: &Ruby) -> Result<(), Error> {
14
- kotoshu::ffi::ruby::init(ruby)
137
+ kotoshu::ffi::ruby::init(ruby)?;
138
+
139
+ let class = RubyLidModel::class(ruby);
140
+ class.define_singleton_method("load", method!(lid_model_load, 2))?;
141
+ class.define_method("detect", method!(lid_model_detect, 1))?;
142
+ Ok(())
15
143
  }
@@ -31,8 +31,11 @@ module Kotoshu
31
31
  # modules (lib/kotoshu/languages/). Staged-only dictionaries in
32
32
  # the dictionaries repo stay outside this list until their
33
33
  # module lands; local files come via setup --aff/--dic instead.
34
+ # Batch 3 (plan 100) promotes the RTL wiring ar/fa/he and adds
35
+ # bg et hr id lt lv sk sl sr.
34
36
  AVAILABLE_LANGUAGES = %w[
35
- ca cs da de el en es fr hu it nb nl pl pt ro ru sv tr uk vi
37
+ ar bg ca cs da de el en es et fa fr he hr hu id it lt lv nb nl
38
+ pl pt ro ru sk sl sr sv tr uk vi
36
39
  ].freeze
37
40
 
38
41
  # Get or download spelling dictionary for a language.
@@ -72,6 +72,22 @@ module Kotoshu
72
72
  # output is stable across machines and runs.
73
73
  TIER_PREFERENCE = %i[mini fluency full].freeze
74
74
 
75
+ # The language-identification artifact pair (plan 106): registry
76
+ # resource `kotoshu://models/lid/lid-176` — the ONNX-converted
77
+ # lid.176 model (int8-per-row, dim 16) plus its vocab.json
78
+ # sidecar. Lives in the same registry as the embedding tiers but
79
+ # is NOT a language tier: it has no language of its own and the
80
+ # tier machinery (resource ids, TIERS) does not apply. It is
81
+ # stored under a top-level `lid/` directory (not
82
+ # `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
83
+ # reporting per-language ids only.
84
+ LID_LANGUAGE = "lid"
85
+ LID_TIER = "lid-176"
86
+ LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
87
+
88
+ # Directory (under the cache root) holding the LID artifact pair.
89
+ LID_DIRECTORY = "lid"
90
+
75
91
  # Normalize and validate a tier name.
76
92
  #
77
93
  # @param tier [String, Symbol] "full", "fluency", or "mini"
@@ -248,6 +264,96 @@ module Kotoshu
248
264
  result
249
265
  end
250
266
 
267
+ # ---- LID artifact pair (plan 106) ----
268
+
269
+ # Whether the LID artifact pair is cached and complete.
270
+ #
271
+ # @return [Boolean]
272
+ def lid_cached?
273
+ !load_cached_lid.nil?
274
+ end
275
+
276
+ # Resolve the LID artifact pair from the cache only — never
277
+ # touches the network (the resolve half of the two-stage model).
278
+ #
279
+ # @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
280
+ # cached, nil when not set up
281
+ # @raise [Kotoshu::IntegrityError] cached bytes fail their
282
+ # recorded checksum
283
+ def load_cached_lid
284
+ dir = File.join(@cache_path, LID_DIRECTORY)
285
+ metadata_path = File.join(dir, "metadata.json")
286
+ return nil unless File.exist?(metadata_path)
287
+
288
+ metadata = read_metadata(metadata_path)
289
+ return nil unless metadata
290
+
291
+ onnx_file = File.join(dir, metadata["file"].to_s)
292
+ vocab_file = File.join(dir, metadata["vocab_file"].to_s)
293
+ return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
294
+ return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
295
+
296
+ verify_lid_integrity!(metadata, onnx_file, vocab_file)
297
+
298
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
299
+ end
300
+
301
+ # Download the LID artifact pair from the models registry
302
+ # (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
303
+ # against the registry sha256 before accepting them (the setup
304
+ # half of the two-stage model; same contract as
305
+ # {#download_tiered_model}).
306
+ #
307
+ # @param force [Boolean] re-fetch the registry first
308
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
309
+ # @raise [Kotoshu::Error] no registry entry, or offline without
310
+ # a cached registry (the registry is fetched once per cache
311
+ # instance; see {#registry})
312
+ # @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
313
+ # bytes are removed before raising)
314
+ def download_lid(force: false)
315
+ entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
316
+ unless entry
317
+ raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
318
+ end
319
+
320
+ dir = File.join(@cache_path, LID_DIRECTORY)
321
+ FileUtils.mkdir_p(dir)
322
+
323
+ onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
324
+ used_url = download_primary_or_mirror!(entry, onnx_file)
325
+ verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
326
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
327
+
328
+ # The vocab sidecar is required, not optional: the reader
329
+ # cannot score without its labels and Huffman counts.
330
+ vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
331
+ begin
332
+ download_file(entry.vocab_url, vocab_file)
333
+ rescue StandardError => e
334
+ File.delete(onnx_file) if File.exist?(onnx_file)
335
+ raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
336
+ end
337
+
338
+ metadata = {
339
+ version: entry.version,
340
+ url: used_url,
341
+ language: LID_LANGUAGE,
342
+ type: "lid",
343
+ tier: LID_TIER,
344
+ file: File.basename(onnx_file),
345
+ vocab_file: File.basename(vocab_file),
346
+ checksum: entry.sha256,
347
+ registry_id: LID_REGISTRY_ID,
348
+ size_bytes: entry.size_bytes,
349
+ cached_at: Time.now.utc.iso8601,
350
+ source: "registry"
351
+ }
352
+ write_metadata(File.join(dir, "metadata.json"), metadata)
353
+
354
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
355
+ end
356
+
251
357
  # Get available model types for a language.
252
358
  #
253
359
  # @param language_code [String] ISO 639-1 language code
@@ -629,7 +735,9 @@ module Kotoshu
629
735
  # Verify downloaded model bytes against the registry's sha256.
630
736
  # Corrupt bytes are removed from disk before raising so the next
631
737
  # attempt re-downloads (same contract as BaseCache#verify_and_audit).
632
- def verify_registry_sha256!(entry, model_file, resource_id, url)
738
+ # +remediation+ overrides the default re-download hint (the LID
739
+ # pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
740
+ def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
633
741
  actual = Digest::SHA256.file(model_file).hexdigest
634
742
  if actual == entry.sha256
635
743
  @audit_log.record(
@@ -649,7 +757,8 @@ module Kotoshu
649
757
  expected: entry.sha256,
650
758
  actual: actual,
651
759
  url: url,
652
- remediation: "Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
760
+ remediation: remediation ||
761
+ "Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
653
762
  )
654
763
  end
655
764
 
@@ -680,6 +789,45 @@ module Kotoshu
680
789
  # Caches written without a checksum field are accepted silently
681
790
  # to preserve backward compatibility with pre-verification caches.
682
791
  #
792
+ # Verify the cached LID pair against the checksum recorded at
793
+ # download time (which, for registry downloads, is the registry's
794
+ # sha256). A Git LFS pointer stub is refused by content first —
795
+ # same contract as {#verify_cached_integrity!}, against the
796
+ # fixed LID paths instead of a resource-id-derived layout.
797
+ #
798
+ # @param metadata [Hash] Parsed metadata.json (string keys)
799
+ # @param onnx_file [String] Path to the cached lid.176.onnx
800
+ # @param vocab_file [String] Path to the cached vocab sidecar
801
+ # @return [void]
802
+ # @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
803
+ # corrupt bytes are removed so the next setup re-downloads
804
+ def verify_lid_integrity!(metadata, onnx_file, vocab_file)
805
+ dir = File.join(@cache_path, LID_DIRECTORY)
806
+ if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
807
+ File.delete(onnx_file)
808
+ File.delete(vocab_file)
809
+ FileUtils.rm_f(File.join(dir, "metadata.json"))
810
+ raise Kotoshu::Error,
811
+ "Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
812
+ "artifact is a Git LFS pointer stub, not model content. Run " \
813
+ "Kotoshu.setup_lid(force: true) to re-download."
814
+ end
815
+
816
+ expected = metadata["checksum"]
817
+ return unless expected
818
+
819
+ actual = Digest::SHA256.file(onnx_file).hexdigest
820
+ return if actual == expected
821
+
822
+ raise Kotoshu::IntegrityError.new(
823
+ LID_REGISTRY_ID,
824
+ expected: expected,
825
+ actual: actual,
826
+ url: metadata["url"],
827
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
828
+ )
829
+ end
830
+
683
831
  # @param resource_id [String] The resource identifier (e.g. "en:onnx")
684
832
  # @param metadata [Hash] Parsed metadata.json (string keys)
685
833
  # @param model_file [String] Path to the cached model file
data/lib/kotoshu/cli.rb CHANGED
@@ -162,6 +162,10 @@ module Kotoshu
162
162
  method_option :exclude,
163
163
  type: :array,
164
164
  desc: "Skip files matching these globs (always wins over --include and the extension default)"
165
+ method_option :personal,
166
+ type: :boolean,
167
+ default: true,
168
+ desc: "Accept words from the personal dictionary (personal.dic); --no-personal flags them again"
165
169
  # Check spelling in files, directories, or stdin. Cache-only —
166
170
  # never downloads; run `kotoshu setup LANG` first. See the
167
171
  # command long description (kotoshu help check) for flags,
@@ -395,6 +399,10 @@ module Kotoshu
395
399
  Kotoshu::Configuration.reset
396
400
  cfg = Kotoshu::Configuration.instance
397
401
  cfg.default_language = options[:language] if options[:language] && options[:language] != "auto"
402
+ # `kotoshu check --no-personal` opts out of the personal
403
+ # dictionary for this run (plan 105). Other commands do not
404
+ # declare the flag, so options[:personal] is nil there.
405
+ cfg.personal_dictionary = options[:personal] unless options[:personal].nil?
398
406
  end
399
407
 
400
408
  # Install a ProgressReporter on Configuration.download_reporter
@@ -209,6 +209,12 @@ module Kotoshu
209
209
  default: "ruby",
210
210
  description: "Engine backend for correct?/suggest (ruby, native, or auto)",
211
211
  type: String
212
+ },
213
+ personal_dictionary: {
214
+ env: "KOTOSHU_PERSONAL_DICTIONARY",
215
+ default: true,
216
+ description: "Words in the personal dictionary (personal.dic) pass the check path",
217
+ type: :boolean
212
218
  }
213
219
  }.freeze
214
220
 
@@ -229,7 +235,8 @@ module Kotoshu
229
235
  default_language: "en",
230
236
  resource_pin: "main",
231
237
  semantic_cascade_threshold: 1.0,
232
- backend: "ruby"
238
+ backend: "ruby",
239
+ personal_dictionary: true
233
240
  }.freeze
234
241
 
235
242
  # @return [String, nil] Path to the dictionary file
@@ -289,6 +296,15 @@ module Kotoshu
289
296
  # NativeBackend for the boundary — the semantic path stays Ruby.
290
297
  attr_accessor :backend
291
298
 
299
+ # @return [Boolean] Whether Spellchecker#check accepts words from
300
+ # the user personal dictionary (~/.config/kotoshu/personal.dic,
301
+ # KOTOSHU_PERSONAL_DIC override) instead of flagging them as
302
+ # errors — the README story and the LSP diagnostics behavior
303
+ # (plan 105). Opt out with KOTOSHU_PERSONAL_DICTIONARY=false or
304
+ # `kotoshu check --no-personal`; the suggest path is unaffected,
305
+ # so personal words stay spellcheckable.
306
+ attr_accessor :personal_dictionary
307
+
292
308
  # @return [String, nil] Path to cache directory
293
309
  attr_accessor :cache_path
294
310
 
@@ -489,7 +505,8 @@ module Kotoshu
489
505
  cache_ttl: @cache_ttl,
490
506
  max_cache_size: @max_cache_size,
491
507
  semantic_cascade_threshold: @semantic_cascade_threshold,
492
- backend: @backend
508
+ backend: @backend,
509
+ personal_dictionary: @personal_dictionary
493
510
  }
494
511
  end
495
512
 
@@ -607,6 +624,7 @@ module Kotoshu
607
624
  @resource_pin = DEFAULTS[:resource_pin]
608
625
  @semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
609
626
  @backend = DEFAULTS[:backend]
627
+ @personal_dictionary = DEFAULTS[:personal_dictionary]
610
628
  end
611
629
 
612
630
  # Apply resolved values from the resolver (ENV, defaults).
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Arabic 101 keyboard layout (standard IBM PC Arabic layout).
7
+ #
8
+ # The Arabic letters sit on the QWERTY key faces: the number row
9
+ # is shared with ASCII, and ض ص ث ق ف غ ع ه خ ح ج د / ش س ي ب ل ا
10
+ # ت ن م ك ط / ئ ء ؤ ر لا ى ة و ز ظ fill the three letter rows.
11
+ # Diacritics (tashkeel) are dead keys and are not in the grid;
12
+ # the hamza carriers أ إ آ and the ه/ة ي/ى spelling slips are
13
+ # handled at the normalizer level, not as physical keys.
14
+ #
15
+ # The key grid is mirrored from the models repo eval harness
16
+ # (kotoshu/models-fasttext-onnx eval/noise.py _AR_101, plan 83
17
+ # batch 2 of the model coverage expansion, verified there
18
+ # against the kbdlayout.info Arabic 101 driver table) so
19
+ # typo-adjacency used for suggestion ranking matches the
20
+ # adjacency used to evaluate the models.
21
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
22
+ # the sync.
23
+ #
24
+ # Languages: ar, ar-SA
25
+ class Arabic101 < Layout
26
+ # Key positions for the Arabic 101 layout.
27
+ # Each key maps to [row, column] coordinates.
28
+ KEY_POSITIONS = {
29
+ # Number row
30
+ 'ذ' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
31
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
32
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
33
+ '=' => [0, 12],
34
+ # Top row (ض ص ث ق ف غ ع ه خ ح ج د)
35
+ 'ض' => [1, 0], 'ص' => [1, 1], 'ث' => [1, 2], 'ق' => [1, 3],
36
+ 'ف' => [1, 4], 'غ' => [1, 5], 'ع' => [1, 6], 'ه' => [1, 7],
37
+ 'خ' => [1, 8], 'ح' => [1, 9], 'ج' => [1, 10], 'د' => [1, 11],
38
+ # Home row (ش س ي ب ل ا ت ن م ك ط)
39
+ 'ش' => [2, 0], 'س' => [2, 1], 'ي' => [2, 2], 'ب' => [2, 3],
40
+ 'ل' => [2, 4], 'ا' => [2, 5], 'ت' => [2, 6], 'ن' => [2, 7],
41
+ 'م' => [2, 8], 'ك' => [2, 9], 'ط' => [2, 10],
42
+ # Bottom row (ئ ء ؤ ر لا ى ة و ز ظ)
43
+ 'ئ' => [3, 0], 'ء' => [3, 1], 'ؤ' => [3, 2], 'ر' => [3, 3],
44
+ 'لا' => [3, 4], 'ى' => [3, 5], 'ة' => [3, 6], 'و' => [3, 7],
45
+ 'ز' => [3, 8], 'ظ' => [3, 9]
46
+ }.freeze
47
+
48
+ # Initialize the Arabic 101 layout.
49
+ def initialize
50
+ super(
51
+ name: 'Arabic-101',
52
+ language_codes: %w[ar ar-SA],
53
+ key_positions: KEY_POSITIONS
54
+ )
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Bulgarian BDS keyboard layout (БДС 5237:1978, KBDBUL.DLL).
7
+ #
8
+ # The national Bulgarian phonetic-adjacent ordering: ы у е и ш щ
9
+ # к с д з ц / ь я а о ж г т н в м ч / ю й ъ э ф х п р л б. The
10
+ # Bulgarian-only letters ъ ь are real keys (unlike JCUKEN, where
11
+ # ъ sits top-right and ы is the home-row opener).
12
+ #
13
+ # The key grid is mirrored from the models repo eval harness
14
+ # (kotoshu/models-fasttext-onnx eval/noise.py _BG_BDS, plan 83
15
+ # batch 2 of the model coverage expansion, verified there
16
+ # against the kbdlayout.info KBDBUL driver table) so
17
+ # typo-adjacency used for suggestion ranking matches the
18
+ # adjacency used to evaluate the models.
19
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
20
+ # the sync.
21
+ #
22
+ # Languages: bg, bg-BG
23
+ class BulgarianBds < Layout
24
+ # Key positions for the Bulgarian BDS layout.
25
+ # Each key maps to [row, column] coordinates.
26
+ KEY_POSITIONS = {
27
+ # Number row
28
+ ')' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
29
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
30
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '$' => [0, 11],
31
+ '€' => [0, 12],
32
+ # Top row (ы у е и ш щ к с д з ц)
33
+ 'ы' => [1, 0], 'у' => [1, 1], 'е' => [1, 2], 'и' => [1, 3],
34
+ 'ш' => [1, 4], 'щ' => [1, 5], 'к' => [1, 6], 'с' => [1, 7],
35
+ 'д' => [1, 8], 'з' => [1, 9], 'ц' => [1, 10],
36
+ # Home row (ь я а о ж г т н в м ч)
37
+ 'ь' => [2, 0], 'я' => [2, 1], 'а' => [2, 2], 'о' => [2, 3],
38
+ 'ж' => [2, 4], 'г' => [2, 5], 'т' => [2, 6], 'н' => [2, 7],
39
+ 'в' => [2, 8], 'м' => [2, 9], 'ч' => [2, 10],
40
+ # Bottom row (ю й ъ э ф х п р л б)
41
+ 'ю' => [3, 0], 'й' => [3, 1], 'ъ' => [3, 2], 'э' => [3, 3],
42
+ 'ф' => [3, 4], 'х' => [3, 5], 'п' => [3, 6], 'р' => [3, 7],
43
+ 'л' => [3, 8], 'б' => [3, 9]
44
+ }.freeze
45
+
46
+ # Initialize the Bulgarian BDS layout.
47
+ def initialize
48
+ super(
49
+ name: 'Bulgarian-BDS',
50
+ language_codes: %w[bg bg-BG],
51
+ key_positions: KEY_POSITIONS
52
+ )
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Hebrew standard keyboard layout (SI-1452).
7
+ #
8
+ # The 27 Hebrew letters all sit on key faces in the standard
9
+ # ordering, including the five final-letter forms ם ן ך ף ץ as
10
+ # real keys. Niqqud (vowel points) are dead keys and are not in
11
+ # the grid; geresh/gershayim are shift states, not base keys.
12
+ #
13
+ # The key grid is mirrored from the models repo eval harness
14
+ # (kotoshu/models-fasttext-onnx eval/noise.py _HE_SI1452, plan 83
15
+ # batch 2 of the model coverage expansion, verified there
16
+ # against the Culmus SI-1452 layout table) so typo-adjacency
17
+ # used for suggestion ranking matches the adjacency used to
18
+ # evaluate the models.
19
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
20
+ # the sync.
21
+ #
22
+ # Languages: he, he-IL
23
+ class HebrewSI1452 < Layout
24
+ # Key positions for the Hebrew SI-1452 layout.
25
+ # Each key maps to [row, column] coordinates.
26
+ KEY_POSITIONS = {
27
+ # Number row
28
+ '`' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
29
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
30
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
31
+ '=' => [0, 12],
32
+ # Top row (ק ר א ט ו ן ם פ)
33
+ '/' => [1, 0], "'" => [1, 1], 'ק' => [1, 2], 'ר' => [1, 3],
34
+ 'א' => [1, 4], 'ט' => [1, 5], 'ו' => [1, 6], 'ן' => [1, 7],
35
+ 'ם' => [1, 8], 'פ' => [1, 9], '[' => [1, 10], ']' => [1, 11],
36
+ # Home row (ש ד ג כ ע י ח ל ך ף)
37
+ 'ש' => [2, 0], 'ד' => [2, 1], 'ג' => [2, 2], 'כ' => [2, 3],
38
+ 'ע' => [2, 4], 'י' => [2, 5], 'ח' => [2, 6], 'ל' => [2, 7],
39
+ 'ך' => [2, 8], 'ף' => [2, 9], ',' => [2, 10],
40
+ # Bottom row (ז ס ב ה נ מ צ ת ץ)
41
+ 'ז' => [3, 0], 'ס' => [3, 1], 'ב' => [3, 2], 'ה' => [3, 3],
42
+ 'נ' => [3, 4], 'מ' => [3, 5], 'צ' => [3, 6], 'ת' => [3, 7],
43
+ 'ץ' => [3, 8], '.' => [3, 9]
44
+ }.freeze
45
+
46
+ # Initialize the Hebrew SI-1452 layout.
47
+ def initialize
48
+ super(
49
+ name: 'Hebrew-SI-1452',
50
+ language_codes: %w[he he-IL],
51
+ key_positions: KEY_POSITIONS
52
+ )
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end