kotoshu 0.9.3 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4891dc67bd85b5c342fdbc7453085990a1e9ff9bea5b394a67f87e11b4bbd6c7
4
- data.tar.gz: 378d1b77bea8113d155774bc07e481afaa9229d6e1edcd7d5fda2805d28a963c
3
+ metadata.gz: a0093eb0f194f55beb4950a039c37e4b3abe6b33d073e7d7a8de0958e22a32cc
4
+ data.tar.gz: 795382646b0e0fdee0010b76d8fcd9afefa55447f513494ec3e832fe225f6110
5
5
  SHA512:
6
- metadata.gz: bebc7b8be31a4f9f0127bddf2cb4a8de23c9908b05ddf91f51a43af5927af4608a0398090cce27deefce0e5c86f4a7d9edb0ffdf6b122f849bb065cfcee44e2d
7
- data.tar.gz: 1240bb5e7d513c4d3b3dd936042c3d177cbad6aebc402fc6ef91f86e3163bc0ba2dcf516f93d7658171d2c952c1f2fed6162928c7a50bdef3e5703233acce1ce
6
+ metadata.gz: 59fb8d99d938013b349ca779593a72b2fa8390cf725950c07acd9e117fde7ff138710fb9dfa1ba3d7581947d261dea0f990f555e1456421156316c746e8d66a8
7
+ data.tar.gz: 18c2713e916171510e45028f08968d443248f5426c614e56ab21a32078543bdad69dd7f6aed4d44ced5ee00f7d18554f9050f19e84a3fb936798d3846bb00838
data/CHANGELOG.md CHANGED
@@ -8,6 +8,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
8
8
  ## [Unreleased]
9
9
 
10
10
  ### Added
11
+ - **Personal dictionary in the check path** (plan 105) -
12
+ `Spellchecker#check` (and therefore `Kotoshu.check`,
13
+ `Kotoshu.check_file`, and `kotoshu check`) no longer flags words
14
+ present in the user personal dictionary
15
+ (~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override),
16
+ matching the semantics the LSP already applies to diagnostics:
17
+ case-insensitive, filtered at result assembly without suppression
18
+ metadata, words still counted, and `suggest`/`correct?` untouched so
19
+ personal words stay spellcheckable. The dictionary is loaded once
20
+ per process (no hot reload; the LSP keeps its mtime-based reload).
21
+ Opt out with `Kotoshu.configure { |c| c.personal_dictionary = false }`
22
+ / KOTOSHU_PERSONAL_DICTIONARY=false / `kotoshu check --no-personal`.
23
+ - **Native language detection** (plan 106) - `Kotoshu.detect_language(text)`
24
+ now returns a `Kotoshu::Language::Detection` (`code` + `score`) scored by
25
+ the Rust extension over the `kotoshu://models/lid/lid-176` registry
26
+ artifact pair (the ONNX-converted lid.176 model, 176 languages,
27
+ bit-faithful to the fastText bindings - parity 55/55 against the frozen
28
+ corpus shared with kotoshu-rs). Fetch the pair once with
29
+ `Kotoshu.setup_lid` (registry-resolved, sha-verified, cached under
30
+ `models-cache/lid/`); detect resolves cache-only and never downloads.
31
+ The 7-language `Language::Detector` heuristic stays the fallback when
32
+ the extension is absent, the model is not set up, or
33
+ `KOTOSHU_BACKEND=ruby` opts out - so pure-Ruby installs are unaffected
34
+ (no new gemspec dependency, no onnxruntime). The magnus shim
35
+ (ext/kotoshu_native) gains `Kotoshu::Native::LidModel.load` +
36
+ `#detect` behind the kotoshu-rs `model` feature; the workspace
37
+ Cargo.lock moves to the kotoshu-rs revision carrying the reader.
38
+ `Kotoshu.detect_language_with_confidence` keeps its `[code, score]`
39
+ shape and now routes through the same engine.
11
40
  - **Full-feature language batch 3** (plan 100) - twelve languages gain the
12
41
  downloadable spelling dictionary, a keyboard layout, and a gem module:
13
42
  `ar` `id` `fa` `he` `bg` `sr` `hr` `sk` `sl` `lt` `lv` `et` (the RTL
data/Cargo.lock CHANGED
@@ -26,7 +26,7 @@ dependencies = [
26
26
  "regex",
27
27
  "rustc-hash",
28
28
  "shlex",
29
- "syn",
29
+ "syn 2.0.119",
30
30
  ]
31
31
 
32
32
  [[package]]
@@ -82,12 +82,20 @@ dependencies = [
82
82
  "either",
83
83
  ]
84
84
 
85
+ [[package]]
86
+ name = "itoa"
87
+ version = "1.0.18"
88
+ source = "registry+https://github.com/rust-lang/crates.io-index"
89
+ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
90
+
85
91
  [[package]]
86
92
  name = "kotoshu"
87
93
  version = "0.1.0"
88
- source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#9941f1ea1a6157c57659e428f475959508d4d0b3"
94
+ source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#e574c445df5f6eee8e978f1f4d26dc48cfb1947b"
89
95
  dependencies = [
90
96
  "magnus",
97
+ "serde",
98
+ "serde_json",
91
99
  ]
92
100
 
93
101
  [[package]]
@@ -141,7 +149,7 @@ checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
141
149
  dependencies = [
142
150
  "proc-macro2",
143
151
  "quote",
144
- "syn",
152
+ "syn 2.0.119",
145
153
  ]
146
154
 
147
155
  [[package]]
@@ -205,7 +213,7 @@ dependencies = [
205
213
  "quote",
206
214
  "regex",
207
215
  "shell-words",
208
- "syn",
216
+ "syn 2.0.119",
209
217
  ]
210
218
 
211
219
  [[package]]
@@ -255,6 +263,49 @@ version = "0.3.6"
255
263
  source = "registry+https://github.com/rust-lang/crates.io-index"
256
264
  checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
257
265
 
266
+ [[package]]
267
+ name = "serde"
268
+ version = "1.0.229"
269
+ source = "registry+https://github.com/rust-lang/crates.io-index"
270
+ checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
271
+ dependencies = [
272
+ "serde_core",
273
+ "serde_derive",
274
+ ]
275
+
276
+ [[package]]
277
+ name = "serde_core"
278
+ version = "1.0.229"
279
+ source = "registry+https://github.com/rust-lang/crates.io-index"
280
+ checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
281
+ dependencies = [
282
+ "serde_derive",
283
+ ]
284
+
285
+ [[package]]
286
+ name = "serde_derive"
287
+ version = "1.0.229"
288
+ source = "registry+https://github.com/rust-lang/crates.io-index"
289
+ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
290
+ dependencies = [
291
+ "proc-macro2",
292
+ "quote",
293
+ "syn 3.0.5",
294
+ ]
295
+
296
+ [[package]]
297
+ name = "serde_json"
298
+ version = "1.0.151"
299
+ source = "registry+https://github.com/rust-lang/crates.io-index"
300
+ checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
301
+ dependencies = [
302
+ "itoa",
303
+ "memchr",
304
+ "serde",
305
+ "serde_core",
306
+ "zmij",
307
+ ]
308
+
258
309
  [[package]]
259
310
  name = "shell-words"
260
311
  version = "1.1.1"
@@ -278,6 +329,17 @@ dependencies = [
278
329
  "unicode-ident",
279
330
  ]
280
331
 
332
+ [[package]]
333
+ name = "syn"
334
+ version = "3.0.5"
335
+ source = "registry+https://github.com/rust-lang/crates.io-index"
336
+ checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
337
+ dependencies = [
338
+ "proc-macro2",
339
+ "quote",
340
+ "unicode-ident",
341
+ ]
342
+
281
343
  [[package]]
282
344
  name = "unicode-ident"
283
345
  version = "1.0.24"
@@ -289,3 +351,9 @@ name = "windows-link"
289
351
  version = "0.2.1"
290
352
  source = "registry+https://github.com/rust-lang/crates.io-index"
291
353
  checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
354
+
355
+ [[package]]
356
+ name = "zmij"
357
+ version = "1.0.23"
358
+ source = "registry+https://github.com/rust-lang/crates.io-index"
359
+ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
@@ -247,3 +247,18 @@ plan name the source.
247
247
  | `docs/ARCHITECTURE_IMPROVEMENTS_PLAN.md` (5 phases) | Phase 1 perf items in `01`/`05`; Phase 2 architecture in `00-vision` (this doc); Phase 3 UX mostly implemented (`PersonalDictionary`, `ProjectConfig`, `FluentChecker` exist); Phase 4 quality in `10`; Phase 5 polish spread |
248
248
  | `docs/TDD_ITERATION_STRATEGY.md` | `10` (test methodology + CI); project-rule overrides (no doubles, behavior-not-implementation) in `CLAUDE.md` |
249
249
  | `docs/ONNX_FUNCTIONAL_VERIFICATION.md` | `05` (semantic path productionization) |
250
+
251
+
252
+ Plans 105-107 plus models 09 and site 11 are **wave 7, detection and
253
+ coverage depth** (2026-09-07): after the wave-6 releases (gem 0.9.3,
254
+ wasm 0.4.0, registry v1.4.0, action v2), the remaining gaps are the
255
+ personal dictionary never reaching the check path, detection stuck on
256
+ a 7-language heuristic, buckets only for en/de, and the playground not
257
+ yet passing bucket bytes to loadModel.
258
+
259
+ | # | Plan | Gap it closes |
260
+ |---|---|---|
261
+ | 105 | [Personal dictionary in check](105-personal-dictionary-in-check.md) | kotoshu personal add then check still flags the word |
262
+ | 106 | [Native LID detection](106-native-lid-detection.md) | detect is a 7-language heuristic; the pure-Rust lid reader exists unused |
263
+ | 107 | models [09 bucket expansion](../../models-fasttext-onnx/TODO.impl/09-bucket-table-expansion.md) | bucket OOV siblings exist only for en and de |
264
+ | — | site 11 playground buckets | wasm 0.4.0 accepts bucketsBytes; the worker never fetches the sibling |
@@ -0,0 +1,34 @@
1
+ # Plan 105 — The personal dictionary reaches the check path
2
+
3
+ ## Why
4
+ `kotoshu personal add recieve` then `kotoshu check` still flags the word —
5
+ PersonalDictionary is wired only into the `personal` subcommand and the LSP
6
+ (resolved server-side), not into `Spellchecker#check`. Users who add words
7
+ see them flagged again on the very next run; the README story ("add words
8
+ you accept") is currently false on the CLI/API path.
9
+
10
+ ## Work (gem repo)
11
+ 1. `Spellchecker#check` consults the personal dictionary: a word present in
12
+ personal.dic is not an error (mirror the LSP's semantics: filter at
13
+ result assembly, keep suppression metadata out of the error rows).
14
+ 2. Reload semantics: the LSP reloads on mtime change; the CLI/one-shot path
15
+ loads once per process. Do not add hot reload here.
16
+ 3. Opt-out: `Kotoshu.configure { personal_dictionary false }` (or the
17
+ existing config seam — find it) and `--no-personal` on `kotoshu check`.
18
+ 4. Specs: add-then-check round trip on a real dictionary; opt-out flags
19
+ again; personal words still spellcheckable via suggest.
20
+ 5. Docs truth: /docs/ignores gains the personal dictionary section (the
21
+ action PR #2 README noted this gap) — deliver the copy in the PR body
22
+ for the site follow-up, do not touch the site repo.
23
+
24
+ ## Verification
25
+ Suite green (3,900+ examples); rubocop clean; `kotoshu personal add x &&
26
+ kotoshu check` live round trip recorded in the PR body.
27
+
28
+ ## Status
29
+ Executed 2026-09-07 — gem PR (branch plan-105-personal-dict-in-check):
30
+ check-path filter mirroring the LSP, one load per process,
31
+ Configuration#personal_dictionary + KOTOSHU_PERSONAL_DICTIONARY +
32
+ `kotoshu check --no-personal`; suite pins KOTOSHU_PERSONAL_DIC for
33
+ hermeticity. /docs/ignores copy delivered in the PR body for the site
34
+ follow-up.
@@ -0,0 +1,36 @@
1
+ # Plan 106 — Document language detection through the pure-Rust LID reader
2
+
3
+ ## Why
4
+ The gem's LanguageIdentifier needs the Python fastText bindings (and breaks
5
+ under numpy 2), so /v1/detect ships a 7-language heuristic. kotoshu-rs now
6
+ carries a pure-Rust lid.176 reader (kotoshu/src/lid, parity 55/55 vs the
7
+ gem) behind the same `model` feature the native ext already builds.
8
+
9
+ ## Work (gem repo)
10
+ 1. Expose `Kotoshu.detect_language(text)` through the native backend:
11
+ download/resolve lid.176.onnx + vocab via the models registry
12
+ (kotoshu://models/lid/lid-176, v1.4.0), score with the ext, return
13
+ { code, score }. Heuristic Language::Detector stays the fallback when
14
+ the native ext is absent or the model is not set up.
15
+ 2. FFI seam: extend the magnus shim (ext/kotoshu_native) with the call —
16
+ mirror how suggest/check already cross; keep it behind the existing
17
+ feature so pure-Ruby installs are unaffected.
18
+ 3. Server follow-up (kotoshu-server repo, separate PR riding this):
19
+ /v1/detect prefers Kotoshu.detect_language when available. Note it in
20
+ the PR body; do not touch the server repo yourself.
21
+ 4. Specs: native-vs-fixture parity (reuse the rs corpus JSON shape),
22
+ fallback behavior, registry miss error.
23
+
24
+ ## Verification
25
+ Suite green; a live detect over en/de/ru/ja/ar samples recorded in the PR
26
+ body; pure-Ruby suite still green (no new hard dependency).
27
+
28
+ ## Status
29
+ Executed 2026-09-07 — gem PR (branch plan-106-native-lid-detection):
30
+ Kotoshu.detect_language -> Language::Detection through the ext
31
+ (LidModel.load/#detect in the magnus shim, kotoshu-rs model feature,
32
+ Cargo.lock bumped to the reader revision); ModelCache gained the
33
+ lid pair (download_lid/load_cached_lid, top-level lid/ dir);
34
+ Kotoshu.setup_lid two-stage; heuristic fallback incl.
35
+ KOTOSHU_BACKEND=ruby; parity 55/55 vs the shared frozen corpus.
36
+ Server follow-up (/v1/detect) noted in the PR body.
@@ -33,5 +33,8 @@ rb-sys = { version = "0.9.124", features = ["global-allocator"] }
33
33
  # and are a documented relax trigger for the P4 gate, not adopted here).
34
34
  magnus = "0.8"
35
35
 
36
- # The engine. Feature "ruby" turns on ffi::ruby (magnus bindings).
37
- kotoshu = { git = "https://github.com/kotoshu/kotoshu-rs", branch = "main", features = ["ruby"] }
36
+ # The engine. Feature "ruby" turns on ffi::ruby (magnus bindings);
37
+ # "model" turns on the pure-Rust LID reader (kotoshu::lid, plan 106) —
38
+ # the same feature the core crate ships it under, so pure-Ruby installs
39
+ # that never build this extension are unaffected.
40
+ kotoshu = { git = "https://github.com/kotoshu/kotoshu-rs", branch = "main", features = ["ruby", "model"] }
@@ -1,15 +1,143 @@
1
1
  //! Kotoshu native extension (plan 66 / TODO.impl 05, P4b).
2
2
  //!
3
- //! The whole extension: a `#[magnus::init]` that forwards to the engine
4
- //! core's Ruby bindings. `kotoshu::ffi::ruby::init` defines the
5
- //! `Kotoshu::Native` module (`VERSION`, `available?`, `Dictionary.load`
6
- //! with `correct?` / `suggest`); this crate contains no logic of its own,
3
+ //! The extension forwards its `#[magnus::init]` to the engine core's
4
+ //! Ruby bindings. `kotoshu::ffi::ruby::init` defines the `Kotoshu::Native`
5
+ //! module (`VERSION`, `available?`, `Dictionary.load` with `correct?` /
6
+ //! `suggest`); this crate contains no engine logic of its own,
7
7
  //! exactly like kotoshu-rs' `tests/ruby_ext` reference shim and
8
8
  //! parsanol-ruby's `ext/parsanol_native`.
9
+ //!
10
+ //! One binding lives here instead of the core crate: the language
11
+ //! identification surface (plan 106). `LidModel.load` +
12
+ //! `LidModel#detect` wrap `kotoshu::lid::LidModel` — the pure-Rust
13
+ //! reader for the `kotoshu://models/lid/lid-176` artifact pair —
14
+ //! behind the same `model` feature the core crate ships it under
15
+ //! (see the features table in Cargo.toml). Only the magnus glue is
16
+ //! gem-side; every scoring decision stays in the core, where the
17
+ //! parity suite freezes it against the shared frozen corpus.
18
+
19
+ use std::path::Path;
20
+
21
+ use magnus::typed_data::Obj;
22
+ use magnus::typed_data::TypedData;
23
+ use magnus::value::Lazy;
24
+ use magnus::{
25
+ Class, DataType, DataTypeFunctions, Error, ExceptionClass, Module, Object, RClass, RHash,
26
+ RModule, Ruby, data_type_builder, method,
27
+ };
28
+
29
+ /// The `Kotoshu` module, defined (idempotently) on first access — the
30
+ /// same helper shape as the core crate's `ffi::ruby`.
31
+ fn kotoshu_module(ruby: &Ruby) -> RModule {
32
+ static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
33
+ ruby.define_module("Kotoshu")
34
+ .expect("cannot define Kotoshu module")
35
+ });
36
+ ruby.get_inner(&MODULE)
37
+ }
38
+
39
+ /// The `Kotoshu::Native` module hosting the engine bindings.
40
+ fn native_module(ruby: &Ruby) -> RModule {
41
+ static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
42
+ kotoshu_module(ruby)
43
+ .define_module("Native")
44
+ .expect("cannot define Kotoshu::Native module")
45
+ });
46
+ ruby.get_inner(&MODULE)
47
+ }
48
+
49
+ /// `Kotoshu::Native::Error`, defined by the core crate's init — every
50
+ /// LID failure crosses the boundary as the same exception class the
51
+ /// dictionary surface raises.
52
+ fn error_class(ruby: &Ruby) -> ExceptionClass {
53
+ static CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
54
+ native_module(ruby)
55
+ .const_get::<_, ExceptionClass>("Error")
56
+ .expect("Kotoshu::Native::Error not defined after ffi::ruby::init")
57
+ });
58
+ ruby.get_inner(&CLASS)
59
+ }
60
+
61
+ /// A loaded LID model wrapped as a Ruby object: the twin of the core
62
+ /// crate's `RubyDictionary`. Ruby's GC owns it once wrapped; dropping
63
+ /// the Ruby object drops the engine model (~1 MB of int8 rows plus
64
+ /// ~1.8 MB of sidecar tables, roughly the artifact pair's own size).
65
+ #[derive(Debug)]
66
+ pub struct RubyLidModel {
67
+ inner: kotoshu::lid::LidModel,
68
+ }
69
+
70
+ // Plain owned data; Ruby objects move between threads only with the
71
+ // GVL held, which TypedData's Send bound models.
72
+ impl DataTypeFunctions for RubyLidModel {}
73
+
74
+ unsafe impl TypedData for RubyLidModel {
75
+ fn class(ruby: &Ruby) -> RClass {
76
+ static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
77
+ let class = native_module(ruby)
78
+ .define_class("LidModel", ruby.class_object())
79
+ .expect("cannot define Kotoshu::Native::LidModel");
80
+ // Instances exist only through `LidModel.load`; `new`/
81
+ // `allocate` would produce a model-less object.
82
+ class.undef_default_alloc_func();
83
+ class
84
+ });
85
+ ruby.get_inner(&CLASS)
86
+ }
87
+
88
+ fn data_type() -> &'static DataType {
89
+ static DATA_TYPE: DataType =
90
+ data_type_builder!(RubyLidModel, "Kotoshu/Native/LidModel").build();
91
+ &DATA_TYPE
92
+ }
93
+ }
9
94
 
10
- use magnus::{Error, Ruby};
95
+ /// `Kotoshu::Native::LidModel.load(onnx_path, vocab_path)` — reads the
96
+ /// registry artifact pair (the `lid.176.onnx` container and its
97
+ /// `lid.176.vocab.json` sidecar) and wraps the parsed model. Failures
98
+ /// raise `Kotoshu::Native::Error` carrying the Rust error message,
99
+ /// mirroring `Dictionary.load`.
100
+ fn lid_model_load(
101
+ ruby: &Ruby,
102
+ _class: RClass,
103
+ onnx_path: String,
104
+ vocab_path: String,
105
+ ) -> Result<Obj<RubyLidModel>, Error> {
106
+ let fail = |message: String| {
107
+ Error::new(
108
+ error_class(ruby),
109
+ format!("failed to load LID model ({onnx_path}, {vocab_path}): {message}"),
110
+ )
111
+ };
112
+
113
+ let onnx_bytes =
114
+ std::fs::read(Path::new(&onnx_path)).map_err(|error| fail(error.to_string()))?;
115
+ let vocab_bytes =
116
+ std::fs::read(Path::new(&vocab_path)).map_err(|error| fail(error.to_string()))?;
117
+ let model = kotoshu::lid::LidModel::parse(&onnx_bytes, &vocab_bytes)
118
+ .map_err(|error| fail(error.to_string()))?;
119
+ Ok(ruby.obj_wrap(RubyLidModel { inner: model }))
120
+ }
121
+
122
+ /// `Kotoshu::Native::LidModel#detect(text)` — the top label and its
123
+ /// probability as `{ "code" => String, "score" => Float }`, the gem's
124
+ /// `LanguageIdentifier` detection pair (labels equal and scores within
125
+ /// 5e-4 of the fastText bindings; frozen by the kotoshu-rs parity
126
+ /// suite over the shared corpus fixture).
127
+ fn lid_model_detect(ruby: &Ruby, rb_self: &RubyLidModel, text: String) -> Result<RHash, Error> {
128
+ let detection = rb_self.inner.detect(&text);
129
+ let hash = ruby.hash_new();
130
+ hash.aset("code", detection.code.as_str())?;
131
+ hash.aset("score", detection.score)?;
132
+ Ok(hash)
133
+ }
11
134
 
12
135
  #[magnus::init]
13
136
  fn init(ruby: &Ruby) -> Result<(), Error> {
14
- kotoshu::ffi::ruby::init(ruby)
137
+ kotoshu::ffi::ruby::init(ruby)?;
138
+
139
+ let class = RubyLidModel::class(ruby);
140
+ class.define_singleton_method("load", method!(lid_model_load, 2))?;
141
+ class.define_method("detect", method!(lid_model_detect, 1))?;
142
+ Ok(())
15
143
  }
@@ -72,6 +72,22 @@ module Kotoshu
72
72
  # output is stable across machines and runs.
73
73
  TIER_PREFERENCE = %i[mini fluency full].freeze
74
74
 
75
+ # The language-identification artifact pair (plan 106): registry
76
+ # resource `kotoshu://models/lid/lid-176` — the ONNX-converted
77
+ # lid.176 model (int8-per-row, dim 16) plus its vocab.json
78
+ # sidecar. Lives in the same registry as the embedding tiers but
79
+ # is NOT a language tier: it has no language of its own and the
80
+ # tier machinery (resource ids, TIERS) does not apply. It is
81
+ # stored under a top-level `lid/` directory (not
82
+ # `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
83
+ # reporting per-language ids only.
84
+ LID_LANGUAGE = "lid"
85
+ LID_TIER = "lid-176"
86
+ LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
87
+
88
+ # Directory (under the cache root) holding the LID artifact pair.
89
+ LID_DIRECTORY = "lid"
90
+
75
91
  # Normalize and validate a tier name.
76
92
  #
77
93
  # @param tier [String, Symbol] "full", "fluency", or "mini"
@@ -248,6 +264,96 @@ module Kotoshu
248
264
  result
249
265
  end
250
266
 
267
+ # ---- LID artifact pair (plan 106) ----
268
+
269
+ # Whether the LID artifact pair is cached and complete.
270
+ #
271
+ # @return [Boolean]
272
+ def lid_cached?
273
+ !load_cached_lid.nil?
274
+ end
275
+
276
+ # Resolve the LID artifact pair from the cache only — never
277
+ # touches the network (the resolve half of the two-stage model).
278
+ #
279
+ # @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
280
+ # cached, nil when not set up
281
+ # @raise [Kotoshu::IntegrityError] cached bytes fail their
282
+ # recorded checksum
283
+ def load_cached_lid
284
+ dir = File.join(@cache_path, LID_DIRECTORY)
285
+ metadata_path = File.join(dir, "metadata.json")
286
+ return nil unless File.exist?(metadata_path)
287
+
288
+ metadata = read_metadata(metadata_path)
289
+ return nil unless metadata
290
+
291
+ onnx_file = File.join(dir, metadata["file"].to_s)
292
+ vocab_file = File.join(dir, metadata["vocab_file"].to_s)
293
+ return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
294
+ return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
295
+
296
+ verify_lid_integrity!(metadata, onnx_file, vocab_file)
297
+
298
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
299
+ end
300
+
301
+ # Download the LID artifact pair from the models registry
302
+ # (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
303
+ # against the registry sha256 before accepting them (the setup
304
+ # half of the two-stage model; same contract as
305
+ # {#download_tiered_model}).
306
+ #
307
+ # @param force [Boolean] re-fetch the registry first
308
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
309
+ # @raise [Kotoshu::Error] no registry entry, or offline without
310
+ # a cached registry (the registry is fetched once per cache
311
+ # instance; see {#registry})
312
+ # @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
313
+ # bytes are removed before raising)
314
+ def download_lid(force: false)
315
+ entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
316
+ unless entry
317
+ raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
318
+ end
319
+
320
+ dir = File.join(@cache_path, LID_DIRECTORY)
321
+ FileUtils.mkdir_p(dir)
322
+
323
+ onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
324
+ used_url = download_primary_or_mirror!(entry, onnx_file)
325
+ verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
326
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
327
+
328
+ # The vocab sidecar is required, not optional: the reader
329
+ # cannot score without its labels and Huffman counts.
330
+ vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
331
+ begin
332
+ download_file(entry.vocab_url, vocab_file)
333
+ rescue StandardError => e
334
+ File.delete(onnx_file) if File.exist?(onnx_file)
335
+ raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
336
+ end
337
+
338
+ metadata = {
339
+ version: entry.version,
340
+ url: used_url,
341
+ language: LID_LANGUAGE,
342
+ type: "lid",
343
+ tier: LID_TIER,
344
+ file: File.basename(onnx_file),
345
+ vocab_file: File.basename(vocab_file),
346
+ checksum: entry.sha256,
347
+ registry_id: LID_REGISTRY_ID,
348
+ size_bytes: entry.size_bytes,
349
+ cached_at: Time.now.utc.iso8601,
350
+ source: "registry"
351
+ }
352
+ write_metadata(File.join(dir, "metadata.json"), metadata)
353
+
354
+ { onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
355
+ end
356
+
251
357
  # Get available model types for a language.
252
358
  #
253
359
  # @param language_code [String] ISO 639-1 language code
@@ -629,7 +735,9 @@ module Kotoshu
629
735
  # Verify downloaded model bytes against the registry's sha256.
630
736
  # Corrupt bytes are removed from disk before raising so the next
631
737
  # attempt re-downloads (same contract as BaseCache#verify_and_audit).
632
- def verify_registry_sha256!(entry, model_file, resource_id, url)
738
+ # +remediation+ overrides the default re-download hint (the LID
739
+ # pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
740
+ def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
633
741
  actual = Digest::SHA256.file(model_file).hexdigest
634
742
  if actual == entry.sha256
635
743
  @audit_log.record(
@@ -649,7 +757,8 @@ module Kotoshu
649
757
  expected: entry.sha256,
650
758
  actual: actual,
651
759
  url: url,
652
- remediation: "Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
760
+ remediation: remediation ||
761
+ "Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
653
762
  )
654
763
  end
655
764
 
@@ -680,6 +789,45 @@ module Kotoshu
680
789
  # Caches written without a checksum field are accepted silently
681
790
  # to preserve backward compatibility with pre-verification caches.
682
791
  #
792
+ # Verify the cached LID pair against the checksum recorded at
793
+ # download time (which, for registry downloads, is the registry's
794
+ # sha256). A Git LFS pointer stub is refused by content first —
795
+ # same contract as {#verify_cached_integrity!}, against the
796
+ # fixed LID paths instead of a resource-id-derived layout.
797
+ #
798
+ # @param metadata [Hash] Parsed metadata.json (string keys)
799
+ # @param onnx_file [String] Path to the cached lid.176.onnx
800
+ # @param vocab_file [String] Path to the cached vocab sidecar
801
+ # @return [void]
802
+ # @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
803
+ # corrupt bytes are removed so the next setup re-downloads
804
+ def verify_lid_integrity!(metadata, onnx_file, vocab_file)
805
+ dir = File.join(@cache_path, LID_DIRECTORY)
806
+ if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
807
+ File.delete(onnx_file)
808
+ File.delete(vocab_file)
809
+ FileUtils.rm_f(File.join(dir, "metadata.json"))
810
+ raise Kotoshu::Error,
811
+ "Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
812
+ "artifact is a Git LFS pointer stub, not model content. Run " \
813
+ "Kotoshu.setup_lid(force: true) to re-download."
814
+ end
815
+
816
+ expected = metadata["checksum"]
817
+ return unless expected
818
+
819
+ actual = Digest::SHA256.file(onnx_file).hexdigest
820
+ return if actual == expected
821
+
822
+ raise Kotoshu::IntegrityError.new(
823
+ LID_REGISTRY_ID,
824
+ expected: expected,
825
+ actual: actual,
826
+ url: metadata["url"],
827
+ remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
828
+ )
829
+ end
830
+
683
831
  # @param resource_id [String] The resource identifier (e.g. "en:onnx")
684
832
  # @param metadata [Hash] Parsed metadata.json (string keys)
685
833
  # @param model_file [String] Path to the cached model file
data/lib/kotoshu/cli.rb CHANGED
@@ -162,6 +162,10 @@ module Kotoshu
162
162
  method_option :exclude,
163
163
  type: :array,
164
164
  desc: "Skip files matching these globs (always wins over --include and the extension default)"
165
+ method_option :personal,
166
+ type: :boolean,
167
+ default: true,
168
+ desc: "Accept words from the personal dictionary (personal.dic); --no-personal flags them again"
165
169
  # Check spelling in files, directories, or stdin. Cache-only —
166
170
  # never downloads; run `kotoshu setup LANG` first. See the
167
171
  # command long description (kotoshu help check) for flags,
@@ -395,6 +399,10 @@ module Kotoshu
395
399
  Kotoshu::Configuration.reset
396
400
  cfg = Kotoshu::Configuration.instance
397
401
  cfg.default_language = options[:language] if options[:language] && options[:language] != "auto"
402
+ # `kotoshu check --no-personal` opts out of the personal
403
+ # dictionary for this run (plan 105). Other commands do not
404
+ # declare the flag, so options[:personal] is nil there.
405
+ cfg.personal_dictionary = options[:personal] unless options[:personal].nil?
398
406
  end
399
407
 
400
408
  # Install a ProgressReporter on Configuration.download_reporter
@@ -209,6 +209,12 @@ module Kotoshu
209
209
  default: "ruby",
210
210
  description: "Engine backend for correct?/suggest (ruby, native, or auto)",
211
211
  type: String
212
+ },
213
+ personal_dictionary: {
214
+ env: "KOTOSHU_PERSONAL_DICTIONARY",
215
+ default: true,
216
+ description: "Words in the personal dictionary (personal.dic) pass the check path",
217
+ type: :boolean
212
218
  }
213
219
  }.freeze
214
220
 
@@ -229,7 +235,8 @@ module Kotoshu
229
235
  default_language: "en",
230
236
  resource_pin: "main",
231
237
  semantic_cascade_threshold: 1.0,
232
- backend: "ruby"
238
+ backend: "ruby",
239
+ personal_dictionary: true
233
240
  }.freeze
234
241
 
235
242
  # @return [String, nil] Path to the dictionary file
@@ -289,6 +296,15 @@ module Kotoshu
289
296
  # NativeBackend for the boundary — the semantic path stays Ruby.
290
297
  attr_accessor :backend
291
298
 
299
+ # @return [Boolean] Whether Spellchecker#check accepts words from
300
+ # the user personal dictionary (~/.config/kotoshu/personal.dic,
301
+ # KOTOSHU_PERSONAL_DIC override) instead of flagging them as
302
+ # errors — the README story and the LSP diagnostics behavior
303
+ # (plan 105). Opt out with KOTOSHU_PERSONAL_DICTIONARY=false or
304
+ # `kotoshu check --no-personal`; the suggest path is unaffected,
305
+ # so personal words stay spellcheckable.
306
+ attr_accessor :personal_dictionary
307
+
292
308
  # @return [String, nil] Path to cache directory
293
309
  attr_accessor :cache_path
294
310
 
@@ -489,7 +505,8 @@ module Kotoshu
489
505
  cache_ttl: @cache_ttl,
490
506
  max_cache_size: @max_cache_size,
491
507
  semantic_cascade_threshold: @semantic_cascade_threshold,
492
- backend: @backend
508
+ backend: @backend,
509
+ personal_dictionary: @personal_dictionary
493
510
  }
494
511
  end
495
512
 
@@ -607,6 +624,7 @@ module Kotoshu
607
624
  @resource_pin = DEFAULTS[:resource_pin]
608
625
  @semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
609
626
  @backend = DEFAULTS[:backend]
627
+ @personal_dictionary = DEFAULTS[:personal_dictionary]
610
628
  end
611
629
 
612
630
  # Apply resolved values from the resolver (ENV, defaults).
@@ -0,0 +1,188 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Language
5
+ # One language detection: the top label code (ISO 639) and its
6
+ # probability, the shape the plan-106 facade returns.
7
+ #
8
+ # A Struct (keyword_init) per the plain-data rule — no behavior
9
+ # beyond readers, #to_s (the code, so string interpolation reads
10
+ # naturally), and equality.
11
+ Detection = Struct.new(:code, :score, keyword_init: true) do
12
+ # The code, so `"detected #{detection}"` reads naturally.
13
+ def to_s
14
+ code.to_s
15
+ end
16
+ end
17
+
18
+ # Document language detection through the pure-Rust LID reader
19
+ # (plan 106).
20
+ #
21
+ # The gem's {LanguageIdentifier} needs the Python fastText
22
+ # bindings (broken under numpy 2), which is why /v1/detect and the
23
+ # CLI shipped a 7-language heuristic. The native extension now
24
+ # wraps kotoshu-rs' `lid` reader over the
25
+ # `kotoshu://models/lid/lid-176` registry artifact pair (the
26
+ # ONNX-converted lid.176 model + vocab sidecar, 176 languages,
27
+ # bit-faithful to the bindings — parity frozen by the kotoshu-rs
28
+ # suite over the shared corpus fixture).
29
+ #
30
+ # Selection mirrors {NativeBackend} with one difference: the
31
+ # native engine serves detect whenever the extension is built and
32
+ # the artifact pair is cached — an explicitly ruby backend
33
+ # (KOTOSHU_BACKEND=ruby) opts out, but the unset default does not,
34
+ # because the {Detector} heuristic is strictly weaker (7 languages
35
+ # vs 176). Pure-Ruby installs keep working unchanged — no new
36
+ # gemspec dependency, no onnxruntime.
37
+ #
38
+ # Two-stage like every resource: {setup} downloads the pair once
39
+ # (registry-resolved, sha-verified); {detect} resolves from the
40
+ # cache only and never downloads implicitly.
41
+ #
42
+ # @example Setup once, then detect
43
+ # Kotoshu.setup_lid
44
+ # Kotoshu.detect_language("Der Ausschuss prüft den Vorschlag")
45
+ # # => #<struct Kotoshu::Language::Detection code="de", score=0.99>
46
+ class LidDetector
47
+ class << self
48
+ # Detect the language of text.
49
+ #
50
+ # Native LID when available (see {LidDetector} for the
51
+ # selection rules), the {Detector} heuristic otherwise.
52
+ #
53
+ # @param text [String] Text to analyze
54
+ # @param config [Configuration] configuration to consult
55
+ # (defaults to the global instance)
56
+ # @return [Detection] code (nil when the heuristic is
57
+ # uncertain) and score in [0, 1]
58
+ def detect(text, config: Kotoshu.configuration)
59
+ # nil is not text; an empty/blank string IS (the native
60
+ # reader scores its EOS token row — "en", ~0.12 — exactly
61
+ # like the fastText bindings, per the frozen corpus).
62
+ return Detection.new(code: nil, score: 0.0) if text.nil?
63
+
64
+ model = native_model(config: config)
65
+ return heuristic_detection(text) unless model
66
+
67
+ row = model.detect(text)
68
+ Detection.new(code: row.fetch("code"), score: row.fetch("score"))
69
+ end
70
+
71
+ # Whether {detect} would run the native engine for this
72
+ # configuration — the extension is built, the backend is not
73
+ # "ruby", and the artifact pair is cached.
74
+ #
75
+ # @param config [Configuration]
76
+ # @return [Boolean]
77
+ def available?(config: Kotoshu.configuration)
78
+ !native_model(config: config).nil?
79
+ end
80
+
81
+ # Download the LID artifact pair into the model cache (stage
82
+ # one of the two-stage model; registry-resolved, sha-verified).
83
+ #
84
+ # @param config [Configuration]
85
+ # @param force [Boolean] re-fetch even if already cached
86
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
87
+ # @raise [Kotoshu::Error] no registry entry, or offline with
88
+ # no cached registry (KOTOSHU_OFFLINE=1)
89
+ # @raise [Kotoshu::IntegrityError] downloaded bytes fail the
90
+ # registry checksum
91
+ def setup(config: Kotoshu.configuration, force: false)
92
+ pair = model_cache(config: config).load_cached_lid
93
+ return pair if pair && !force
94
+
95
+ model_cache(config: config).download_lid(force: force)
96
+ end
97
+
98
+ # Whether the LID artifact pair is cached (cache-only check).
99
+ #
100
+ # @param config [Configuration]
101
+ # @return [Boolean]
102
+ def setup?(config: Kotoshu.configuration)
103
+ model_cache(config: config).lid_cached?
104
+ end
105
+
106
+ # Drop the memoized native model. The next {detect} re-resolves
107
+ # from the cache — the seam tests and embedders use after
108
+ # {setup} or a cache purge (mirrors Kotoshu.reset_spellchecker).
109
+ #
110
+ # @return [nil]
111
+ def reset
112
+ @native_model = nil
113
+ nil
114
+ end
115
+
116
+ private
117
+
118
+ # The loaded native LID model, memoized per process — one
119
+ # parse of the ~3 MB pair, mirroring how spellcheckers are
120
+ # cached per language. Nil whenever the native engine cannot
121
+ # serve: extension absent, Kotoshu::Native::LidModel missing
122
+ # (extension predates plan 106), an explicit ruby backend, or
123
+ # the pair not set up.
124
+ #
125
+ # @param config [Configuration]
126
+ # @return [Kotoshu::Native::LidModel, nil]
127
+ def native_model(config:)
128
+ return nil unless native_lid_surface?
129
+ return nil if backend_opts_out?(config)
130
+
131
+ @native_model ||= begin
132
+ pair = model_cache(config: config).load_cached_lid
133
+ pair && Kotoshu::Native::LidModel.load(pair[:onnx_path], pair[:vocab_path])
134
+ end
135
+ end
136
+
137
+ # Whether the extension carries the LID surface at all.
138
+ def native_lid_surface?
139
+ Kotoshu::Native.available? && Kotoshu::Native.const_defined?(:LidModel)
140
+ end
141
+
142
+ # Backend selection for LID (plan 106).
143
+ #
144
+ # KOTOSHU_BACKEND=ruby opts the LID out of the native engine.
145
+ # When the backend is UNSET, LID still prefers the native
146
+ # engine — unlike correct?/suggest, where "ruby" is the engine
147
+ # default and "native" the explicit accelerator. The heuristic
148
+ # is strictly weaker than the 176-language model, so it stays
149
+ # the fallback for the cannot-serve cases (no extension, no
150
+ # cached model), not for an unset preference.
151
+ #
152
+ # Explicitness is read through the resolver layers (live ENV,
153
+ # constructor settings, CLI options) so the "ruby" DEFAULT
154
+ # never counts as an opt-out. Mutating
155
+ # Kotoshu.configuration.backend after construction is not
156
+ # visible here — rebuild the configuration instead.
157
+ def backend_opts_out?(config)
158
+ config.backend.to_s == "ruby" && backend_set?(config)
159
+ end
160
+
161
+ # Whether the backend was explicitly set at any resolver layer
162
+ # (rather than left at its "ruby" default).
163
+ def backend_set?(config)
164
+ resolver = config.resolver
165
+ ENV.key?("KOTOSHU_BACKEND") ||
166
+ resolver.programmatic.key?(:backend) ||
167
+ resolver.cli.key?(:backend)
168
+ end
169
+
170
+ # The heuristic fallback, wrapped in the same {Detection}
171
+ # shape the native path returns.
172
+ def heuristic_detection(text)
173
+ code, confidence = Detector.detect_with_confidence(text)
174
+ Detection.new(code: code, score: confidence)
175
+ end
176
+
177
+ # A cache over the configuration's model cache path and
178
+ # source registry (so KOTOSHU_MODELS_PIN is honored).
179
+ def model_cache(config:)
180
+ Cache::ModelCache.new(
181
+ cache_path: config.cache_path,
182
+ source_registry: config.source_registry
183
+ )
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
@@ -17,7 +17,11 @@ module Kotoshu
17
17
  module Language
18
18
  autoload :Registry, "kotoshu/language/registry"
19
19
  autoload :Detector, "kotoshu/language/detector"
20
+ # Detection value + the native LID reader behind it (plan 106) share
21
+ # lid_detector.rb.
22
+ autoload :Detection, "kotoshu/language/lid_detector"
20
23
  autoload :LanguageIdentifier, "kotoshu/language/identifier"
24
+ autoload :LidDetector, "kotoshu/language/lid_detector"
21
25
  autoload :Segmenter, "kotoshu/language/segmenter"
22
26
  autoload :Segment, "kotoshu/language/segmenter"
23
27
  autoload :Suika, "kotoshu/language/suika"
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "set"
4
+
3
5
  module Kotoshu
4
6
  # Main spellchecker class.
5
7
  #
@@ -175,6 +177,15 @@ module Kotoshu
175
177
 
176
178
  # Check text for spelling errors.
177
179
  #
180
+ # Words present in the user's personal dictionary
181
+ # (~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override)
182
+ # never surface as errors — the same semantics the LSP applies to
183
+ # its diagnostics (plan 105). They are dropped at result assembly
184
+ # without suppression metadata, still count toward +word_count+,
185
+ # and remain spellcheckable through {#suggest}. Opt out with
186
+ # {Configuration#personal_dictionary} (KOTOSHU_PERSONAL_DICTIONARY,
187
+ # `kotoshu check --no-personal`).
188
+ #
178
189
  # @param text [String] The text to check
179
190
  # @return [Models::Result::DocumentResult] The check result
180
191
  #
@@ -191,9 +202,12 @@ module Kotoshu
191
202
  # because the facade sees raw text without a format parser.
192
203
  words = tokenize(text)
193
204
  suppressions = Documents::Suppressions.scan(text, format: :auto)
205
+ personal = personal_words
194
206
  errors = []
195
207
  suppressed_errors = []
196
208
  words.each do |word, pos|
209
+ next if personal.include?(word.downcase)
210
+
197
211
  result = check_word(word)
198
212
  next if result.correct?
199
213
 
@@ -318,6 +332,27 @@ module Kotoshu
318
332
 
319
333
  private
320
334
 
335
+ # The personal dictionary as a downcased Set for the check path
336
+ # (plan 105), mirroring the LSP's filter. Loaded once per process:
337
+ # the first {#check} call snapshots the words, and later edits to
338
+ # personal.dic take effect in the next process (the LSP reloads on
339
+ # mtime change; this path deliberately does not — one-shot CLI runs
340
+ # always read fresh, and long-lived embedders rebuild the
341
+ # spellchecker via {Kotoshu.reset_spellchecker}). An empty set when
342
+ # disabled through {Configuration#personal_dictionary}.
343
+ #
344
+ # @return [Set<String>]
345
+ def personal_words
346
+ return @personal_words if instance_variable_defined?(:@personal_words)
347
+
348
+ @personal_words =
349
+ if @config.personal_dictionary
350
+ PersonalDictionary.words.map(&:downcase).to_set
351
+ else
352
+ Set.new
353
+ end
354
+ end
355
+
321
356
  # Resolve the tokenizer of the language being checked for word
322
357
  # extraction (plan 91).
323
358
  #
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kotoshu
4
- VERSION = '0.9.3'
4
+ VERSION = '0.10.0'
5
5
  end
data/lib/kotoshu.rb CHANGED
@@ -446,18 +446,31 @@ module Kotoshu
446
446
 
447
447
  # Detect language of text.
448
448
  #
449
+ # Native LID when available (plan 106): the Rust extension scoring
450
+ # the `kotoshu://models/lid/lid-176` artifact pair over 176
451
+ # languages, after a one-time {setup_lid}. Falls back to the
452
+ # {Language::Detector} heuristic when the extension is absent,
453
+ # KOTOSHU_BACKEND=ruby, or the model is not set up — so pure-Ruby
454
+ # installs keep the previous behavior.
455
+ #
449
456
  # @param text [String] Text to analyze
450
- # @return [String, nil] Detected language code
457
+ # @return [Language::Detection] +code+ (String, or nil when the
458
+ # heuristic is uncertain) and +score+ (Float in [0, 1]);
459
+ # interpolation yields the code
451
460
  #
452
461
  # @example
453
- # Kotoshu.detect_language("Bonjour le monde") # => "fr"
454
- # Kotoshu.detect_language("こんにちは") # => "ja"
462
+ # Kotoshu.setup_lid
463
+ # Kotoshu.detect_language("Bonjour le monde").code # => "fr"
464
+ # Kotoshu.detect_language("こんにちは").score # => 0.99
455
465
  def self.detect_language(text)
456
- Language.detect(text)
466
+ Language::LidDetector.detect(text)
457
467
  end
458
468
 
459
469
  # Detect language with confidence score.
460
470
  #
471
+ # Same engine selection as {detect_language}, returned as the
472
+ # historical [code, confidence] pair.
473
+ #
461
474
  # @param text [String] Text to analyze
462
475
  # @return [Array<String, Float>] Language code and confidence
463
476
  #
@@ -466,7 +479,34 @@ module Kotoshu
466
479
  # lang # => "en"
467
480
  # conf # => 0.85
468
481
  def self.detect_language_with_confidence(text)
469
- Language.detect_with_confidence(text)
482
+ detection = Language::LidDetector.detect(text)
483
+ [detection.code, detection.score]
484
+ end
485
+
486
+ # Set up the language-identification model: downloads the
487
+ # lid.176.onnx + vocab.json pair from the models registry into the
488
+ # model cache (stage one of the two-stage model — {detect_language}
489
+ # never downloads implicitly). Idempotent; +force+ re-downloads.
490
+ #
491
+ # @param force [Boolean] Re-fetch even if already cached
492
+ # @return [Hash] { onnx_path:, vocab_path:, metadata }
493
+ # @raise [Kotoshu::Error] registry unreachable while offline with no
494
+ # cached registry (KOTOSHU_OFFLINE=1), or no registry entry
495
+ # @raise [Kotoshu::IntegrityError] downloaded bytes fail the
496
+ # registry checksum
497
+ #
498
+ # @example
499
+ # Kotoshu.setup_lid
500
+ # Kotoshu.setup_lid? # => true
501
+ def self.setup_lid(force: false)
502
+ Language::LidDetector.setup(config: configuration, force: force)
503
+ end
504
+
505
+ # Whether the language-identification model is cached (cache-only).
506
+ #
507
+ # @return [Boolean]
508
+ def self.setup_lid?
509
+ Language::LidDetector.setup?(config: configuration)
470
510
  end
471
511
 
472
512
  # Get language class by code.
data/sig/kotoshu.rbs CHANGED
@@ -106,6 +106,27 @@ def spellcheck_word_regex: () -> Regexp
106
106
  def normalize: (String? text, ?options ::Hash[Symbol, untyped]) -> String
107
107
  end
108
108
  end
109
+
110
+ # One language detection: top label code and its probability
111
+ # (plan 106). Plain-data Struct.
112
+ class Detection
113
+ attr_reader code: String?
114
+ attr_reader score: Float
115
+
116
+ def initialize: (?code: String?, ?score: Float) -> void
117
+ def to_s: () -> String
118
+ end
119
+
120
+ # Document language detection through the pure-Rust LID reader
121
+ # (plan 106): kotoshu://models/lid/lid-176 scored by the native
122
+ # extension, Language::Detector heuristic as the fallback.
123
+ class LidDetector
124
+ def self.detect: (String? text, ?config: Configuration) -> Detection
125
+ def self.available?: (?config: Configuration) -> bool
126
+ def self.setup: (?config: Configuration, ?force: bool) -> Hash[Symbol, untyped]
127
+ def self.setup?: (?config: Configuration) -> bool
128
+ def self.reset: () -> nil
129
+ end
109
130
  end
110
131
 
111
132
  module Languages
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kotoshu
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.3
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
@@ -108,6 +108,8 @@ files:
108
108
  - TODO.impl/102-lid-chain.md
109
109
  - TODO.impl/103-oov-bucket-tables.md
110
110
  - TODO.impl/104-vscode-marketplace-readiness.md
111
+ - TODO.impl/105-personal-dictionary-in-check.md
112
+ - TODO.impl/106-native-lid-detection.md
111
113
  - TODO.impl/11-release-v1.md
112
114
  - TODO.impl/20-encapsulation-cleanup.md
113
115
  - TODO.impl/21-autoload-migration.md
@@ -356,6 +358,7 @@ files:
356
358
  - lib/kotoshu/language/detector.rb
357
359
  - lib/kotoshu/language/identifier.rb
358
360
  - lib/kotoshu/language/languages/base.rb
361
+ - lib/kotoshu/language/lid_detector.rb
359
362
  - lib/kotoshu/language/normalizer/arabic.rb
360
363
  - lib/kotoshu/language/normalizer/base.rb
361
364
  - lib/kotoshu/language/normalizer/greek.rb