kotoshu 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/Cargo.lock +72 -4
- data/TODO.impl/00-vision.md +15 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/ext/kotoshu_native/Cargo.toml +5 -2
- data/ext/kotoshu_native/src/lib.rs +134 -6
- data/lib/kotoshu/cache/model_cache.rb +150 -2
- data/lib/kotoshu/cli.rb +8 -0
- data/lib/kotoshu/configuration.rb +20 -2
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language.rb +4 -0
- data/lib/kotoshu/spellchecker.rb +35 -0
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +45 -5
- data/sig/kotoshu.rbs +21 -0
- metadata +4 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a0093eb0f194f55beb4950a039c37e4b3abe6b33d073e7d7a8de0958e22a32cc
|
|
4
|
+
data.tar.gz: 795382646b0e0fdee0010b76d8fcd9afefa55447f513494ec3e832fe225f6110
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 59fb8d99d938013b349ca779593a72b2fa8390cf725950c07acd9e117fde7ff138710fb9dfa1ba3d7581947d261dea0f990f555e1456421156316c746e8d66a8
|
|
7
|
+
data.tar.gz: 18c2713e916171510e45028f08968d443248f5426c614e56ab21a32078543bdad69dd7f6aed4d44ced5ee00f7d18554f9050f19e84a3fb936798d3846bb00838
|
data/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
10
|
### Added
|
|
11
|
+
- **Personal dictionary in the check path** (plan 105) -
|
|
12
|
+
`Spellchecker#check` (and therefore `Kotoshu.check`,
|
|
13
|
+
`Kotoshu.check_file`, and `kotoshu check`) no longer flags words
|
|
14
|
+
present in the user personal dictionary
|
|
15
|
+
(~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override),
|
|
16
|
+
matching the semantics the LSP already applies to diagnostics:
|
|
17
|
+
case-insensitive, filtered at result assembly without suppression
|
|
18
|
+
metadata, words still counted, and `suggest`/`correct?` untouched so
|
|
19
|
+
personal words stay spellcheckable. The dictionary is loaded once
|
|
20
|
+
per process (no hot reload; the LSP keeps its mtime-based reload).
|
|
21
|
+
Opt out with `Kotoshu.configure { |c| c.personal_dictionary = false }`
|
|
22
|
+
/ KOTOSHU_PERSONAL_DICTIONARY=false / `kotoshu check --no-personal`.
|
|
23
|
+
- **Native language detection** (plan 106) - `Kotoshu.detect_language(text)`
|
|
24
|
+
now returns a `Kotoshu::Language::Detection` (`code` + `score`) scored by
|
|
25
|
+
the Rust extension over the `kotoshu://models/lid/lid-176` registry
|
|
26
|
+
artifact pair (the ONNX-converted lid.176 model, 176 languages,
|
|
27
|
+
bit-faithful to the fastText bindings - parity 55/55 against the frozen
|
|
28
|
+
corpus shared with kotoshu-rs). Fetch the pair once with
|
|
29
|
+
`Kotoshu.setup_lid` (registry-resolved, sha-verified, cached under
|
|
30
|
+
`models-cache/lid/`); detect resolves cache-only and never downloads.
|
|
31
|
+
The 7-language `Language::Detector` heuristic stays the fallback when
|
|
32
|
+
the extension is absent, the model is not set up, or
|
|
33
|
+
`KOTOSHU_BACKEND=ruby` opts out - so pure-Ruby installs are unaffected
|
|
34
|
+
(no new gemspec dependency, no onnxruntime). The magnus shim
|
|
35
|
+
(ext/kotoshu_native) gains `Kotoshu::Native::LidModel.load` +
|
|
36
|
+
`#detect` behind the kotoshu-rs `model` feature; the workspace
|
|
37
|
+
Cargo.lock moves to the kotoshu-rs revision carrying the reader.
|
|
38
|
+
`Kotoshu.detect_language_with_confidence` keeps its `[code, score]`
|
|
39
|
+
shape and now routes through the same engine.
|
|
11
40
|
- **Full-feature language batch 3** (plan 100) - twelve languages gain the
|
|
12
41
|
downloadable spelling dictionary, a keyboard layout, and a gem module:
|
|
13
42
|
`ar` `id` `fa` `he` `bg` `sr` `hr` `sk` `sl` `lt` `lv` `et` (the RTL
|
data/Cargo.lock
CHANGED
|
@@ -26,7 +26,7 @@ dependencies = [
|
|
|
26
26
|
"regex",
|
|
27
27
|
"rustc-hash",
|
|
28
28
|
"shlex",
|
|
29
|
-
"syn",
|
|
29
|
+
"syn 2.0.119",
|
|
30
30
|
]
|
|
31
31
|
|
|
32
32
|
[[package]]
|
|
@@ -82,12 +82,20 @@ dependencies = [
|
|
|
82
82
|
"either",
|
|
83
83
|
]
|
|
84
84
|
|
|
85
|
+
[[package]]
|
|
86
|
+
name = "itoa"
|
|
87
|
+
version = "1.0.18"
|
|
88
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
89
|
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
90
|
+
|
|
85
91
|
[[package]]
|
|
86
92
|
name = "kotoshu"
|
|
87
93
|
version = "0.1.0"
|
|
88
|
-
source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#
|
|
94
|
+
source = "git+https://github.com/kotoshu/kotoshu-rs?branch=main#e574c445df5f6eee8e978f1f4d26dc48cfb1947b"
|
|
89
95
|
dependencies = [
|
|
90
96
|
"magnus",
|
|
97
|
+
"serde",
|
|
98
|
+
"serde_json",
|
|
91
99
|
]
|
|
92
100
|
|
|
93
101
|
[[package]]
|
|
@@ -141,7 +149,7 @@ checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
|
|
|
141
149
|
dependencies = [
|
|
142
150
|
"proc-macro2",
|
|
143
151
|
"quote",
|
|
144
|
-
"syn",
|
|
152
|
+
"syn 2.0.119",
|
|
145
153
|
]
|
|
146
154
|
|
|
147
155
|
[[package]]
|
|
@@ -205,7 +213,7 @@ dependencies = [
|
|
|
205
213
|
"quote",
|
|
206
214
|
"regex",
|
|
207
215
|
"shell-words",
|
|
208
|
-
"syn",
|
|
216
|
+
"syn 2.0.119",
|
|
209
217
|
]
|
|
210
218
|
|
|
211
219
|
[[package]]
|
|
@@ -255,6 +263,49 @@ version = "0.3.6"
|
|
|
255
263
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
256
264
|
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|
257
265
|
|
|
266
|
+
[[package]]
|
|
267
|
+
name = "serde"
|
|
268
|
+
version = "1.0.229"
|
|
269
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
270
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
271
|
+
dependencies = [
|
|
272
|
+
"serde_core",
|
|
273
|
+
"serde_derive",
|
|
274
|
+
]
|
|
275
|
+
|
|
276
|
+
[[package]]
|
|
277
|
+
name = "serde_core"
|
|
278
|
+
version = "1.0.229"
|
|
279
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
280
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
281
|
+
dependencies = [
|
|
282
|
+
"serde_derive",
|
|
283
|
+
]
|
|
284
|
+
|
|
285
|
+
[[package]]
|
|
286
|
+
name = "serde_derive"
|
|
287
|
+
version = "1.0.229"
|
|
288
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
289
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
290
|
+
dependencies = [
|
|
291
|
+
"proc-macro2",
|
|
292
|
+
"quote",
|
|
293
|
+
"syn 3.0.5",
|
|
294
|
+
]
|
|
295
|
+
|
|
296
|
+
[[package]]
|
|
297
|
+
name = "serde_json"
|
|
298
|
+
version = "1.0.151"
|
|
299
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
300
|
+
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
|
301
|
+
dependencies = [
|
|
302
|
+
"itoa",
|
|
303
|
+
"memchr",
|
|
304
|
+
"serde",
|
|
305
|
+
"serde_core",
|
|
306
|
+
"zmij",
|
|
307
|
+
]
|
|
308
|
+
|
|
258
309
|
[[package]]
|
|
259
310
|
name = "shell-words"
|
|
260
311
|
version = "1.1.1"
|
|
@@ -278,6 +329,17 @@ dependencies = [
|
|
|
278
329
|
"unicode-ident",
|
|
279
330
|
]
|
|
280
331
|
|
|
332
|
+
[[package]]
|
|
333
|
+
name = "syn"
|
|
334
|
+
version = "3.0.5"
|
|
335
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
336
|
+
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
|
|
337
|
+
dependencies = [
|
|
338
|
+
"proc-macro2",
|
|
339
|
+
"quote",
|
|
340
|
+
"unicode-ident",
|
|
341
|
+
]
|
|
342
|
+
|
|
281
343
|
[[package]]
|
|
282
344
|
name = "unicode-ident"
|
|
283
345
|
version = "1.0.24"
|
|
@@ -289,3 +351,9 @@ name = "windows-link"
|
|
|
289
351
|
version = "0.2.1"
|
|
290
352
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
291
353
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
354
|
+
|
|
355
|
+
[[package]]
|
|
356
|
+
name = "zmij"
|
|
357
|
+
version = "1.0.23"
|
|
358
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
359
|
+
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
data/TODO.impl/00-vision.md
CHANGED
|
@@ -247,3 +247,18 @@ plan name the source.
|
|
|
247
247
|
| `docs/ARCHITECTURE_IMPROVEMENTS_PLAN.md` (5 phases) | Phase 1 perf items in `01`/`05`; Phase 2 architecture in `00-vision` (this doc); Phase 3 UX mostly implemented (`PersonalDictionary`, `ProjectConfig`, `FluentChecker` exist); Phase 4 quality in `10`; Phase 5 polish spread |
|
|
248
248
|
| `docs/TDD_ITERATION_STRATEGY.md` | `10` (test methodology + CI); project-rule overrides (no doubles, behavior-not-implementation) in `CLAUDE.md` |
|
|
249
249
|
| `docs/ONNX_FUNCTIONAL_VERIFICATION.md` | `05` (semantic path productionization) |
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
Plans 105-107 plus models 09 and site 11 are **wave 7, detection and
|
|
253
|
+
coverage depth** (2026-09-07): after the wave-6 releases (gem 0.9.3,
|
|
254
|
+
wasm 0.4.0, registry v1.4.0, action v2), the remaining gaps are the
|
|
255
|
+
personal dictionary never reaching the check path, detection stuck on
|
|
256
|
+
a 7-language heuristic, buckets only for en/de, and the playground not
|
|
257
|
+
yet passing bucket bytes to loadModel.
|
|
258
|
+
|
|
259
|
+
| # | Plan | Gap it closes |
|
|
260
|
+
|---|---|---|
|
|
261
|
+
| 105 | [Personal dictionary in check](105-personal-dictionary-in-check.md) | kotoshu personal add then check still flags the word |
|
|
262
|
+
| 106 | [Native LID detection](106-native-lid-detection.md) | detect is a 7-language heuristic; the pure-Rust lid reader exists unused |
|
|
263
|
+
| 107 | models [09 bucket expansion](../../models-fasttext-onnx/TODO.impl/09-bucket-table-expansion.md) | bucket OOV siblings exist only for en and de |
|
|
264
|
+
| — | site 11 playground buckets | wasm 0.4.0 accepts bucketsBytes; the worker never fetches the sibling |
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Plan 105 — The personal dictionary reaches the check path
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
`kotoshu personal add recieve` then `kotoshu check` still flags the word —
|
|
5
|
+
PersonalDictionary is wired only into the `personal` subcommand and the LSP
|
|
6
|
+
(resolved server-side), not into `Spellchecker#check`. Users who add words
|
|
7
|
+
see them flagged again on the very next run; the README story ("add words
|
|
8
|
+
you accept") is currently false on the CLI/API path.
|
|
9
|
+
|
|
10
|
+
## Work (gem repo)
|
|
11
|
+
1. `Spellchecker#check` consults the personal dictionary: a word present in
|
|
12
|
+
personal.dic is not an error (mirror the LSP's semantics: filter at
|
|
13
|
+
result assembly, keep suppression metadata out of the error rows).
|
|
14
|
+
2. Reload semantics: the LSP reloads on mtime change; the CLI/one-shot path
|
|
15
|
+
loads once per process. Do not add hot reload here.
|
|
16
|
+
3. Opt-out: `Kotoshu.configure { personal_dictionary false }` (or the
|
|
17
|
+
existing config seam — find it) and `--no-personal` on `kotoshu check`.
|
|
18
|
+
4. Specs: add-then-check round trip on a real dictionary; opt-out flags
|
|
19
|
+
again; personal words still spellcheckable via suggest.
|
|
20
|
+
5. Docs truth: /docs/ignores gains the personal dictionary section (the
|
|
21
|
+
action PR #2 README noted this gap) — deliver the copy in the PR body
|
|
22
|
+
for the site follow-up, do not touch the site repo.
|
|
23
|
+
|
|
24
|
+
## Verification
|
|
25
|
+
Suite green (3,900+ examples); rubocop clean; `kotoshu personal add x &&
|
|
26
|
+
kotoshu check` live round trip recorded in the PR body.
|
|
27
|
+
|
|
28
|
+
## Status
|
|
29
|
+
Executed 2026-09-07 — gem PR (branch plan-105-personal-dict-in-check):
|
|
30
|
+
check-path filter mirroring the LSP, one load per process,
|
|
31
|
+
Configuration#personal_dictionary + KOTOSHU_PERSONAL_DICTIONARY +
|
|
32
|
+
`kotoshu check --no-personal`; suite pins KOTOSHU_PERSONAL_DIC for
|
|
33
|
+
hermeticity. /docs/ignores copy delivered in the PR body for the site
|
|
34
|
+
follow-up.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Plan 106 — Document language detection through the pure-Rust LID reader
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
The gem's LanguageIdentifier needs the Python fastText bindings (and breaks
|
|
5
|
+
under numpy 2), so /v1/detect ships a 7-language heuristic. kotoshu-rs now
|
|
6
|
+
carries a pure-Rust lid.176 reader (kotoshu/src/lid, parity 55/55 vs the
|
|
7
|
+
gem) behind the same `model` feature the native ext already builds.
|
|
8
|
+
|
|
9
|
+
## Work (gem repo)
|
|
10
|
+
1. Expose `Kotoshu.detect_language(text)` through the native backend:
|
|
11
|
+
download/resolve lid.176.onnx + vocab via the models registry
|
|
12
|
+
(kotoshu://models/lid/lid-176, v1.4.0), score with the ext, return
|
|
13
|
+
{ code, score }. Heuristic Language::Detector stays the fallback when
|
|
14
|
+
the native ext is absent or the model is not set up.
|
|
15
|
+
2. FFI seam: extend the magnus shim (ext/kotoshu_native) with the call —
|
|
16
|
+
mirror how suggest/check already cross; keep it behind the existing
|
|
17
|
+
feature so pure-Ruby installs are unaffected.
|
|
18
|
+
3. Server follow-up (kotoshu-server repo, separate PR riding this):
|
|
19
|
+
/v1/detect prefers Kotoshu.detect_language when available. Note it in
|
|
20
|
+
the PR body; do not touch the server repo yourself.
|
|
21
|
+
4. Specs: native-vs-fixture parity (reuse the rs corpus JSON shape),
|
|
22
|
+
fallback behavior, registry miss error.
|
|
23
|
+
|
|
24
|
+
## Verification
|
|
25
|
+
Suite green; a live detect over en/de/ru/ja/ar samples recorded in the PR
|
|
26
|
+
body; pure-Ruby suite still green (no new hard dependency).
|
|
27
|
+
|
|
28
|
+
## Status
|
|
29
|
+
Executed 2026-09-07 — gem PR (branch plan-106-native-lid-detection):
|
|
30
|
+
Kotoshu.detect_language -> Language::Detection through the ext
|
|
31
|
+
(LidModel.load/#detect in the magnus shim, kotoshu-rs model feature,
|
|
32
|
+
Cargo.lock bumped to the reader revision); ModelCache gained the
|
|
33
|
+
lid pair (download_lid/load_cached_lid, top-level lid/ dir);
|
|
34
|
+
Kotoshu.setup_lid two-stage; heuristic fallback incl.
|
|
35
|
+
KOTOSHU_BACKEND=ruby; parity 55/55 vs the shared frozen corpus.
|
|
36
|
+
Server follow-up (/v1/detect) noted in the PR body.
|
|
@@ -33,5 +33,8 @@ rb-sys = { version = "0.9.124", features = ["global-allocator"] }
|
|
|
33
33
|
# and are a documented relax trigger for the P4 gate, not adopted here).
|
|
34
34
|
magnus = "0.8"
|
|
35
35
|
|
|
36
|
-
# The engine. Feature "ruby" turns on ffi::ruby (magnus bindings)
|
|
37
|
-
|
|
36
|
+
# The engine. Feature "ruby" turns on ffi::ruby (magnus bindings);
|
|
37
|
+
# "model" turns on the pure-Rust LID reader (kotoshu::lid, plan 106) —
|
|
38
|
+
# the same feature the core crate ships it under, so pure-Ruby installs
|
|
39
|
+
# that never build this extension are unaffected.
|
|
40
|
+
kotoshu = { git = "https://github.com/kotoshu/kotoshu-rs", branch = "main", features = ["ruby", "model"] }
|
|
@@ -1,15 +1,143 @@
|
|
|
1
1
|
//! Kotoshu native extension (plan 66 / TODO.impl 05, P4b).
|
|
2
2
|
//!
|
|
3
|
-
//! The
|
|
4
|
-
//!
|
|
5
|
-
//!
|
|
6
|
-
//!
|
|
3
|
+
//! The extension forwards its `#[magnus::init]` to the engine core's
|
|
4
|
+
//! Ruby bindings. `kotoshu::ffi::ruby::init` defines the `Kotoshu::Native`
|
|
5
|
+
//! module (`VERSION`, `available?`, `Dictionary.load` with `correct?` /
|
|
6
|
+
//! `suggest`); this crate contains no engine logic of its own,
|
|
7
7
|
//! exactly like kotoshu-rs' `tests/ruby_ext` reference shim and
|
|
8
8
|
//! parsanol-ruby's `ext/parsanol_native`.
|
|
9
|
+
//!
|
|
10
|
+
//! One binding lives here instead of the core crate: the language
|
|
11
|
+
//! identification surface (plan 106). `LidModel.load` +
|
|
12
|
+
//! `LidModel#detect` wrap `kotoshu::lid::LidModel` — the pure-Rust
|
|
13
|
+
//! reader for the `kotoshu://models/lid/lid-176` artifact pair —
|
|
14
|
+
//! behind the same `model` feature the core crate ships it under
|
|
15
|
+
//! (see the features table in Cargo.toml). Only the magnus glue is
|
|
16
|
+
//! gem-side; every scoring decision stays in the core, where the
|
|
17
|
+
//! parity suite freezes it against the shared frozen corpus.
|
|
18
|
+
|
|
19
|
+
use std::path::Path;
|
|
20
|
+
|
|
21
|
+
use magnus::typed_data::Obj;
|
|
22
|
+
use magnus::typed_data::TypedData;
|
|
23
|
+
use magnus::value::Lazy;
|
|
24
|
+
use magnus::{
|
|
25
|
+
Class, DataType, DataTypeFunctions, Error, ExceptionClass, Module, Object, RClass, RHash,
|
|
26
|
+
RModule, Ruby, data_type_builder, method,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/// The `Kotoshu` module, defined (idempotently) on first access — the
|
|
30
|
+
/// same helper shape as the core crate's `ffi::ruby`.
|
|
31
|
+
fn kotoshu_module(ruby: &Ruby) -> RModule {
|
|
32
|
+
static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
|
|
33
|
+
ruby.define_module("Kotoshu")
|
|
34
|
+
.expect("cannot define Kotoshu module")
|
|
35
|
+
});
|
|
36
|
+
ruby.get_inner(&MODULE)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// The `Kotoshu::Native` module hosting the engine bindings.
|
|
40
|
+
fn native_module(ruby: &Ruby) -> RModule {
|
|
41
|
+
static MODULE: Lazy<RModule> = Lazy::new(|ruby| {
|
|
42
|
+
kotoshu_module(ruby)
|
|
43
|
+
.define_module("Native")
|
|
44
|
+
.expect("cannot define Kotoshu::Native module")
|
|
45
|
+
});
|
|
46
|
+
ruby.get_inner(&MODULE)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/// `Kotoshu::Native::Error`, defined by the core crate's init — every
|
|
50
|
+
/// LID failure crosses the boundary as the same exception class the
|
|
51
|
+
/// dictionary surface raises.
|
|
52
|
+
fn error_class(ruby: &Ruby) -> ExceptionClass {
|
|
53
|
+
static CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
|
|
54
|
+
native_module(ruby)
|
|
55
|
+
.const_get::<_, ExceptionClass>("Error")
|
|
56
|
+
.expect("Kotoshu::Native::Error not defined after ffi::ruby::init")
|
|
57
|
+
});
|
|
58
|
+
ruby.get_inner(&CLASS)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// A loaded LID model wrapped as a Ruby object: the twin of the core
|
|
62
|
+
/// crate's `RubyDictionary`. Ruby's GC owns it once wrapped; dropping
|
|
63
|
+
/// the Ruby object drops the engine model (~1 MB of int8 rows plus
|
|
64
|
+
/// ~1.8 MB of sidecar tables, roughly the artifact pair's own size).
|
|
65
|
+
#[derive(Debug)]
|
|
66
|
+
pub struct RubyLidModel {
|
|
67
|
+
inner: kotoshu::lid::LidModel,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Plain owned data; Ruby objects move between threads only with the
|
|
71
|
+
// GVL held, which TypedData's Send bound models.
|
|
72
|
+
impl DataTypeFunctions for RubyLidModel {}
|
|
73
|
+
|
|
74
|
+
unsafe impl TypedData for RubyLidModel {
|
|
75
|
+
fn class(ruby: &Ruby) -> RClass {
|
|
76
|
+
static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
|
|
77
|
+
let class = native_module(ruby)
|
|
78
|
+
.define_class("LidModel", ruby.class_object())
|
|
79
|
+
.expect("cannot define Kotoshu::Native::LidModel");
|
|
80
|
+
// Instances exist only through `LidModel.load`; `new`/
|
|
81
|
+
// `allocate` would produce a model-less object.
|
|
82
|
+
class.undef_default_alloc_func();
|
|
83
|
+
class
|
|
84
|
+
});
|
|
85
|
+
ruby.get_inner(&CLASS)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
fn data_type() -> &'static DataType {
|
|
89
|
+
static DATA_TYPE: DataType =
|
|
90
|
+
data_type_builder!(RubyLidModel, "Kotoshu/Native/LidModel").build();
|
|
91
|
+
&DATA_TYPE
|
|
92
|
+
}
|
|
93
|
+
}
|
|
9
94
|
|
|
10
|
-
|
|
95
|
+
/// `Kotoshu::Native::LidModel.load(onnx_path, vocab_path)` — reads the
|
|
96
|
+
/// registry artifact pair (the `lid.176.onnx` container and its
|
|
97
|
+
/// `lid.176.vocab.json` sidecar) and wraps the parsed model. Failures
|
|
98
|
+
/// raise `Kotoshu::Native::Error` carrying the Rust error message,
|
|
99
|
+
/// mirroring `Dictionary.load`.
|
|
100
|
+
fn lid_model_load(
|
|
101
|
+
ruby: &Ruby,
|
|
102
|
+
_class: RClass,
|
|
103
|
+
onnx_path: String,
|
|
104
|
+
vocab_path: String,
|
|
105
|
+
) -> Result<Obj<RubyLidModel>, Error> {
|
|
106
|
+
let fail = |message: String| {
|
|
107
|
+
Error::new(
|
|
108
|
+
error_class(ruby),
|
|
109
|
+
format!("failed to load LID model ({onnx_path}, {vocab_path}): {message}"),
|
|
110
|
+
)
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
let onnx_bytes =
|
|
114
|
+
std::fs::read(Path::new(&onnx_path)).map_err(|error| fail(error.to_string()))?;
|
|
115
|
+
let vocab_bytes =
|
|
116
|
+
std::fs::read(Path::new(&vocab_path)).map_err(|error| fail(error.to_string()))?;
|
|
117
|
+
let model = kotoshu::lid::LidModel::parse(&onnx_bytes, &vocab_bytes)
|
|
118
|
+
.map_err(|error| fail(error.to_string()))?;
|
|
119
|
+
Ok(ruby.obj_wrap(RubyLidModel { inner: model }))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/// `Kotoshu::Native::LidModel#detect(text)` — the top label and its
|
|
123
|
+
/// probability as `{ "code" => String, "score" => Float }`, the gem's
|
|
124
|
+
/// `LanguageIdentifier` detection pair (labels equal and scores within
|
|
125
|
+
/// 5e-4 of the fastText bindings; frozen by the kotoshu-rs parity
|
|
126
|
+
/// suite over the shared corpus fixture).
|
|
127
|
+
fn lid_model_detect(ruby: &Ruby, rb_self: &RubyLidModel, text: String) -> Result<RHash, Error> {
|
|
128
|
+
let detection = rb_self.inner.detect(&text);
|
|
129
|
+
let hash = ruby.hash_new();
|
|
130
|
+
hash.aset("code", detection.code.as_str())?;
|
|
131
|
+
hash.aset("score", detection.score)?;
|
|
132
|
+
Ok(hash)
|
|
133
|
+
}
|
|
11
134
|
|
|
12
135
|
#[magnus::init]
|
|
13
136
|
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
14
|
-
kotoshu::ffi::ruby::init(ruby)
|
|
137
|
+
kotoshu::ffi::ruby::init(ruby)?;
|
|
138
|
+
|
|
139
|
+
let class = RubyLidModel::class(ruby);
|
|
140
|
+
class.define_singleton_method("load", method!(lid_model_load, 2))?;
|
|
141
|
+
class.define_method("detect", method!(lid_model_detect, 1))?;
|
|
142
|
+
Ok(())
|
|
15
143
|
}
|
|
@@ -72,6 +72,22 @@ module Kotoshu
|
|
|
72
72
|
# output is stable across machines and runs.
|
|
73
73
|
TIER_PREFERENCE = %i[mini fluency full].freeze
|
|
74
74
|
|
|
75
|
+
# The language-identification artifact pair (plan 106): registry
|
|
76
|
+
# resource `kotoshu://models/lid/lid-176` — the ONNX-converted
|
|
77
|
+
# lid.176 model (int8-per-row, dim 16) plus its vocab.json
|
|
78
|
+
# sidecar. Lives in the same registry as the embedding tiers but
|
|
79
|
+
# is NOT a language tier: it has no language of its own and the
|
|
80
|
+
# tier machinery (resource ids, TIERS) does not apply. It is
|
|
81
|
+
# stored under a top-level `lid/` directory (not
|
|
82
|
+
# `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
|
|
83
|
+
# reporting per-language ids only.
|
|
84
|
+
LID_LANGUAGE = "lid"
|
|
85
|
+
LID_TIER = "lid-176"
|
|
86
|
+
LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
|
|
87
|
+
|
|
88
|
+
# Directory (under the cache root) holding the LID artifact pair.
|
|
89
|
+
LID_DIRECTORY = "lid"
|
|
90
|
+
|
|
75
91
|
# Normalize and validate a tier name.
|
|
76
92
|
#
|
|
77
93
|
# @param tier [String, Symbol] "full", "fluency", or "mini"
|
|
@@ -248,6 +264,96 @@ module Kotoshu
|
|
|
248
264
|
result
|
|
249
265
|
end
|
|
250
266
|
|
|
267
|
+
# ---- LID artifact pair (plan 106) ----
|
|
268
|
+
|
|
269
|
+
# Whether the LID artifact pair is cached and complete.
|
|
270
|
+
#
|
|
271
|
+
# @return [Boolean]
|
|
272
|
+
def lid_cached?
|
|
273
|
+
!load_cached_lid.nil?
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Resolve the LID artifact pair from the cache only — never
|
|
277
|
+
# touches the network (the resolve half of the two-stage model).
|
|
278
|
+
#
|
|
279
|
+
# @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
|
|
280
|
+
# cached, nil when not set up
|
|
281
|
+
# @raise [Kotoshu::IntegrityError] cached bytes fail their
|
|
282
|
+
# recorded checksum
|
|
283
|
+
def load_cached_lid
|
|
284
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
285
|
+
metadata_path = File.join(dir, "metadata.json")
|
|
286
|
+
return nil unless File.exist?(metadata_path)
|
|
287
|
+
|
|
288
|
+
metadata = read_metadata(metadata_path)
|
|
289
|
+
return nil unless metadata
|
|
290
|
+
|
|
291
|
+
onnx_file = File.join(dir, metadata["file"].to_s)
|
|
292
|
+
vocab_file = File.join(dir, metadata["vocab_file"].to_s)
|
|
293
|
+
return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
294
|
+
return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
|
|
295
|
+
|
|
296
|
+
verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
297
|
+
|
|
298
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Download the LID artifact pair from the models registry
|
|
302
|
+
# (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
|
|
303
|
+
# against the registry sha256 before accepting them (the setup
|
|
304
|
+
# half of the two-stage model; same contract as
|
|
305
|
+
# {#download_tiered_model}).
|
|
306
|
+
#
|
|
307
|
+
# @param force [Boolean] re-fetch the registry first
|
|
308
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
309
|
+
# @raise [Kotoshu::Error] no registry entry, or offline without
|
|
310
|
+
# a cached registry (the registry is fetched once per cache
|
|
311
|
+
# instance; see {#registry})
|
|
312
|
+
# @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
|
|
313
|
+
# bytes are removed before raising)
|
|
314
|
+
def download_lid(force: false)
|
|
315
|
+
entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
|
|
316
|
+
unless entry
|
|
317
|
+
raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
321
|
+
FileUtils.mkdir_p(dir)
|
|
322
|
+
|
|
323
|
+
onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
|
|
324
|
+
used_url = download_primary_or_mirror!(entry, onnx_file)
|
|
325
|
+
verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
|
|
326
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
|
|
327
|
+
|
|
328
|
+
# The vocab sidecar is required, not optional: the reader
|
|
329
|
+
# cannot score without its labels and Huffman counts.
|
|
330
|
+
vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
|
|
331
|
+
begin
|
|
332
|
+
download_file(entry.vocab_url, vocab_file)
|
|
333
|
+
rescue StandardError => e
|
|
334
|
+
File.delete(onnx_file) if File.exist?(onnx_file)
|
|
335
|
+
raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
metadata = {
|
|
339
|
+
version: entry.version,
|
|
340
|
+
url: used_url,
|
|
341
|
+
language: LID_LANGUAGE,
|
|
342
|
+
type: "lid",
|
|
343
|
+
tier: LID_TIER,
|
|
344
|
+
file: File.basename(onnx_file),
|
|
345
|
+
vocab_file: File.basename(vocab_file),
|
|
346
|
+
checksum: entry.sha256,
|
|
347
|
+
registry_id: LID_REGISTRY_ID,
|
|
348
|
+
size_bytes: entry.size_bytes,
|
|
349
|
+
cached_at: Time.now.utc.iso8601,
|
|
350
|
+
source: "registry"
|
|
351
|
+
}
|
|
352
|
+
write_metadata(File.join(dir, "metadata.json"), metadata)
|
|
353
|
+
|
|
354
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
355
|
+
end
|
|
356
|
+
|
|
251
357
|
# Get available model types for a language.
|
|
252
358
|
#
|
|
253
359
|
# @param language_code [String] ISO 639-1 language code
|
|
@@ -629,7 +735,9 @@ module Kotoshu
|
|
|
629
735
|
# Verify downloaded model bytes against the registry's sha256.
|
|
630
736
|
# Corrupt bytes are removed from disk before raising so the next
|
|
631
737
|
# attempt re-downloads (same contract as BaseCache#verify_and_audit).
|
|
632
|
-
|
|
738
|
+
# +remediation+ overrides the default re-download hint (the LID
|
|
739
|
+
# pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
|
|
740
|
+
def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
|
|
633
741
|
actual = Digest::SHA256.file(model_file).hexdigest
|
|
634
742
|
if actual == entry.sha256
|
|
635
743
|
@audit_log.record(
|
|
@@ -649,7 +757,8 @@ module Kotoshu
|
|
|
649
757
|
expected: entry.sha256,
|
|
650
758
|
actual: actual,
|
|
651
759
|
url: url,
|
|
652
|
-
remediation:
|
|
760
|
+
remediation: remediation ||
|
|
761
|
+
"Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
|
|
653
762
|
)
|
|
654
763
|
end
|
|
655
764
|
|
|
@@ -680,6 +789,45 @@ module Kotoshu
|
|
|
680
789
|
# Caches written without a checksum field are accepted silently
|
|
681
790
|
# to preserve backward compatibility with pre-verification caches.
|
|
682
791
|
#
|
|
792
|
+
# Verify the cached LID pair against the checksum recorded at
|
|
793
|
+
# download time (which, for registry downloads, is the registry's
|
|
794
|
+
# sha256). A Git LFS pointer stub is refused by content first —
|
|
795
|
+
# same contract as {#verify_cached_integrity!}, against the
|
|
796
|
+
# fixed LID paths instead of a resource-id-derived layout.
|
|
797
|
+
#
|
|
798
|
+
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
799
|
+
# @param onnx_file [String] Path to the cached lid.176.onnx
|
|
800
|
+
# @param vocab_file [String] Path to the cached vocab sidecar
|
|
801
|
+
# @return [void]
|
|
802
|
+
# @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
|
|
803
|
+
# corrupt bytes are removed so the next setup re-downloads
|
|
804
|
+
def verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
805
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
806
|
+
if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
|
|
807
|
+
File.delete(onnx_file)
|
|
808
|
+
File.delete(vocab_file)
|
|
809
|
+
FileUtils.rm_f(File.join(dir, "metadata.json"))
|
|
810
|
+
raise Kotoshu::Error,
|
|
811
|
+
"Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
|
|
812
|
+
"artifact is a Git LFS pointer stub, not model content. Run " \
|
|
813
|
+
"Kotoshu.setup_lid(force: true) to re-download."
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
expected = metadata["checksum"]
|
|
817
|
+
return unless expected
|
|
818
|
+
|
|
819
|
+
actual = Digest::SHA256.file(onnx_file).hexdigest
|
|
820
|
+
return if actual == expected
|
|
821
|
+
|
|
822
|
+
raise Kotoshu::IntegrityError.new(
|
|
823
|
+
LID_REGISTRY_ID,
|
|
824
|
+
expected: expected,
|
|
825
|
+
actual: actual,
|
|
826
|
+
url: metadata["url"],
|
|
827
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
|
|
828
|
+
)
|
|
829
|
+
end
|
|
830
|
+
|
|
683
831
|
# @param resource_id [String] The resource identifier (e.g. "en:onnx")
|
|
684
832
|
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
685
833
|
# @param model_file [String] Path to the cached model file
|
data/lib/kotoshu/cli.rb
CHANGED
|
@@ -162,6 +162,10 @@ module Kotoshu
|
|
|
162
162
|
method_option :exclude,
|
|
163
163
|
type: :array,
|
|
164
164
|
desc: "Skip files matching these globs (always wins over --include and the extension default)"
|
|
165
|
+
method_option :personal,
|
|
166
|
+
type: :boolean,
|
|
167
|
+
default: true,
|
|
168
|
+
desc: "Accept words from the personal dictionary (personal.dic); --no-personal flags them again"
|
|
165
169
|
# Check spelling in files, directories, or stdin. Cache-only —
|
|
166
170
|
# never downloads; run `kotoshu setup LANG` first. See the
|
|
167
171
|
# command long description (kotoshu help check) for flags,
|
|
@@ -395,6 +399,10 @@ module Kotoshu
|
|
|
395
399
|
Kotoshu::Configuration.reset
|
|
396
400
|
cfg = Kotoshu::Configuration.instance
|
|
397
401
|
cfg.default_language = options[:language] if options[:language] && options[:language] != "auto"
|
|
402
|
+
# `kotoshu check --no-personal` opts out of the personal
|
|
403
|
+
# dictionary for this run (plan 105). Other commands do not
|
|
404
|
+
# declare the flag, so options[:personal] is nil there.
|
|
405
|
+
cfg.personal_dictionary = options[:personal] unless options[:personal].nil?
|
|
398
406
|
end
|
|
399
407
|
|
|
400
408
|
# Install a ProgressReporter on Configuration.download_reporter
|
|
@@ -209,6 +209,12 @@ module Kotoshu
|
|
|
209
209
|
default: "ruby",
|
|
210
210
|
description: "Engine backend for correct?/suggest (ruby, native, or auto)",
|
|
211
211
|
type: String
|
|
212
|
+
},
|
|
213
|
+
personal_dictionary: {
|
|
214
|
+
env: "KOTOSHU_PERSONAL_DICTIONARY",
|
|
215
|
+
default: true,
|
|
216
|
+
description: "Words in the personal dictionary (personal.dic) pass the check path",
|
|
217
|
+
type: :boolean
|
|
212
218
|
}
|
|
213
219
|
}.freeze
|
|
214
220
|
|
|
@@ -229,7 +235,8 @@ module Kotoshu
|
|
|
229
235
|
default_language: "en",
|
|
230
236
|
resource_pin: "main",
|
|
231
237
|
semantic_cascade_threshold: 1.0,
|
|
232
|
-
backend: "ruby"
|
|
238
|
+
backend: "ruby",
|
|
239
|
+
personal_dictionary: true
|
|
233
240
|
}.freeze
|
|
234
241
|
|
|
235
242
|
# @return [String, nil] Path to the dictionary file
|
|
@@ -289,6 +296,15 @@ module Kotoshu
|
|
|
289
296
|
# NativeBackend for the boundary — the semantic path stays Ruby.
|
|
290
297
|
attr_accessor :backend
|
|
291
298
|
|
|
299
|
+
# @return [Boolean] Whether Spellchecker#check accepts words from
|
|
300
|
+
# the user personal dictionary (~/.config/kotoshu/personal.dic,
|
|
301
|
+
# KOTOSHU_PERSONAL_DIC override) instead of flagging them as
|
|
302
|
+
# errors — the README story and the LSP diagnostics behavior
|
|
303
|
+
# (plan 105). Opt out with KOTOSHU_PERSONAL_DICTIONARY=false or
|
|
304
|
+
# `kotoshu check --no-personal`; the suggest path is unaffected,
|
|
305
|
+
# so personal words stay spellcheckable.
|
|
306
|
+
attr_accessor :personal_dictionary
|
|
307
|
+
|
|
292
308
|
# @return [String, nil] Path to cache directory
|
|
293
309
|
attr_accessor :cache_path
|
|
294
310
|
|
|
@@ -489,7 +505,8 @@ module Kotoshu
|
|
|
489
505
|
cache_ttl: @cache_ttl,
|
|
490
506
|
max_cache_size: @max_cache_size,
|
|
491
507
|
semantic_cascade_threshold: @semantic_cascade_threshold,
|
|
492
|
-
backend: @backend
|
|
508
|
+
backend: @backend,
|
|
509
|
+
personal_dictionary: @personal_dictionary
|
|
493
510
|
}
|
|
494
511
|
end
|
|
495
512
|
|
|
@@ -607,6 +624,7 @@ module Kotoshu
|
|
|
607
624
|
@resource_pin = DEFAULTS[:resource_pin]
|
|
608
625
|
@semantic_cascade_threshold = DEFAULTS[:semantic_cascade_threshold]
|
|
609
626
|
@backend = DEFAULTS[:backend]
|
|
627
|
+
@personal_dictionary = DEFAULTS[:personal_dictionary]
|
|
610
628
|
end
|
|
611
629
|
|
|
612
630
|
# Apply resolved values from the resolver (ENV, defaults).
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
# One language detection: the top label code (ISO 639) and its
|
|
6
|
+
# probability, the shape the plan-106 facade returns.
|
|
7
|
+
#
|
|
8
|
+
# A Struct (keyword_init) per the plain-data rule — no behavior
|
|
9
|
+
# beyond readers, #to_s (the code, so string interpolation reads
|
|
10
|
+
# naturally), and equality.
|
|
11
|
+
Detection = Struct.new(:code, :score, keyword_init: true) do
|
|
12
|
+
# The code, so `"detected #{detection}"` reads naturally.
|
|
13
|
+
def to_s
|
|
14
|
+
code.to_s
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Document language detection through the pure-Rust LID reader
|
|
19
|
+
# (plan 106).
|
|
20
|
+
#
|
|
21
|
+
# The gem's {LanguageIdentifier} needs the Python fastText
|
|
22
|
+
# bindings (broken under numpy 2), which is why /v1/detect and the
|
|
23
|
+
# CLI shipped a 7-language heuristic. The native extension now
|
|
24
|
+
# wraps kotoshu-rs' `lid` reader over the
|
|
25
|
+
# `kotoshu://models/lid/lid-176` registry artifact pair (the
|
|
26
|
+
# ONNX-converted lid.176 model + vocab sidecar, 176 languages,
|
|
27
|
+
# bit-faithful to the bindings — parity frozen by the kotoshu-rs
|
|
28
|
+
# suite over the shared corpus fixture).
|
|
29
|
+
#
|
|
30
|
+
# Selection mirrors {NativeBackend} with one difference: the
|
|
31
|
+
# native engine serves detect whenever the extension is built and
|
|
32
|
+
# the artifact pair is cached — an explicitly ruby backend
|
|
33
|
+
# (KOTOSHU_BACKEND=ruby) opts out, but the unset default does not,
|
|
34
|
+
# because the {Detector} heuristic is strictly weaker (7 languages
|
|
35
|
+
# vs 176). Pure-Ruby installs keep working unchanged — no new
|
|
36
|
+
# gemspec dependency, no onnxruntime.
|
|
37
|
+
#
|
|
38
|
+
# Two-stage like every resource: {setup} downloads the pair once
|
|
39
|
+
# (registry-resolved, sha-verified); {detect} resolves from the
|
|
40
|
+
# cache only and never downloads implicitly.
|
|
41
|
+
#
|
|
42
|
+
# @example Setup once, then detect
|
|
43
|
+
# Kotoshu.setup_lid
|
|
44
|
+
# Kotoshu.detect_language("Der Ausschuss prüft den Vorschlag")
|
|
45
|
+
# # => #<struct Kotoshu::Language::Detection code="de", score=0.99>
|
|
46
|
+
class LidDetector
|
|
47
|
+
class << self
|
|
48
|
+
# Detect the language of text.
|
|
49
|
+
#
|
|
50
|
+
# Native LID when available (see {LidDetector} for the
|
|
51
|
+
# selection rules), the {Detector} heuristic otherwise.
|
|
52
|
+
#
|
|
53
|
+
# @param text [String] Text to analyze
|
|
54
|
+
# @param config [Configuration] configuration to consult
|
|
55
|
+
# (defaults to the global instance)
|
|
56
|
+
# @return [Detection] code (nil when the heuristic is
|
|
57
|
+
# uncertain) and score in [0, 1]
|
|
58
|
+
def detect(text, config: Kotoshu.configuration)
|
|
59
|
+
# nil is not text; an empty/blank string IS (the native
|
|
60
|
+
# reader scores its EOS token row — "en", ~0.12 — exactly
|
|
61
|
+
# like the fastText bindings, per the frozen corpus).
|
|
62
|
+
return Detection.new(code: nil, score: 0.0) if text.nil?
|
|
63
|
+
|
|
64
|
+
model = native_model(config: config)
|
|
65
|
+
return heuristic_detection(text) unless model
|
|
66
|
+
|
|
67
|
+
row = model.detect(text)
|
|
68
|
+
Detection.new(code: row.fetch("code"), score: row.fetch("score"))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Whether {detect} would run the native engine for this
|
|
72
|
+
# configuration — the extension is built, the backend is not
|
|
73
|
+
# "ruby", and the artifact pair is cached.
|
|
74
|
+
#
|
|
75
|
+
# @param config [Configuration]
|
|
76
|
+
# @return [Boolean]
|
|
77
|
+
def available?(config: Kotoshu.configuration)
|
|
78
|
+
!native_model(config: config).nil?
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Download the LID artifact pair into the model cache (stage
|
|
82
|
+
# one of the two-stage model; registry-resolved, sha-verified).
|
|
83
|
+
#
|
|
84
|
+
# @param config [Configuration]
|
|
85
|
+
# @param force [Boolean] re-fetch even if already cached
|
|
86
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
87
|
+
# @raise [Kotoshu::Error] no registry entry, or offline with
|
|
88
|
+
# no cached registry (KOTOSHU_OFFLINE=1)
|
|
89
|
+
# @raise [Kotoshu::IntegrityError] downloaded bytes fail the
|
|
90
|
+
# registry checksum
|
|
91
|
+
def setup(config: Kotoshu.configuration, force: false)
|
|
92
|
+
pair = model_cache(config: config).load_cached_lid
|
|
93
|
+
return pair if pair && !force
|
|
94
|
+
|
|
95
|
+
model_cache(config: config).download_lid(force: force)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Whether the LID artifact pair is cached (cache-only check).
|
|
99
|
+
#
|
|
100
|
+
# @param config [Configuration]
|
|
101
|
+
# @return [Boolean]
|
|
102
|
+
def setup?(config: Kotoshu.configuration)
|
|
103
|
+
model_cache(config: config).lid_cached?
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Drop the memoized native model. The next {detect} re-resolves
|
|
107
|
+
# from the cache — the seam tests and embedders use after
|
|
108
|
+
# {setup} or a cache purge (mirrors Kotoshu.reset_spellchecker).
|
|
109
|
+
#
|
|
110
|
+
# @return [nil]
|
|
111
|
+
def reset
|
|
112
|
+
@native_model = nil
|
|
113
|
+
nil
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# The loaded native LID model, memoized per process — one
|
|
119
|
+
# parse of the ~3 MB pair, mirroring how spellcheckers are
|
|
120
|
+
# cached per language. Nil whenever the native engine cannot
|
|
121
|
+
# serve: extension absent, Kotoshu::Native::LidModel missing
|
|
122
|
+
# (extension predates plan 106), an explicit ruby backend, or
|
|
123
|
+
# the pair not set up.
|
|
124
|
+
#
|
|
125
|
+
# @param config [Configuration]
|
|
126
|
+
# @return [Kotoshu::Native::LidModel, nil]
|
|
127
|
+
def native_model(config:)
|
|
128
|
+
return nil unless native_lid_surface?
|
|
129
|
+
return nil if backend_opts_out?(config)
|
|
130
|
+
|
|
131
|
+
@native_model ||= begin
|
|
132
|
+
pair = model_cache(config: config).load_cached_lid
|
|
133
|
+
pair && Kotoshu::Native::LidModel.load(pair[:onnx_path], pair[:vocab_path])
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Whether the extension carries the LID surface at all.
|
|
138
|
+
def native_lid_surface?
|
|
139
|
+
Kotoshu::Native.available? && Kotoshu::Native.const_defined?(:LidModel)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Backend selection for LID (plan 106).
|
|
143
|
+
#
|
|
144
|
+
# KOTOSHU_BACKEND=ruby opts the LID out of the native engine.
|
|
145
|
+
# When the backend is UNSET, LID still prefers the native
|
|
146
|
+
# engine — unlike correct?/suggest, where "ruby" is the engine
|
|
147
|
+
# default and "native" the explicit accelerator. The heuristic
|
|
148
|
+
# is strictly weaker than the 176-language model, so it stays
|
|
149
|
+
# the fallback for the cannot-serve cases (no extension, no
|
|
150
|
+
# cached model), not for an unset preference.
|
|
151
|
+
#
|
|
152
|
+
# Explicitness is read through the resolver layers (live ENV,
|
|
153
|
+
# constructor settings, CLI options) so the "ruby" DEFAULT
|
|
154
|
+
# never counts as an opt-out. Mutating
|
|
155
|
+
# Kotoshu.configuration.backend after construction is not
|
|
156
|
+
# visible here — rebuild the configuration instead.
|
|
157
|
+
def backend_opts_out?(config)
|
|
158
|
+
config.backend.to_s == "ruby" && backend_set?(config)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Whether the backend was explicitly set at any resolver layer
|
|
162
|
+
# (rather than left at its "ruby" default).
|
|
163
|
+
def backend_set?(config)
|
|
164
|
+
resolver = config.resolver
|
|
165
|
+
ENV.key?("KOTOSHU_BACKEND") ||
|
|
166
|
+
resolver.programmatic.key?(:backend) ||
|
|
167
|
+
resolver.cli.key?(:backend)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# The heuristic fallback, wrapped in the same {Detection}
|
|
171
|
+
# shape the native path returns.
|
|
172
|
+
def heuristic_detection(text)
|
|
173
|
+
code, confidence = Detector.detect_with_confidence(text)
|
|
174
|
+
Detection.new(code: code, score: confidence)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# A cache over the configuration's model cache path and
|
|
178
|
+
# source registry (so KOTOSHU_MODELS_PIN is honored).
|
|
179
|
+
def model_cache(config:)
|
|
180
|
+
Cache::ModelCache.new(
|
|
181
|
+
cache_path: config.cache_path,
|
|
182
|
+
source_registry: config.source_registry
|
|
183
|
+
)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
data/lib/kotoshu/language.rb
CHANGED
|
@@ -17,7 +17,11 @@ module Kotoshu
|
|
|
17
17
|
module Language
|
|
18
18
|
autoload :Registry, "kotoshu/language/registry"
|
|
19
19
|
autoload :Detector, "kotoshu/language/detector"
|
|
20
|
+
# Detection value + the native LID reader behind it (plan 106) share
|
|
21
|
+
# lid_detector.rb.
|
|
22
|
+
autoload :Detection, "kotoshu/language/lid_detector"
|
|
20
23
|
autoload :LanguageIdentifier, "kotoshu/language/identifier"
|
|
24
|
+
autoload :LidDetector, "kotoshu/language/lid_detector"
|
|
21
25
|
autoload :Segmenter, "kotoshu/language/segmenter"
|
|
22
26
|
autoload :Segment, "kotoshu/language/segmenter"
|
|
23
27
|
autoload :Suika, "kotoshu/language/suika"
|
data/lib/kotoshu/spellchecker.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
3
5
|
module Kotoshu
|
|
4
6
|
# Main spellchecker class.
|
|
5
7
|
#
|
|
@@ -175,6 +177,15 @@ module Kotoshu
|
|
|
175
177
|
|
|
176
178
|
# Check text for spelling errors.
|
|
177
179
|
#
|
|
180
|
+
# Words present in the user's personal dictionary
|
|
181
|
+
# (~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override)
|
|
182
|
+
# never surface as errors — the same semantics the LSP applies to
|
|
183
|
+
# its diagnostics (plan 105). They are dropped at result assembly
|
|
184
|
+
# without suppression metadata, still count toward +word_count+,
|
|
185
|
+
# and remain spellcheckable through {#suggest}. Opt out with
|
|
186
|
+
# {Configuration#personal_dictionary} (KOTOSHU_PERSONAL_DICTIONARY,
|
|
187
|
+
# `kotoshu check --no-personal`).
|
|
188
|
+
#
|
|
178
189
|
# @param text [String] The text to check
|
|
179
190
|
# @return [Models::Result::DocumentResult] The check result
|
|
180
191
|
#
|
|
@@ -191,9 +202,12 @@ module Kotoshu
|
|
|
191
202
|
# because the facade sees raw text without a format parser.
|
|
192
203
|
words = tokenize(text)
|
|
193
204
|
suppressions = Documents::Suppressions.scan(text, format: :auto)
|
|
205
|
+
personal = personal_words
|
|
194
206
|
errors = []
|
|
195
207
|
suppressed_errors = []
|
|
196
208
|
words.each do |word, pos|
|
|
209
|
+
next if personal.include?(word.downcase)
|
|
210
|
+
|
|
197
211
|
result = check_word(word)
|
|
198
212
|
next if result.correct?
|
|
199
213
|
|
|
@@ -318,6 +332,27 @@ module Kotoshu
|
|
|
318
332
|
|
|
319
333
|
private
|
|
320
334
|
|
|
335
|
+
# The personal dictionary as a downcased Set for the check path
|
|
336
|
+
# (plan 105), mirroring the LSP's filter. Loaded once per process:
|
|
337
|
+
# the first {#check} call snapshots the words, and later edits to
|
|
338
|
+
# personal.dic take effect in the next process (the LSP reloads on
|
|
339
|
+
# mtime change; this path deliberately does not — one-shot CLI runs
|
|
340
|
+
# always read fresh, and long-lived embedders rebuild the
|
|
341
|
+
# spellchecker via {Kotoshu.reset_spellchecker}). An empty set when
|
|
342
|
+
# disabled through {Configuration#personal_dictionary}.
|
|
343
|
+
#
|
|
344
|
+
# @return [Set<String>]
|
|
345
|
+
def personal_words
|
|
346
|
+
return @personal_words if instance_variable_defined?(:@personal_words)
|
|
347
|
+
|
|
348
|
+
@personal_words =
|
|
349
|
+
if @config.personal_dictionary
|
|
350
|
+
PersonalDictionary.words.map(&:downcase).to_set
|
|
351
|
+
else
|
|
352
|
+
Set.new
|
|
353
|
+
end
|
|
354
|
+
end
|
|
355
|
+
|
|
321
356
|
# Resolve the tokenizer of the language being checked for word
|
|
322
357
|
# extraction (plan 91).
|
|
323
358
|
#
|
data/lib/kotoshu/version.rb
CHANGED
data/lib/kotoshu.rb
CHANGED
|
@@ -446,18 +446,31 @@ module Kotoshu
|
|
|
446
446
|
|
|
447
447
|
# Detect language of text.
|
|
448
448
|
#
|
|
449
|
+
# Native LID when available (plan 106): the Rust extension scoring
|
|
450
|
+
# the `kotoshu://models/lid/lid-176` artifact pair over 176
|
|
451
|
+
# languages, after a one-time {setup_lid}. Falls back to the
|
|
452
|
+
# {Language::Detector} heuristic when the extension is absent,
|
|
453
|
+
# KOTOSHU_BACKEND=ruby, or the model is not set up — so pure-Ruby
|
|
454
|
+
# installs keep the previous behavior.
|
|
455
|
+
#
|
|
449
456
|
# @param text [String] Text to analyze
|
|
450
|
-
# @return [String, nil
|
|
457
|
+
# @return [Language::Detection] +code+ (String, or nil when the
|
|
458
|
+
# heuristic is uncertain) and +score+ (Float in [0, 1]);
|
|
459
|
+
# interpolation yields the code
|
|
451
460
|
#
|
|
452
461
|
# @example
|
|
453
|
-
# Kotoshu.
|
|
454
|
-
# Kotoshu.detect_language("
|
|
462
|
+
# Kotoshu.setup_lid
|
|
463
|
+
# Kotoshu.detect_language("Bonjour le monde").code # => "fr"
|
|
464
|
+
# Kotoshu.detect_language("こんにちは").score # => 0.99
|
|
455
465
|
def self.detect_language(text)
|
|
456
|
-
Language.detect(text)
|
|
466
|
+
Language::LidDetector.detect(text)
|
|
457
467
|
end
|
|
458
468
|
|
|
459
469
|
# Detect language with confidence score.
|
|
460
470
|
#
|
|
471
|
+
# Same engine selection as {detect_language}, returned as the
|
|
472
|
+
# historical [code, confidence] pair.
|
|
473
|
+
#
|
|
461
474
|
# @param text [String] Text to analyze
|
|
462
475
|
# @return [Array<String, Float>] Language code and confidence
|
|
463
476
|
#
|
|
@@ -466,7 +479,34 @@ module Kotoshu
|
|
|
466
479
|
# lang # => "en"
|
|
467
480
|
# conf # => 0.85
|
|
468
481
|
def self.detect_language_with_confidence(text)
|
|
469
|
-
Language.
|
|
482
|
+
detection = Language::LidDetector.detect(text)
|
|
483
|
+
[detection.code, detection.score]
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
# Set up the language-identification model: downloads the
|
|
487
|
+
# lid.176.onnx + vocab.json pair from the models registry into the
|
|
488
|
+
# model cache (stage one of the two-stage model — {detect_language}
|
|
489
|
+
# never downloads implicitly). Idempotent; +force+ re-downloads.
|
|
490
|
+
#
|
|
491
|
+
# @param force [Boolean] Re-fetch even if already cached
|
|
492
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
493
|
+
# @raise [Kotoshu::Error] registry unreachable while offline with no
|
|
494
|
+
# cached registry (KOTOSHU_OFFLINE=1), or no registry entry
|
|
495
|
+
# @raise [Kotoshu::IntegrityError] downloaded bytes fail the
|
|
496
|
+
# registry checksum
|
|
497
|
+
#
|
|
498
|
+
# @example
|
|
499
|
+
# Kotoshu.setup_lid
|
|
500
|
+
# Kotoshu.setup_lid? # => true
|
|
501
|
+
def self.setup_lid(force: false)
|
|
502
|
+
Language::LidDetector.setup(config: configuration, force: force)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# Whether the language-identification model is cached (cache-only).
|
|
506
|
+
#
|
|
507
|
+
# @return [Boolean]
|
|
508
|
+
def self.setup_lid?
|
|
509
|
+
Language::LidDetector.setup?(config: configuration)
|
|
470
510
|
end
|
|
471
511
|
|
|
472
512
|
# Get language class by code.
|
data/sig/kotoshu.rbs
CHANGED
|
@@ -106,6 +106,27 @@ def spellcheck_word_regex: () -> Regexp
|
|
|
106
106
|
def normalize: (String? text, ?options ::Hash[Symbol, untyped]) -> String
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
|
+
|
|
110
|
+
# One language detection: top label code and its probability
|
|
111
|
+
# (plan 106). Plain-data Struct.
|
|
112
|
+
class Detection
|
|
113
|
+
attr_reader code: String?
|
|
114
|
+
attr_reader score: Float
|
|
115
|
+
|
|
116
|
+
def initialize: (?code: String?, ?score: Float) -> void
|
|
117
|
+
def to_s: () -> String
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Document language detection through the pure-Rust LID reader
|
|
121
|
+
# (plan 106): kotoshu://models/lid/lid-176 scored by the native
|
|
122
|
+
# extension, Language::Detector heuristic as the fallback.
|
|
123
|
+
class LidDetector
|
|
124
|
+
def self.detect: (String? text, ?config: Configuration) -> Detection
|
|
125
|
+
def self.available?: (?config: Configuration) -> bool
|
|
126
|
+
def self.setup: (?config: Configuration, ?force: bool) -> Hash[Symbol, untyped]
|
|
127
|
+
def self.setup?: (?config: Configuration) -> bool
|
|
128
|
+
def self.reset: () -> nil
|
|
129
|
+
end
|
|
109
130
|
end
|
|
110
131
|
|
|
111
132
|
module Languages
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kotoshu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.10.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -108,6 +108,8 @@ files:
|
|
|
108
108
|
- TODO.impl/102-lid-chain.md
|
|
109
109
|
- TODO.impl/103-oov-bucket-tables.md
|
|
110
110
|
- TODO.impl/104-vscode-marketplace-readiness.md
|
|
111
|
+
- TODO.impl/105-personal-dictionary-in-check.md
|
|
112
|
+
- TODO.impl/106-native-lid-detection.md
|
|
111
113
|
- TODO.impl/11-release-v1.md
|
|
112
114
|
- TODO.impl/20-encapsulation-cleanup.md
|
|
113
115
|
- TODO.impl/21-autoload-migration.md
|
|
@@ -356,6 +358,7 @@ files:
|
|
|
356
358
|
- lib/kotoshu/language/detector.rb
|
|
357
359
|
- lib/kotoshu/language/identifier.rb
|
|
358
360
|
- lib/kotoshu/language/languages/base.rb
|
|
361
|
+
- lib/kotoshu/language/lid_detector.rb
|
|
359
362
|
- lib/kotoshu/language/normalizer/arabic.rb
|
|
360
363
|
- lib/kotoshu/language/normalizer/base.rb
|
|
361
364
|
- lib/kotoshu/language/normalizer/greek.rb
|