kotoshu 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +24 -0
  3. data/TODO.impl/00-vision.md +22 -0
  4. data/TODO.impl/100-full-feature-batch3.md +28 -0
  5. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  6. data/TODO.impl/102-lid-chain.md +28 -0
  7. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  8. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  9. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  10. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  11. data/lib/kotoshu/algorithms.rb +1 -0
  12. data/lib/kotoshu/cache/language_cache.rb +4 -1
  13. data/lib/kotoshu/dictionary/base.rb +32 -5
  14. data/lib/kotoshu/dictionary/custom.rb +3 -0
  15. data/lib/kotoshu/dictionary/hunspell.rb +5 -2
  16. data/lib/kotoshu/dictionary/plain_text.rb +2 -0
  17. data/lib/kotoshu/dictionary/unix_words.rb +2 -0
  18. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  19. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  20. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  21. data/lib/kotoshu/keyboard/layouts/hrsl.rb +78 -0
  22. data/lib/kotoshu/keyboard/layouts/latin.rb +47 -1
  23. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  24. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  25. data/lib/kotoshu/keyboard/registry.rb +20 -4
  26. data/lib/kotoshu/keyboard.rb +9 -0
  27. data/lib/kotoshu/languages/ar/language.rb +14 -0
  28. data/lib/kotoshu/languages/bg/language.rb +37 -0
  29. data/lib/kotoshu/languages/et/language.rb +24 -0
  30. data/lib/kotoshu/languages/fa/language.rb +16 -0
  31. data/lib/kotoshu/languages/he/language.rb +15 -0
  32. data/lib/kotoshu/languages/hr/language.rb +24 -0
  33. data/lib/kotoshu/languages/id/language.rb +25 -0
  34. data/lib/kotoshu/languages/lt/language.rb +24 -0
  35. data/lib/kotoshu/languages/lv/language.rb +24 -0
  36. data/lib/kotoshu/languages/sk/language.rb +25 -0
  37. data/lib/kotoshu/languages/sl/language.rb +25 -0
  38. data/lib/kotoshu/languages/sr/language.rb +38 -0
  39. data/lib/kotoshu/languages.rb +12 -0
  40. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +25 -6
  41. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +30 -57
  42. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  43. data/lib/kotoshu/suggestions.rb +1 -0
  44. data/lib/kotoshu/version.rb +1 -1
  45. data/scripts/extract_eval_grids.rb +4 -1
  46. data/sig/kotoshu.rbs +24 -0
  47. metadata +24 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c1082e88abf75329c969089181472e5a1e511bbb48b7f1c7ab4c113b5e6fc310
4
- data.tar.gz: 92cb046340c93ffdf72f7c9c69eb776a683757723020a96b3c4805416c43d2f7
3
+ metadata.gz: 4891dc67bd85b5c342fdbc7453085990a1e9ff9bea5b394a67f87e11b4bbd6c7
4
+ data.tar.gz: 378d1b77bea8113d155774bc07e481afaa9229d6e1edcd7d5fda2805d28a963c
5
5
  SHA512:
6
- metadata.gz: f5aee6be1e7c35c0bb8f3e8bcac0aa089e1caac0bd8cb0e05925c74f5e4a925fd5e0206ba22759421140e96fcfc5610596bd3bdfe14fde489b1117cc46969c15
7
- data.tar.gz: a62aa1ec6ddbd3cc499958ad45b07fd281c5f1b919b7692960a1b59602fd3bbc8352f2d769946e91518de29e7e2cc620c3e2cba818e19fcd4eb295a48af0ddd7
6
+ metadata.gz: bebc7b8be31a4f9f0127bddf2cb4a8de23c9908b05ddf91f51a43af5927af4608a0398090cce27deefce0e5c86f4a7d9edb0ffdf6b122f849bb065cfcee44e2d
7
+ data.tar.gz: 1240bb5e7d513c4d3b3dd936042c3d177cbad6aebc402fc6ef91f86e3163bc0ba2dcf516f93d7658171d2c952c1f2fed6162928c7a50bdef3e5703233acce1ce
data/CHANGELOG.md CHANGED
@@ -7,7 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ### Added
11
+ - **Full-feature language batch 3** (plan 100) - twelve languages gain the
12
+ downloadable spelling dictionary, a keyboard layout, and a gem module:
13
+ `ar` `id` `fa` `he` `bg` `sr` `hr` `sk` `sl` `lt` `lv` `et` (the RTL
14
+ ar/fa/he modules are promoted from tokenizer-only wiring; the other nine
15
+ are new thin modules - id hr sl sk et lt lv on `LatinBase`, bg sr with
16
+ the Cyrillic tokenizer). Six national key grids join the layout registry
17
+ (Arabic 101, Persian standard ISIRI 9147, Hebrew SI-1452, Bulgarian BDS,
18
+ Serbian Cyrillic, and the shared Croatian/Slovenian QWERTZ), all mirrored
19
+ from the models repo eval harness and drift-checked against it; five new
20
+ Latin family members (Indonesian Slovak Estonian Lithuanian Latvian)
21
+ type on the US QWERTY grid like the eval model. Bulgarian and Serbian
22
+ register ahead of JCUKEN so their Cyrillic grids win the lookup. The
23
+ eval-grid drift fixture now covers all nine national grids. Specimen
24
+ misspelling -> correction pairs for every language are engine-verified
25
+ against the staged dictionaries in a `:network` spec.
26
+
10
27
  ### Fixed
28
+ - **RTL word extraction on the check path** - the plan-91 script-aware
29
+ word regex never reached the Arabic, Persian and Hebrew tokenizers, so
30
+ `Kotoshu.check` / `kotoshu check -l ar|fa|he` extracted zero words and
31
+ silently passed every RTL document. The three tokenizers now declare
32
+ their script sets (`\p{Arabic}`, `\p{Arabic}` + ZWNJ for Persian
33
+ compounds like می‌روم, `\p{Hebrew}` + geresh/gershayim), and RTL
34
+ misspellings surface with ranked suggestions.
11
35
  - **Suggestion candidate sweep missed transpositions, substitutions, and
12
36
  dictionary forms** - `EditDistanceStrategy` only scored raw dictionary
13
37
  stems, so valid dictionary forms could never be suggested no matter how
@@ -188,6 +188,28 @@ convertible model pool exhausted at 55 languages; owner-side
188
188
  remnants (PyPI registrations, marketplace PAT, nds/fi calls) recorded
189
189
  in the plans, not pending work.
190
190
 
191
+ Plans 99-104 are **wave 6, the live-quality follow-through**
192
+ (2026-09-07), written after the sweep-performance arc (gem 0.9.1 ->
193
+ 0.9.2, @kotoshu/wasm 0.3.0 -> 0.3.2) and a fresh state audit: 55 model
194
+ languages x 3 tiers confirmed (ja/ko/zh rerank-only by design), no LID
195
+ resource mirrored, lsp/server `~> 0.6` constraints already resolving
196
+ current engines, site docs carrying stale version references.
197
+
198
+ | # | Plan | Gap it closes |
199
+ |---|---|---|
200
+ | 99 | [Site truth + performance docs](99-site-truth-performance-docs.md) | Docs misstate current versions; the measured per-language latency is unpublished |
201
+ | 100 | [Full-feature batch 3](100-full-feature-batch3.md) | ~35 languages have dictionaries + models but no gem module - the big-audience remainder (ar id fa he ...) |
202
+ | 101 | [int4 tier experiment](101-int4-tier-experiment.md) | Halving tier bytes was named (B1) but never run through the gates |
203
+ | 102 | [LID chain](102-lid-chain.md) | No detection in the registry, wasm, or playground - users pick the language blind |
204
+ | 103 | [OOV bucket tables](103-oov-bucket-tables.md) | Model candidates miss n-grams absent from vocab (the Teh gap's model half) |
205
+ | 104 | [VS Code marketplace readiness](104-vscode-marketplace-readiness.md) | The extension is built but not installable; everything short of the owner PAT |
206
+
207
+ Execution notes: plan 103 is sequenced after 101 (same repo, one
208
+ worktree at a time); all site-facing flips (99, 100, 102's playground
209
+ half, 104's link line) land through one coordinated site pass to keep
210
+ the repo conflict-free; lsp/server 0.1.2 bumps are deliberately NOT
211
+ planned - the `~> 0.6` floor already resolves 0.9.2 for fresh installs.
212
+
191
213
  Dependency order across the ecosystem track:
192
214
 
193
215
  ```
@@ -0,0 +1,28 @@
1
+ # Plan 100 — Full-feature language batch 3: the highest-audience remainder
2
+
3
+ ## Why
4
+ 20 languages are full-feature (gem module + keyboard layout + verified
5
+ specimens); the dictionaries manifest and the model registry carry ~35 more
6
+ with both resources. Every remaining language without a module leaves
7
+ speakers on the degraded path. The gaps are the big-audience ones: ar (~400M),
8
+ id (~200M), fa (~80M), he, bg, sr, hr, sk, sl, and more.
9
+
10
+ ## Work (gem repo only — the site flip rides a follow-up)
11
+ 1. Enumerate: languages present in BOTH the dictionaries manifest and the
12
+ models registry that lack a gem module. Rank by speaker count.
13
+ 2. For the top ~12 (expected: ar id fa he bg sr hr sk sl lt lv et — confirm
14
+ against data): create the language module (pattern of the 19 from PR
15
+ #123), keyboard layout (Arabic 101 for ar; Persian for fa; Hebrew for he;
16
+ Cyrillic/YUSI for bg/sr; QWERTZ/Slovak for the Latinate), AVAILABLE_LANGUAGES
17
+ extension, and specimen pairs verified against the STAGED dictionary via
18
+ the real engine (correct()/suggest() round trips) — the PR #123 discipline.
19
+ 3. RTL already wired (ar/id from batch 2); no tokenizer work needed.
20
+ 4. Suite green; rubocop clean.
21
+
22
+ ## Verification
23
+ One spec per module (specimen corrections engine-verified, layout smoke);
24
+ `kotoshu setup <lang>` + `check -l <lang>` exercised live for at least ar,
25
+ id, fa, he. Deliver the verified specimen table + sizes for the site flip.
26
+
27
+ ## Status
28
+ Executed 2026-09-07 — gem PR #151 merged. Twelve languages (ar id fa he bg sr hr sk sl lt lv et) full-feature; AVAILABLE_LANGUAGES 20→32; 49 specimens; RTL tokenizer fix for ar/fa/he. Site flip PR #27.
@@ -0,0 +1,32 @@
1
+ # Plan 101 — int4 tier experiment (models repo), eval-gated
2
+
3
+ ## Why
4
+ The tier recipe settled at int8 (rank_corr 0.9999, near-lossless) after SVD
5
+ failed its gates. int4 halves the bytes again — mini ~1.5 MB, fluency ~8 MB —
6
+ which changes browser adoption math for the wasm playground. It was named in
7
+ the original kotoshu-rs TODO (B1) but never run through the gates.
8
+
9
+ ## Work (models repo)
10
+ 1. Quantization pass: per-row int4 (packed nibbles) + fp16 scale, over the
11
+ same vocab cuts as fluency/mini.
12
+ 2. The existing eval gates unchanged and unweakened: rank correlation vs
13
+ full float32 on the keyboard-aware eval; accept only if rank_corr within
14
+ 0.001 of int8's and top-1 agreement holds on the probe-limited languages.
15
+ 3. If gates pass: registry v1.4.0 adding the int4 variants as NEW resources
16
+ (`{lang}/mini4`? or a field) — additive, never replacing; mirror with
17
+ ACAO:* like v1.2.1+. If gates fail: record the numbers as the verdict
18
+ (the SVD precedent — measured, not assumed).
19
+ 4. ONNX reader side (kotoshu-rs) only if the gates pass.
20
+
21
+ ## Verification
22
+ Gates table per language; byte sizes; the release registry byte-identical to
23
+ the committed one; no existing tier URLs change.
24
+
25
+ ## Status
26
+ Executed 2026-09-07 — REJECTED on data (models PR #17, merged). int4-per-row
27
+ (one fp16 scale, packed nibbles) over 8 languages: rank_corr deltas -0.0146
28
+ to -0.0199 vs int8 (15-20x outside the 0.001 window), top-1 fails on 8/8
29
+ fluency4 and 2/8 mini4. Sizes halve (mini4 1.52 MB, fluency4 7.60 MB) but
30
+ accuracy does not hold. With the earlier group-128/64/32 sweep the scale
31
+ ladder is measured end to end — int8-per-row stands final. Registry
32
+ untouched; evidence in models eval/reports/.
@@ -0,0 +1,28 @@
1
+ # Plan 102 — Language detection end to end: registry mirror, wasm, playground
2
+
3
+ ## Why
4
+ The gem detects document language (server /v1/detect) using its own detection
5
+ model — but the registry mirrors none of it, the wasm surface has no detect
6
+ API, and the playground makes a user pick a language dropdown blind. A
7
+ writer pasting text should be told what it is. The FastText LID model is
8
+ tiny (~1 MB) and the mirror pattern is proven (v1.2.1+).
9
+
10
+ ## Work
11
+ 1. Models repo: add the LID model as a registry resource (converted like the
12
+ tiers, int8-quantized if gates hold vs the float32 labels), mirrored with
13
+ ACAO:*; registry v1.4.x alongside plan 101 (coordinate the version).
14
+ 2. kotoshu-rs: `detect_language(bytes, vocab_bytes) -> {code, score}` on the
15
+ model surface beside loadModel/rerank; wasm export `detectLanguage`;
16
+ tests against the gem's detection outputs on a fixed multilingual corpus.
17
+ 3. Playground (rides the site follow-up with plans 99/100 flips): on load
18
+ and on demand ("detect" affordance beside the lang select), load the LID
19
+ model (~1 MB, cached like the semantic tier) and set/select the language,
20
+ showing the detected code + confidence. Never auto-switch a user's
21
+ explicit choice silently — propose it.
22
+
23
+ ## Verification
24
+ Registry mirror ACAO + size checks; wasm parity vs gem detect on the corpus;
25
+ playground verified with multilingual paste (en/de/ru/ja/ar).
26
+
27
+ ## Status
28
+ Executed 2026-09-07 (core) — models PR #18 (lid-176, registry v1.4.0) + kotoshu-rs PR #25 (loadLid/detectLanguage). Playground propose-UI still pending a wasm cut that ships the export.
@@ -0,0 +1,27 @@
1
+ # Plan 103 — OOV bucket-table artifacts: model-generated candidates for the last mile
2
+
3
+ ## Why
4
+ semanticSuggest embeds an OOV word via subword n-grams PRESENT IN THE VOCAB;
5
+ a typo whose n-grams never appear (Teh — only n-gram "teh") embeds nothing
6
+ and returns empty. fastText's real OOV path hashes n-grams into buckets the
7
+ training wrote to; exporting the bucket table closes that gap so the model,
8
+ not just the Damerau sweep, can reach the intended word. Recorded as the
9
+ honest gap in kotoshu-rs oov.rs docs and PR #18's report.
10
+
11
+ ## Work (models repo, then kotoshu-rs)
12
+ 1. Export the bucket-table rows the FastText training produced (hash bucket
13
+ → vector) for each tier alongside the vocab; size audit first — the
14
+ bucket table may exceed the tier itself; if so, gate to the top-K buckets
15
+ by usage and measure the quality delta honestly.
16
+ 2. kotoshu-rs: extend the int8 reader to consult bucket rows for n-grams
17
+ absent from vocab (feature-gated like `model`); semantic_neighbors then
18
+ covers bucket-backed OOV.
19
+ 3. Gates: existing rerank conformance must not move (frozen); new specs for
20
+ bucket-backed neighbors (Teh→the expected); eval on the rerank probe set.
21
+
22
+ ## Verification
23
+ Bucket-backed neighbor specs; size deltas recorded; rerank conformance 0
24
+ change; a 0.4.x wasm release only after both engines agree.
25
+
26
+ ## Status
27
+ Executed 2026-09-07 — models PR #20 (en/de buckets) + kotoshu-rs PR #26 (bucket-backed OOV). Teh→the via buckets. Release assets + wasm 0.4.0 pin still owner/coordinated cut. (sequenced AFTER plan 101 — same repo, one worktree at a time)
@@ -0,0 +1,32 @@
1
+ # Plan 104 — VS Code marketplace + Open VSX readiness
2
+
3
+ ## Why
4
+ The extension is built, CI-green, with a .vsix artifact — but it is not
5
+ installable without the marketplace listing, which is the single largest
6
+ adoption blocker for the editor audience. Publishing needs the owner's
7
+ publisher PAT (a credential we never hold); everything else can be ready
8
+ so publishing is one secret + one click.
9
+
10
+ ## Work (vscode repo)
11
+ 1. Open VSX publish workflow (release-ovsx.yml, secret-gated) beside the
12
+ existing CI — Open VSX serves VSCodium/Code-OSS users without the
13
+ Microsoft marketplace.
14
+ 2. Metadata polish: gallery banner/colors, keywords, categories, repository
15
+ links, the kotoshu-vscode README (quickstart with kotoshu-lsp install,
16
+ settings reference: log level, personal dictionary command).
17
+ 3. PREPUBLISH.md: exact owner steps — create publisher, add PAT as secret,
18
+ run the workflow; nothing else.
19
+ 4. Site follow-up line (rides plan 99's pass): /docs/clients/lsp gains the
20
+ marketplace link placeholder to fill on publish day.
21
+
22
+ ## Verification
23
+ `vsce package` clean with no warnings; workflow lint-valid; PREPUBLISH
24
+ steps verified against the actual GitHub UI labels.
25
+
26
+ ## Status
27
+ Executed 2026-09-07 — vscode PR #3, merged. release-marketplace.yml (VSCE_PAT)
28
+ + release-ovsx.yml (OPEN_VSX_TOKEN), both version-gated; PREPUBLISH.md with
29
+ verified owner steps; package.json publisher/categories/keywords/banner;
30
+ README quickstart against gem 0.1.1; vsce package warning-clean. Publishing
31
+ itself is one PAT secret + one tag — owner action. The /docs/clients/lsp
32
+ marketplace link rides the coordinated site pass.
@@ -0,0 +1,31 @@
1
+ # Plan 99 — Site truth pass + measured performance documentation
2
+
3
+ ## Why
4
+ The docs were written against the versions live at the time; three releases
5
+ later some pages misstate reality. Users deserve the measured numbers the
6
+ perf work produced — no spellchecker site publishes per-language sweep
7
+ latency; we have it, verified, for eight languages.
8
+
9
+ ## Work (site repo)
10
+ 1. `/docs/performance` (new page): the measured tables — whole-text check
11
+ latency (1-33 ms per 100 words), per-sweep averages post-0.3.2
12
+ (en 45 ms ... pt 423 ms, fr worst-case note), tier sizes (full/fluency/mini),
13
+ and tuning: personal dictionaries, baselines, tier choice guidance
14
+ (`--tier`), backend choice (ruby vs native vs wasm). Link from the
15
+ playground footnote, /languages, and /install.
16
+ 2. Truth pass: /docs/clients/action still says the action "installs the gem
17
+ from RubyGems where 0.7.0 gates six languages" — stale; the action now
18
+ rides 0.9.2 (20 full-feature languages, baselines, directory mode).
19
+ Audit every page for version references (0.7.0/0.8.0/0.1.1/wasm 0.2.0)
20
+ and flip to current-with-dates. /playground/server: verify the hosted
21
+ demo state and the run-your-own instructions.
22
+ 3. Footer/readout: confirm every registry badge resolves (RubyGems 0.9.2,
23
+ npm 0.3.2, PyPI, crates.io).
24
+
25
+ ## Verification
26
+ Rendered text extraction on the built site for stale version strings;
27
+ every measured number in /docs/performance traceable to the 0.3.2 bench
28
+ harness (kotoshu-rs PR #23 table) — no inferred numbers.
29
+
30
+ ## Status
31
+ Executed 2026-09-07 — site PR #26: /docs/performance + truth pass. Action gap found and closed via action-kotoshu PR #2 + site PR #27 docs flip.
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Algorithms
5
+ # Soundex phonetic coding.
6
+ #
7
+ # Soundex is a phonetic algorithm developed by Robert C. Russell
8
+ # and Margaret King Odell in the early 1900s. A word maps to a
9
+ # four-character code (letter + 3 digits); words that sound alike
10
+ # share a code, which is what {Suggestions::Strategies::PhoneticStrategy}
11
+ # matches on.
12
+ #
13
+ # Coding rules (the gem's exact semantics — first letter verbatim,
14
+ # H/W never reset the previous code, non-ASCII stripped after
15
+ # upcase, letter-less words code to the empty string):
16
+ #
17
+ # - The first letter is kept verbatim and never coded.
18
+ # - B/P/F/V code 1, C/S/K/G/J/Q/X/Z code 2, D/T code 3, L codes 4,
19
+ # M/N code 5, R codes 6; vowels and H/W code 0 (skipped).
20
+ # H and W do NOT reset the previous code; every other coded 0
21
+ # letter does.
22
+ # - Adjacent letters sharing a code collapse to one digit.
23
+ # - The code is truncated to and zero-padded to exactly 4 chars.
24
+ #
25
+ # @example
26
+ # Soundex.code("Robert") # => "R163"
27
+ # Soundex.code("Rupert") # => "R163"
28
+ # Soundex.code("Ashcraft") # => "A261"
29
+ # Soundex.code("é") # => "" (no A-Z letters remain)
30
+ module Soundex
31
+ module_function
32
+
33
+ # Calculate the Soundex code for a word.
34
+ #
35
+ # @param word [String, nil] The word
36
+ # @return [String] The Soundex code (letter + 3 digits), or ""
37
+ # when the word has no A-Z letters
38
+ def code(word)
39
+ return "" if word.nil? || word.empty?
40
+
41
+ letters = word.upcase.gsub(/[^A-Z]/, "")
42
+ return "" if letters.empty?
43
+
44
+ # Keep first letter. The code is built into one mutable
45
+ # buffer (`<<`) instead of `code += digit`, which allocated
46
+ # a fresh String per encoded letter — the phonetic sweep
47
+ # pays this once per dictionary word.
48
+ code = +letters[0]
49
+
50
+ prev_code = encode(code)
51
+ i = 1
52
+ length = letters.length
53
+
54
+ while code.length < 4 && i < length
55
+ encoded = encode(letters[i])
56
+
57
+ # Add code if different from previous (ignore h and w)
58
+ code << encoded if encoded != "0" && encoded != prev_code
59
+
60
+ prev_code = encoded if encoded != "0"
61
+ i += 1
62
+ end
63
+
64
+ # Pad with zeros if needed
65
+ code.ljust(4, "0")[0...4]
66
+ end
67
+
68
+ # Soundex encoding table.
69
+ #
70
+ # @param char [String] The character
71
+ # @return [String] The encoded digit or "0" for no code
72
+ def encode(char)
73
+ case char.upcase
74
+ when "B", "P", "F", "V"
75
+ "1"
76
+ when "C", "S", "K", "G", "J", "Q", "X", "Z"
77
+ "2"
78
+ when "D", "T"
79
+ "3"
80
+ when "L"
81
+ "4"
82
+ when "M", "N"
83
+ "5"
84
+ when "R"
85
+ "6"
86
+ else
87
+ "0"
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -18,5 +18,6 @@ module Kotoshu
18
18
  autoload :PhonetSuggest, "kotoshu/algorithms/phonet_suggest"
19
19
  autoload :Capitalization, "kotoshu/algorithms/capitalization"
20
20
  autoload :EditDistance, "kotoshu/algorithms/edit_distance"
21
+ autoload :Soundex, "kotoshu/algorithms/soundex"
21
22
  end
22
23
  end
@@ -31,8 +31,11 @@ module Kotoshu
31
31
  # modules (lib/kotoshu/languages/). Staged-only dictionaries in
32
32
  # the dictionaries repo stay outside this list until their
33
33
  # module lands; local files come via setup --aff/--dic instead.
34
+ # Batch 3 (plan 100) promotes the RTL wiring ar/fa/he and adds
35
+ # bg et hr id lt lv sk sl sr.
34
36
  AVAILABLE_LANGUAGES = %w[
35
- ca cs da de el en es fr hu it nb nl pl pt ro ru sv tr uk vi
37
+ ar bg ca cs da de el en es et fa fr he hr hu id it lt lv nb nl
38
+ pl pt ro ru sk sl sr sv tr uk vi
36
39
  ].freeze
37
40
 
38
41
  # Get or download spelling dictionary for a language.
@@ -149,18 +149,45 @@ module Kotoshu
149
149
  nil
150
150
  end
151
151
 
152
+ # The sweep invariants for this dictionary's word list — char
153
+ # lengths, Soundex codes and length buckets — built once at the
154
+ # first suggestion sweep and shared by every later sweep (see
155
+ # {Suggestions::SweepIndex}).
156
+ #
157
+ # The index derives only from {#words}. Backends that mutate the
158
+ # word list (their {#add_word} / {#remove_word} implementations)
159
+ # call {#reset_sweep_index} so the memo cannot go stale; backends
160
+ # that cannot mutate after load (e.g. CSpell's frozen trie) never
161
+ # need to.
162
+ #
163
+ # @return [Suggestions::SweepIndex] The memoized sweep index
164
+ def sweep_index
165
+ @sweep_index ||= Suggestions::SweepIndex.build(words)
166
+ end
167
+
168
+ # Drop the memoized {#sweep_index} so the next sweep rebuilds it
169
+ # from the current word list. Mutating backends call this after
170
+ # every successful word-list change.
171
+ #
172
+ # @return [self] Self for chaining
173
+ def reset_sweep_index
174
+ @sweep_index = nil
175
+ self
176
+ end
177
+
152
178
  # Return words whose length is in [min_length, max_length].
153
179
  #
154
- # Default implementation filters all_words. Subclasses with a
155
- # length index (e.g., PlainText) override this for O(buckets)
156
- # instead of O(n) lookups — important for edit-distance
157
- # strategies that prune candidates by length.
180
+ # Uses the memoized {#sweep_index} length buckets — O(buckets in
181
+ # range) instead of an O(n) scan — and restores word-list order,
182
+ # exactly the set and order the historical whole-list filter
183
+ # produced. Subclasses that maintain their own length index
184
+ # (e.g. PlainText) keep their override.
158
185
  #
159
186
  # @param min_length [Integer] Minimum length (inclusive)
160
187
  # @param max_length [Integer] Maximum length (inclusive)
161
188
  # @return [Array<String>] Words in the length range
162
189
  def find_by_length_range(min_length:, max_length:)
163
- all_words.select { |w| w.length >= min_length && w.length <= max_length }
190
+ sweep_index.words_in_length_range(min_length, max_length)
164
191
  end
165
192
 
166
193
  # Get the number of words in the dictionary.
@@ -88,6 +88,7 @@ module Kotoshu
88
88
 
89
89
  @words << lookup_word
90
90
  @word_set[lookup_word] = true
91
+ reset_sweep_index
91
92
 
92
93
  true
93
94
  end
@@ -104,6 +105,7 @@ module Kotoshu
104
105
 
105
106
  @word_set.delete(lookup_word)
106
107
  @words.delete(lookup_word)
108
+ reset_sweep_index
107
109
 
108
110
  true
109
111
  end
@@ -121,6 +123,7 @@ module Kotoshu
121
123
  def clear
122
124
  @words.clear
123
125
  @word_set.clear
126
+ reset_sweep_index
124
127
  self
125
128
  end
126
129
 
@@ -313,6 +313,7 @@ module Kotoshu
313
313
 
314
314
  word_key = word.downcase
315
315
  @word_index[word_key] = flags
316
+ reset_sweep_index
316
317
 
317
318
  true
318
319
  end
@@ -320,12 +321,14 @@ module Kotoshu
320
321
  # Remove a word from the dictionary.
321
322
  #
322
323
  # @param word [String] The word to remove
323
- # @return [Boolean] True if removed
324
+ # @return [Boolean] True if the word was removed
324
325
  def remove_word(word)
325
326
  return false if word.nil? || word.empty?
326
327
 
327
328
  word_key = word.downcase
328
- !@word_index.delete(word_key).nil?
329
+ removed = @word_index.delete(word_key)
330
+ reset_sweep_index unless removed.nil?
331
+ !removed.nil?
329
332
  end
330
333
 
331
334
  # Get all words in the dictionary.
@@ -114,6 +114,7 @@ module Kotoshu
114
114
  @words << lookup_word
115
115
  @word_set[lookup_word] = true
116
116
  (@length_index[lookup_word.length] ||= []) << lookup_word
117
+ reset_sweep_index
117
118
 
118
119
  true
119
120
  end
@@ -131,6 +132,7 @@ module Kotoshu
131
132
  @word_set.delete(lookup_word)
132
133
  @words.delete(lookup_word)
133
134
  @length_index[lookup_word.length]&.delete(lookup_word)
135
+ reset_sweep_index
134
136
 
135
137
  true
136
138
  end
@@ -104,6 +104,7 @@ module Kotoshu
104
104
 
105
105
  @words << lookup_word
106
106
  @word_set[lookup_word] = true
107
+ reset_sweep_index
107
108
 
108
109
  true
109
110
  end
@@ -120,6 +121,7 @@ module Kotoshu
120
121
 
121
122
  @word_set.delete(lookup_word)
122
123
  @words.delete(lookup_word)
124
+ reset_sweep_index
123
125
 
124
126
  true
125
127
  end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Arabic 101 keyboard layout (standard IBM PC Arabic layout).
7
+ #
8
+ # The Arabic letters sit on the QWERTY key faces: the number row
9
+ # is shared with ASCII, and ض ص ث ق ف غ ع ه خ ح ج د / ش س ي ب ل ا
10
+ # ت ن م ك ط / ئ ء ؤ ر لا ى ة و ز ظ fill the three letter rows.
11
+ # Diacritics (tashkeel) are dead keys and are not in the grid;
12
+ # the hamza carriers أ إ آ and the ه/ة ي/ى spelling slips are
13
+ # handled at the normalizer level, not as physical keys.
14
+ #
15
+ # The key grid is mirrored from the models repo eval harness
16
+ # (kotoshu/models-fasttext-onnx eval/noise.py _AR_101, plan 83
17
+ # batch 2 of the model coverage expansion, verified there
18
+ # against the kbdlayout.info Arabic 101 driver table) so
19
+ # typo-adjacency used for suggestion ranking matches the
20
+ # adjacency used to evaluate the models.
21
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
22
+ # the sync.
23
+ #
24
+ # Languages: ar, ar-SA
25
+ class Arabic101 < Layout
26
+ # Key positions for the Arabic 101 layout.
27
+ # Each key maps to [row, column] coordinates.
28
+ KEY_POSITIONS = {
29
+ # Number row
30
+ 'ذ' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
31
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
32
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
33
+ '=' => [0, 12],
34
+ # Top row (ض ص ث ق ف غ ع ه خ ح ج د)
35
+ 'ض' => [1, 0], 'ص' => [1, 1], 'ث' => [1, 2], 'ق' => [1, 3],
36
+ 'ف' => [1, 4], 'غ' => [1, 5], 'ع' => [1, 6], 'ه' => [1, 7],
37
+ 'خ' => [1, 8], 'ح' => [1, 9], 'ج' => [1, 10], 'د' => [1, 11],
38
+ # Home row (ش س ي ب ل ا ت ن م ك ط)
39
+ 'ش' => [2, 0], 'س' => [2, 1], 'ي' => [2, 2], 'ب' => [2, 3],
40
+ 'ل' => [2, 4], 'ا' => [2, 5], 'ت' => [2, 6], 'ن' => [2, 7],
41
+ 'م' => [2, 8], 'ك' => [2, 9], 'ط' => [2, 10],
42
+ # Bottom row (ئ ء ؤ ر لا ى ة و ز ظ)
43
+ 'ئ' => [3, 0], 'ء' => [3, 1], 'ؤ' => [3, 2], 'ر' => [3, 3],
44
+ 'لا' => [3, 4], 'ى' => [3, 5], 'ة' => [3, 6], 'و' => [3, 7],
45
+ 'ز' => [3, 8], 'ظ' => [3, 9]
46
+ }.freeze
47
+
48
+ # Initialize the Arabic 101 layout.
49
+ def initialize
50
+ super(
51
+ name: 'Arabic-101',
52
+ language_codes: %w[ar ar-SA],
53
+ key_positions: KEY_POSITIONS
54
+ )
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Keyboard
5
+ module Layouts
6
+ # Bulgarian BDS keyboard layout (БДС 5237:1978, KBDBUL.DLL).
7
+ #
8
+ # The national Bulgarian phonetic-adjacent ordering: ы у е и ш щ
9
+ # к с д з ц / ь я а о ж г т н в м ч / ю й ъ э ф х п р л б. The
10
+ # Bulgarian-only letters ъ ь are real keys (unlike JCUKEN, where
11
+ # ъ sits top-right and ы is the home-row opener).
12
+ #
13
+ # The key grid is mirrored from the models repo eval harness
14
+ # (kotoshu/models-fasttext-onnx eval/noise.py _BG_BDS, plan 83
15
+ # batch 2 of the model coverage expansion, verified there
16
+ # against the kbdlayout.info KBDBUL driver table) so
17
+ # typo-adjacency used for suggestion ranking matches the
18
+ # adjacency used to evaluate the models.
19
+ # spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
20
+ # the sync.
21
+ #
22
+ # Languages: bg, bg-BG
23
+ class BulgarianBds < Layout
24
+ # Key positions for the Bulgarian BDS layout.
25
+ # Each key maps to [row, column] coordinates.
26
+ KEY_POSITIONS = {
27
+ # Number row
28
+ ')' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
29
+ '4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
30
+ '8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '$' => [0, 11],
31
+ '€' => [0, 12],
32
+ # Top row (ы у е и ш щ к с д з ц)
33
+ 'ы' => [1, 0], 'у' => [1, 1], 'е' => [1, 2], 'и' => [1, 3],
34
+ 'ш' => [1, 4], 'щ' => [1, 5], 'к' => [1, 6], 'с' => [1, 7],
35
+ 'д' => [1, 8], 'з' => [1, 9], 'ц' => [1, 10],
36
+ # Home row (ь я а о ж г т н в м ч)
37
+ 'ь' => [2, 0], 'я' => [2, 1], 'а' => [2, 2], 'о' => [2, 3],
38
+ 'ж' => [2, 4], 'г' => [2, 5], 'т' => [2, 6], 'н' => [2, 7],
39
+ 'в' => [2, 8], 'м' => [2, 9], 'ч' => [2, 10],
40
+ # Bottom row (ю й ъ э ф х п р л б)
41
+ 'ю' => [3, 0], 'й' => [3, 1], 'ъ' => [3, 2], 'э' => [3, 3],
42
+ 'ф' => [3, 4], 'х' => [3, 5], 'п' => [3, 6], 'р' => [3, 7],
43
+ 'л' => [3, 8], 'б' => [3, 9]
44
+ }.freeze
45
+
46
+ # Initialize the Bulgarian BDS layout.
47
+ def initialize
48
+ super(
49
+ name: 'Bulgarian-BDS',
50
+ language_codes: %w[bg bg-BG],
51
+ key_positions: KEY_POSITIONS
52
+ )
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end