kotoshu 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/.pre-commit-hooks.yaml +18 -0
  3. data/.yardopts +8 -0
  4. data/CHANGELOG.md +109 -0
  5. data/CLAUDE.md +5 -22
  6. data/README.adoc +90 -0
  7. data/TODO.impl/00-vision.md +51 -0
  8. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  9. data/TODO.impl/78-in-browser-playground.md +69 -0
  10. data/TODO.impl/79-news-and-releases.md +60 -0
  11. data/TODO.impl/80-ecosystem-docs.md +56 -0
  12. data/TODO.impl/81-editor-reach.md +68 -0
  13. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  14. data/TODO.impl/83-models-batch-2.md +56 -0
  15. data/TODO.impl/84-language-modules-wave2.md +75 -0
  16. data/TODO.impl/85-wasm-model-api.md +48 -0
  17. data/TODO.impl/86-site-truth-wave2.md +45 -0
  18. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  19. data/TODO.impl/88-cli-directory-mode.md +54 -0
  20. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  21. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  22. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  23. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  24. data/TODO.impl/93-python-wheels.md +32 -0
  25. data/TODO.impl/94-site-integrations-docs.md +35 -0
  26. data/TODO.impl/95-editor-action-polish.md +27 -0
  27. data/TODO.impl/96-server-semantic-models.md +50 -0
  28. data/TODO.impl/97-yard-execution.md +73 -0
  29. data/lib/kotoshu/analyzers/semantic_analyzer.rb +1 -1
  30. data/lib/kotoshu/baseline/application.rb +18 -0
  31. data/lib/kotoshu/baseline/entry.rb +33 -0
  32. data/lib/kotoshu/baseline/store.rb +152 -0
  33. data/lib/kotoshu/baseline.rb +20 -0
  34. data/lib/kotoshu/cache/frequency_cache.rb +2 -2
  35. data/lib/kotoshu/cache/language_cache.rb +33 -11
  36. data/lib/kotoshu/cache/model_cache.rb +2 -2
  37. data/lib/kotoshu/cli/baseline_command.rb +81 -0
  38. data/lib/kotoshu/cli/completions_command.rb +1 -1
  39. data/lib/kotoshu/cli/directory_check.rb +141 -0
  40. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  41. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  42. data/lib/kotoshu/cli.rb +304 -59
  43. data/lib/kotoshu/configuration.rb +8 -2
  44. data/lib/kotoshu/core/models/result/document_result.rb +13 -3
  45. data/lib/kotoshu/core/models/result/word_result.rb +19 -1
  46. data/lib/kotoshu/data/common_words_loader.rb +0 -1
  47. data/lib/kotoshu/dictionary/cspell.rb +3 -8
  48. data/lib/kotoshu/documents/document.rb +12 -2
  49. data/lib/kotoshu/documents/plain_text_document.rb +5 -2
  50. data/lib/kotoshu/documents/source_position.rb +8 -0
  51. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  52. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  53. data/lib/kotoshu/documents/suppressions.rb +44 -0
  54. data/lib/kotoshu/documents.rb +1 -0
  55. data/lib/kotoshu/embeddings/embedding_pipeline.rb +3 -5
  56. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +2 -2
  57. data/lib/kotoshu/jekyll.rb +98 -0
  58. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  59. data/lib/kotoshu/keyboard/layouts/latin.rb +220 -0
  60. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  61. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  62. data/lib/kotoshu/keyboard/registry.rb +20 -0
  63. data/lib/kotoshu/keyboard.rb +8 -0
  64. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  65. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  66. data/lib/kotoshu/language/tokenizer/base.rb +17 -0
  67. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  68. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  69. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +12 -0
  70. data/lib/kotoshu/language.rb +4 -0
  71. data/lib/kotoshu/languages/ca/language.rb +22 -0
  72. data/lib/kotoshu/languages/cs/language.rb +22 -0
  73. data/lib/kotoshu/languages/da/language.rb +22 -0
  74. data/lib/kotoshu/languages/el/language.rb +50 -0
  75. data/lib/kotoshu/languages/hu/language.rb +22 -0
  76. data/lib/kotoshu/languages/it/language.rb +22 -0
  77. data/lib/kotoshu/languages/latin_base.rb +53 -0
  78. data/lib/kotoshu/languages/nl/language.rb +22 -0
  79. data/lib/kotoshu/languages/pl/language.rb +22 -0
  80. data/lib/kotoshu/languages/ro/language.rb +22 -0
  81. data/lib/kotoshu/languages/sv/language.rb +22 -0
  82. data/lib/kotoshu/languages/tr/language.rb +36 -0
  83. data/lib/kotoshu/languages/uk/language.rb +38 -0
  84. data/lib/kotoshu/languages/vi/language.rb +22 -0
  85. data/lib/kotoshu/languages.rb +18 -0
  86. data/lib/kotoshu/metrics_collector.rb +2 -2
  87. data/lib/kotoshu/models/nearest_neighbor.rb +11 -3
  88. data/lib/kotoshu/paths.rb +26 -0
  89. data/lib/kotoshu/readers/aff_data.rb +7 -3
  90. data/lib/kotoshu/readers/aff_reader.rb +39 -31
  91. data/lib/kotoshu/readers/lookup_builder.rb +1 -1
  92. data/lib/kotoshu/resource_bundle.rb +32 -7
  93. data/lib/kotoshu/resource_manager.rb +137 -1
  94. data/lib/kotoshu/rspec.rb +163 -0
  95. data/lib/kotoshu/source_registry.rb +5 -0
  96. data/lib/kotoshu/spellchecker.rb +64 -22
  97. data/lib/kotoshu/suggestions/generator.rb +4 -5
  98. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +2 -2
  99. data/lib/kotoshu/tasks/check_task.rb +98 -0
  100. data/lib/kotoshu/tasks.rb +11 -0
  101. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  102. data/lib/kotoshu/validators.rb +9 -0
  103. data/lib/kotoshu/version.rb +1 -1
  104. data/lib/kotoshu.rb +23 -9
  105. data/scripts/extract_eval_grids.rb +34 -0
  106. data/sig/kotoshu.rbs +331 -1
  107. metadata +69 -3
@@ -0,0 +1,68 @@
1
+ # 81 — Editor reach: VS Code extension + pre-commit
2
+
3
+ > Supersedes the unshipped parts of
4
+ > [61-editor-ecosystem.md](61-editor-ecosystem.md). Status
5
+ > correction: kotoshu-lsp 0.1.0 (plan 60), kotoshu-server 0.1.0 +
6
+ > SDKs (plan 64, partial), and action-kotoshu v1 shipped during the
7
+ > 2026-09 campaign; what remains is the editor package and the
8
+ > pre-commit hook.
9
+
10
+ ## Context
11
+
12
+ The LSP server exists (`kotoshu-lsp` gem) but nothing delivers it
13
+ into editors. The generic-LSP path requires manual config most
14
+ users never do. VS Code via a marketplace extension is the single
15
+ highest-leverage editor surface. pre-commit is where CI-minded
16
+ Python/Go shops hook spellchecking.
17
+
18
+ ## Track A — `kotoshu-vscode` (new repo)
19
+
20
+ - Scaffold: TypeScript extension, `vscode-languageclient`, no
21
+ bundling surprises (esbuild), `package.json` contributes
22
+ `kotoshu.languageTool` (document selectors: markdown, asciidoc,
23
+ plaintext, code comments opt-in later).
24
+ - Server discovery order: `kotoshu-lsp.serverPath` setting →
25
+ `bundle exec kotoshu-lsp` in workspace → global `kotoshu-lsp` →
26
+ actionable error message with install command (no silent
27
+ failure).
28
+ - Features v0.1: publish diagnostics on open/change/save;
29
+ suggestions as code actions; personal-dictionary add action
30
+ (server command if supported, else documented).
31
+ - Repo: `kotoshu/kotoshu-vscode`, CI (build + `vscode-test` smoke
32
+ against a fixture workspace with a deliberate misspelling),
33
+ LICENSE BSD-2-Clause, README with GIF slot.
34
+ - Packaging: `vsce package` produces a .vsix in CI artifacts.
35
+ **Marketplace publish is owner-gated** (needs publisher PAT);
36
+ Open VSX after that.
37
+
38
+ ## Track B — pre-commit hook
39
+
40
+ - `.pre-commit-hooks.yaml` living where the hook can `gem install`
41
+ or reuse a bundled environment. Evaluate: hook id
42
+ `kotoshu-spell` running `kotoshu check` (requires Ruby+gem —
43
+ document honestly), vs a container hook (`language: docker_ruby`
44
+ via the published image). Ship the honest one with clear docs;
45
+ note the wasm client as the future zero-Ruby path.
46
+ - Placement: `action-kotoshu` repo (CI-adjacent) or the gem repo
47
+ (ships with the tool). Prefer the **gem repo** — the hook file
48
+ documents itself beside the CLI it calls.
49
+
50
+ ## Phases
51
+
52
+ A1 scaffold → A2 discovery + diagnostics → A3 code actions + smoke
53
+ CI → B1 hook + docs → B2 (owner) marketplace publish.
54
+
55
+ ## Owner gates
56
+
57
+ - Marketplace publish (Track A final step).
58
+ - Open VSX mirror.
59
+
60
+ ## Status
61
+
62
+ **Track A done (2026-09-05): kotoshu/kotoshu-vscode exists** - TS
63
+ extension wrapping kotoshu-lsp over stdio, diagnostics + quickfix code
64
+ actions + hover, server discovery (setting -> bundler -> PATH) with
65
+ actionable install error, CI green including a vscode-test smoke
66
+ against a misspelling fixture, vsce package artifact gated by a manifest
67
+ check. Marketplace publish remains owner-gated (publisher PAT).
68
+ **Track B done: .pre-commit-hooks.yaml shipped in gem PR #118.**
@@ -0,0 +1,81 @@
1
+ # 82 — Document ignores + CI baselines
2
+
3
+ ## Context
4
+
5
+ Two long-standing user gaps in the gem (verified 2026-09-05 — no
6
+ `disable` handling exists anywhere in `lib/kotoshu/documents/`):
7
+
8
+ 1. **Inline ignores** — a word that is intentionally "wrong"
9
+ (product names, code identifiers in prose, dialect) cannot be
10
+ suppressed per-line; users fall back to polluting the personal
11
+ dictionary.
12
+ 2. **CI baselines** — `kotoshu check` in CI fails on ALL known
13
+ errors forever; there is no way to freeze existing debt and fail
14
+ only on new errors. Every serious linter has this (ESLint
15
+ `--baseline`, RuboCop `--exclude-limit` culture, cspell
16
+ `--no-cache` + wordlists).
17
+
18
+ ## Track A — ignore directives
19
+
20
+ Syntax (comments recognized by each document format's parser):
21
+
22
+ ```
23
+ kotoshu:disable-line — suppress everything on this line
24
+ kotoshu:disable-next-line — suppress on the following line
25
+ kotoshu:disable-next-line foo bar — suppress only these words
26
+ kotoshu:disable-file / kotoshu:enable-file — block suppression
27
+ ```
28
+
29
+ - Markdown: HTML comments `<!-- kotoshu:disable-next-line -->`;
30
+ AsciiDoc: `// kotoshu:disable-next-line`; plain text: a line whose
31
+ trimmed content is the directive. Trailing directives
32
+ (`text <!-- kotoshu:disable-line -->`) supported in MD/ADOC.
33
+ - Implementation: document layer (`documents/`) attaches
34
+ `suppressions` (source ranges) during parse; the checker filters
35
+ results whose range/word matches. Suppression data flows through
36
+ the public result objects (`results/*`) so SARIF/JSON consumers
37
+ see suppressed entries either omitted or marked (omitted by
38
+ default; `--show-suppressed` lists them).
39
+ - Also recognized: the spellchecker facade (`.check`) so the gem
40
+ API gets it for free.
41
+
42
+ ## Track B — baseline files
43
+
44
+ - `kotoshu check --baseline .kotoshu-baseline.json`:
45
+ - Baseline format: canonical JSON `{file, line, word, count}` —
46
+ stable across reformatting (count-based, not position-based).
47
+ - `kotoshu baseline init` generates it from current findings.
48
+ - Semantics: an error passes if it matches the baseline
49
+ (same file+word, count not exceeded); new errors fail; a
50
+ summary line reports how many baseline entries are now stale
51
+ (file no longer has the error) so debt shrinks visibly.
52
+ - SARIF output marks suppressed entries with the baseline tag.
53
+
54
+ ## Specs
55
+
56
+ Behavior specs throughout (no doubles; real documents). Cover:
57
+ directive forms per format, word-scoped suppression, block
58
+ disable/enable nesting, baseline count semantics, stale-entry
59
+ reporting, SARIF marking.
60
+
61
+ ## Phases
62
+
63
+ A1 directive parsing + filtering → A2 per-format specs + docs page
64
+ section → B1 baseline format + commands → B2 CI docs example.
65
+
66
+ ## Owner gates
67
+
68
+ None. CHANGELOG entry under Unreleased (version = owner).
69
+
70
+ ## Status
71
+
72
+ **Implemented (2026-09-05, gem PR #118).** Shared
73
+ Documents::Suppressions scanner (disable-line, disable-next-line with
74
+ optional word list, nestable disable-file/enable-file blocks; per-format
75
+ comment detection; directive lines never spellchecked); check filters
76
+ via the :auto profile; suppressed errors ride DocumentResult with
77
+ suppressed/suppressed_by flags; --show-suppressed. Baselines: baseline
78
+ init + check --baseline with count-based budgets, stale-entry
79
+ reporting, SARIF suppression marking; serialized via lutaml-model.
80
+ 55 new examples; default suite 3410/0/26. Release cut pending owner
81
+ version decision.
@@ -0,0 +1,56 @@
1
+ # 83 — Models batch 2: every dictionary language FastText can serve
2
+
3
+ > Wave-2 continuation of [77](77-model-coverage-expansion.md).
4
+ > v1.1.0 shipped 22 languages; the user asks again: are the models
5
+ > comprehensive? Not yet — this plan drives toward full intersection
6
+ > coverage.
7
+
8
+ ## Context
9
+
10
+ - Dictionaries repo: 98 languages. Models registry v1.1.0: 22.
11
+ - FastText CC vectors exist for 157+ languages.
12
+ - The intersection (dictionaries ∩ fastText, minus the 22 shipped) is
13
+ the candidate pool — enumerate it exactly; expect ~50-60.
14
+ - Notable misses with gem language modules already wired: **ar, fa,
15
+ he** (RTL modules exist; no models). These are the highest-priority
16
+ additions — semantic reranking for RTL users currently does nothing.
17
+
18
+ ## Phases
19
+
20
+ 1. **Enumerate** the intersection; rank by (existing gem module,
21
+ speaker base, dictionary size). ar, fa, he lead.
22
+ 2. **Ship in priority order under a compute budget** — same pipeline,
23
+ same gates (fluency rank_corr ≥ 0.97 / top1 ≥ 0.95, mini ≥ 0.90 /
24
+ 0.85), same keyboard-aware eval. Extend `eval/noise.py` per
25
+ language as needed (Hebrew/Arabic grids; reuse wave-1 patterns).
26
+ Target ≥ 15 shipped in this batch (ar fa he + a dozen more);
27
+ record the remaining tail in the registry README as the backlog.
28
+ 3. **fi / id dictionaries** — not in the dictionaries repo. Source
29
+ from upstream (LibreOffice/hunspell releases) INTO the
30
+ dictionaries repo with license metadata per its manifest schema;
31
+ then run them through the same pipeline. LICENSE VERIFICATION IS
32
+ A HARD GATE — if provenance is unclear, skip and report.
33
+ 4. **`no` → nb/nn mapping** — proposal only: registry aliases or gem
34
+ resolution mapping `no` to `nb` (Bokmål). Owner decision; do not
35
+ implement unilaterally.
36
+ 5. **Release v1.2.0** (plan 05 policy: coverage = minor), 3 tiers per
37
+ new language, media-host mirrors, byte-identical registry proof.
38
+
39
+ ## Owner gates
40
+
41
+ - `no` mapping decision (proposal delivered, not executed).
42
+ - Version follows the plan-05 pre-declared policy as v1.1.0 did.
43
+
44
+ ## Status
45
+
46
+ **Implemented (2026-09-05, models PR #14 + tag v1.2.0).** 22 -> 54
47
+ languages (32 shipped: fa he ar id hy hr bg sr sk mn lt ka ne nn sl
48
+ mk lv et tk la gl eu cy eo is lb fy br ga gd oc ia), 162 registry
49
+ resources rev 3, 326 assets, byte-identity proven. ar/id
50
+ dictionaries sourced with verified licenses (dictionaries PRs #1-#3,
51
+ that repo's default branch is v1). Gates never weakened. DROPS:
52
+ nds (ISO 639-2-only vs the two-letter schema contract - owner spec
53
+ decision), fi (no license-clear upstream - hard gate), fo rw ie fur
54
+ tlh (no fastText CC vectors). BACKLOG EMPTY: every dictionaries
55
+ language FastText can serve ships. Owner proposal pending: no->nb/nn
56
+ (cc.no as Bokmal-labeled nb + gem resolves no->nb).
@@ -0,0 +1,75 @@
1
+ # 84 — Language modules + keyboards for the 22 model languages
2
+
3
+ ## Context
4
+
5
+ The models registry now serves 22 languages, but gem language modules
6
+ exist for only 10 (ar de en es fa fr he ja pt ru) and keyboard layouts
7
+ for 5 (azerty dvorak jcuken qwerty qwertz). A Turkish user gets a
8
+ model but no Turkish-Q keyboard proximity; Greek, Ukrainian, Polish,
9
+ Czech, Hungarian, Nordic users likewise. The site honestly labels this
10
+ gap ("staged dictionary + model until the module is wired") — close it.
11
+
12
+ ## Track A — keyboard layouts
13
+
14
+ New layouts in `lib/kotoshu/keyboard/layouts/`: tr (Turkish-Q), uk
15
+ (ЙЦУКЕН Ukrainian — extends jcuken with ҐЄІЇ), el (Greek), pl, cs,
16
+ hu, da/nb/sv (Nordic shared layout with per-language diacritics), ro,
17
+ ca, it, nl (Latin layouts parameterized over the qwerty/qwertz base —
18
+ prefer ONE parameterized Latin layout family + per-language key maps
19
+ over 11 near-duplicate files; keep the existing five as-is).
20
+
21
+ **Single source of truth**: the models repo's `eval/noise.py` grids
22
+ (wave-1 added Turkish-Q / Ukrainian-JCUKEN / Greek-phonetic) encode
23
+ the same adjacency data — derive the gem layouts to match them and
24
+ record the sync note in both repos' plans (drift check: a spec
25
+ asserting the gem layout keys ⊇ the eval grid keys for tr/uk/el).
26
+
27
+ ## Track B — language modules
28
+
29
+ `lib/kotoshu/languages/` modules for the 13 wave-1 Latin newcomers
30
+ (ca cs da el hu it nl pl ro sv tr uk vi): most are thin — Latin
31
+ tokenizer/normalizer composition + layout + registry entry. Follow the
32
+ existing module pattern (see `de`, `pt`); el gets Greek-aware
33
+ normalization; tr gets case-folding care (dotless i); vi may reuse
34
+ Latin defaults. Each module ships with specs against real dictionary
35
+ fixtures (no doubles).
36
+
37
+ ## Track C — site truth (executed by the site agent, not here)
38
+
39
+ Full-feature list grows; the "staged + model" note shrinks. Recorded
40
+ here as the dependency so the site agent can sequence on this PR.
41
+
42
+ ## Constraints
43
+
44
+ - No behavior change for the existing 10 modules.
45
+ - `register_suggestion_algorithm`/layout registry patterns unchanged
46
+ (OCP: layouts register, nothing edits a switch).
47
+ - Suite stays green; add specs per layout + module.
48
+
49
+ ## Sync note (the drift check)
50
+
51
+ The gem layouts for tr/uk/el are derived from
52
+ models-fasttext-onnx `eval/noise.py` (`_TR_Q`, `_UK_JCUKEN`,
53
+ `_EL_PHONETIC`, wave 1 / registry v1.1.0) and must stay byte-equal.
54
+ `spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb` enforces it
55
+ both ways: against the live models-repo checkout when one is present
56
+ (workspace sibling or `KOTOSHU_MODELS_REPO`; skipped when the
57
+ checkout predates the wave-1 grids) and against the committed
58
+ snapshot `spec/fixtures/models_eval_grids.json` otherwise (CI).
59
+ After changing the eval grids, refresh the snapshot with
60
+ `ruby scripts/extract_eval_grids.rb PATH/eval/noise.py`. The models
61
+ repo carries the mirror-direction note in its plan 77 status.
62
+
63
+ ## Status
64
+
65
+ **Implemented 2026-09-05 (gem PR feat/language-modules-wave2).**
66
+ Track A: Turkish-Q, Ukrainian-JCUKEN, Greek-Phonetic national
67
+ layouts plus one parameterized Latin family (11 members over the
68
+ qwerty/qwertz base grids; Nordic members carry real å/æ/ø and å/ä/ö
69
+ keys); registry 5 -> 19 layouts; the pre-existing five layout files
70
+ are untouched (Ukrainian registers before JCUKEN so `uk` resolves to
71
+ its dedicated grid). Track B: 13 thin modules on a shared
72
+ `Languages::LatinBase` (tokenizer/normalizer composition), with
73
+ Turkish dotless-i folding, Greek final-sigma/accent normalizer,
74
+ Greek/Cyrillic script tokenizers; the existing ten modules are
75
+ unchanged. Track C (site) stays with the site agent.
@@ -0,0 +1,48 @@
1
+ # 85 — Browser semantic reranking: wasm model API
2
+
3
+ ## Context
4
+
5
+ The playground is honestly dictionary-only for two reasons found in
6
+ wave 1: GitHub release assets send no CORS headers, and the wasm
7
+ surface exposes no model API. The first is a distribution problem;
8
+ the second is the real gap — this plan closes both.
9
+
10
+ ## Track A — CORS spike (decides the distribution path)
11
+
12
+ Check `Access-Control-Allow-Origin` on: the media-host mirror URLs in
13
+ the registry, the models repo's raw.githubusercontent paths, and the
14
+ registry.json itself (which the browser must read to discover URLs).
15
+ Outcomes:
16
+ - Media host serves ACAO → use registry mirror URLs directly.
17
+ - Nothing serves ACAO → tier artifacts move into the jsDelivr-servable
18
+ git tree (tiers are LFS-free and ~3-15 MB; jsDelivr gh caps at
19
+ 20 MB) via a registry "browser_url" field — additive schema, old
20
+ consumers unaffected.
21
+ Record the verdict + evidence in this file.
22
+
23
+ ## Track B — wasm model API
24
+
25
+ Expose on `@kotoshu/wasm` (wasm-bindgen surface, kotoshu-wasm crate):
26
+ - `createModelDictionary`-adjacent: `loadModel(bytes, vocabBytes)`
27
+ parsing the int8 tier (quantized matrix + vocab) into memory.
28
+ - `rerank(word, context) -> f32 score` (dot-product against the
29
+ quantized embeddings, dequantized on the fly) — enough to drive the
30
+ same reranking the gem does. Pure Rust, NO ort in the wasm build
31
+ (load-dynamic cannot work in a browser); the int8 tiers exist
32
+ precisely so this is cheap.
33
+ - Memory discipline: document peak memory (vocab 10k-50k × int8 ×
34
+ dim ≈ 3-15 MB) and free on `drop`.
35
+ - Node smoke in CI exercises load + rerank against a real mini-tier
36
+ fixture (checked-in, small) — the same pattern as the dictionary
37
+ smoke.
38
+
39
+ ## Track C — release readiness (owner-gated)
40
+
41
+ npm version 0.2.0 candidate: PR + CI only. Publishing @kotoshu/wasm
42
+ is the owner's version call (tag `@kotoshu/wasm-v*` publishes keyless
43
+ once decided). Playground wiring happens after the API ships, as a
44
+ site follow-up.
45
+
46
+ ## Status
47
+
48
+ **Pending.**
@@ -0,0 +1,45 @@
1
+ # 86 — Site truth wave 2: ignores, Windows, playground breadth
2
+
3
+ ## Context
4
+
5
+ Wave 1 shipped features the site does not document yet, and one docs
6
+ bug: the CLI page claims the personal dictionary lives at
7
+ `~/.config/kotoshu/personal_dictionary` — the gem actually uses
8
+ `personal.dic` (Paths.personal_dictionary_path, KOTOSHU_PERSONAL_DIC
9
+ override). Wrong paths in docs are worse than no docs.
10
+
11
+ ## Phases
12
+
13
+ 1. **CLI docs fixes + ignoring & baselines guide**: correct the
14
+ personal-dictionary path everywhere it appears; new docs page
15
+ covering `kotoshu:disable-line` / `-next-line [WORDS]` /
16
+ `-file` blocks per format, `kotoshu baseline init` + `check
17
+ --baseline` semantics (count budgets, stale entries, SARIF
18
+ suppression marking), and the pre-commit hook — matching gem PR
19
+ #118 exactly (read its README sections + specs as the source of
20
+ truth, not this plan).
21
+ 2. **Windows page**: the CI matrix now runs 3.3/3.4/4.0 on
22
+ windows-latest green — document first-class Windows support:
23
+ `gem install kotoshu` on Windows, onnxruntime soft-dep behavior,
24
+ path handling, and the native extension's Windows status (it is
25
+ NOT built in CI — pending; say so honestly). Link from /install.
26
+ 3. **Playground breadth**: offer every language whose dictionary the
27
+ CDN can serve well — at minimum the 22 model languages plus the
28
+ staged-dictionary set with sane sizes (cap the list at languages
29
+ whose .aff+.dic < ~5 MB on the CDN; largest dictionaries get a
30
+ size note). Dictionary-only behavior unchanged until plan 85
31
+ ships.
32
+ 4. **Matrix flip** (sequences on plan 84): when the gem modules land,
33
+ FULL_LANGUAGES grows per the merged module list; the "staged +
34
+ model" note shrinks. If 84 has not merged when you reach this
35
+ phase, skip it and note it — do not block the other phases.
36
+
37
+ ## Constraints
38
+
39
+ House rules: verify against the gem source (paths, flags), run the
40
+ build, CDP-check the playground still works after the language list
41
+ change. No new dependencies.
42
+
43
+ ## Status
44
+
45
+ **Pending.**
@@ -0,0 +1,36 @@
1
+ # 87 — kotoshu-lsp: read the personal dictionary
2
+
3
+ ## Context
4
+
5
+ The VS Code extension's "add to personal dictionary" writes
6
+ `~/.config/kotoshu/personal.dic` (same path + format as the gem CLI —
7
+ `Kotoshu::Paths.personal_dictionary_path`), but the LSP server never
8
+ READS it. The user adds a word, the diagnostic stays, the feature
9
+ feels broken. Found by the vscode agent (wave 1) and confirmed by
10
+ grep: server.rb only *writes* suggestions for the action.
11
+
12
+ ## Design
13
+
14
+ - On server start AND on dictionary-file change, load the personal
15
+ dictionary (gem API: `Kotoshu::PersonalDictionary` — reuse, do not
16
+ re-parse) into the spellcheck path so its words are correct.
17
+ - File watching: simplest honest approach that works everywhere —
18
+ mtime check on didOpen/didChange re-checks (cheap stat), plus a
19
+ documented limitation note in README (no OS file watcher needed
20
+ for v0.1.x).
21
+ - After an add-to-dictionary code action (server executes the write
22
+ itself, or the client notifies via workspace/didChangeConfiguration
23
+ / custom notification), re-publish diagnostics for open documents.
24
+ Prefer: the SERVER performs the write (single writer, no
25
+ client/server filename drift) — the extension's client-side write
26
+ moves to the server command; keep the extension working with both
27
+ (it may still write for its own UI; server write wins when present).
28
+ - Specs: real temp personal.dic files (no doubles); add → diagnostic
29
+ clears; KOTOSHU_PERSONAL_DIC env honored (gem Paths already does).
30
+
31
+ Leaves the repo 0.1.1-ready (owner dispatches the release; RubyGems
32
+ 0.1.0 is empty — see the wave-1 fixes).
33
+
34
+ ## Status
35
+
36
+ **Pending.**
@@ -0,0 +1,54 @@
1
+ # 88 — CLI directory mode: check a whole tree
2
+
3
+ ## Context
4
+
5
+ `kotoshu check` handles one file or stdin. Every serious checker
6
+ (cspell, codespell, hunspell wrappers) checks a directory tree — that
7
+ is the actual CI workflow (`action-kotoshu` users fake it with
8
+ find/xargs today). Baselines (gem PR #118) already key on file paths,
9
+ so directory mode + baselines is the complete CI story.
10
+
11
+ ## Design
12
+
13
+ - `kotoshu check DIR [DIR...]` — walk the tree; check files whose
14
+ extension is a known text format (markdown, asciidoc, txt, rst,
15
+ adoc, mdx) unless `--include`/`--exclude` globs say otherwise.
16
+ - Respect `.gitignore` (and `.ignore`) semantics: implement the
17
+ standard glob subset (directories, `*`, `**`, `?`, negation `!`,
18
+ trailing `/`); document the subset honestly. NO shelling out to
19
+ git — the vision forbids external binaries.
20
+ - Hidden files/dirs and `node_modules`/`.git`/`vendor`/`target`
21
+ skipped by default.
22
+ - Output: per-file sections in text mode; one combined SARIF/JSON
23
+ document with per-file runs. `--baseline` applies per file+word as
24
+ shipped. Exit codes unchanged (1 = errors found).
25
+ - Interactive mode stays file-only (documented no-op with a message
26
+ for directories).
27
+
28
+ ## Specs
29
+
30
+ Real directory trees in specs (tmpdirs with fixtures + .gitignore
31
+ cases: negation, nested ignores, globstar). Behavior specs, no
32
+ doubles.
33
+
34
+ ## Status
35
+
36
+ **Implemented 2026-09-05 (gem PR feat/cli-directory-mode).**
37
+ `kotoshu check FILE [DIR ...]`: DirectoryWalker selects known text
38
+ extensions by default (md markdown asciidoc adoc txt rst mdx) with
39
+ --include/--exclude globs replacing/overriding the default
40
+ (slashless globs match basenames, slashed globs match the path from
41
+ the root); hidden entries and .git/node_modules/vendor/target are
42
+ always skipped. IgnoreMatcher implements the .gitignore/.ignore
43
+ glob subset in Ruby (no shelling out): * ? ** anchoring trailing-/
44
+ directory patterns ! negation with last-match-wins, nested
45
+ scoping, and git-style no-re-include under ignored directories.
46
+ Output: per-file text sections + summary; one combined JSON
47
+ document (fileCount/wordCount/errorCount/files[] with the
48
+ single-file payload keys); one SARIF run per file. --baseline
49
+ applies per file unchanged; exit codes unchanged; interactive mode
50
+ prints a file-only notice for directory targets. Single-file and
51
+ stdin paths are byte-identical (the single-file code path is
52
+ preserved verbatim). Behavior specs run the real CLI against real
53
+ tmpdir trees with the real en dictionary from committed fixtures -
54
+ no network, no doubles.
@@ -0,0 +1,59 @@
1
+ # 89 — Framework integrations (executes plan 62)
2
+
3
+ > [62-framework-integrations.md](62-framework-integrations.md) is the
4
+ > original plan — still Pending. This file is the wave-2 execution
5
+ > order; 62's design stands.
6
+
7
+ ## What ships (inside the gem, soft-dependency style)
8
+
9
+ 1. **Rails / ActiveModel validator** — `validates :body,
10
+ spelling: true` (an EachValidator): checks the attribute with
11
+ Kotoshu, adds one error per misspelling with the top suggestion.
12
+ Config per-attribute: language, personal words. NO Rails
13
+ dependency in the gemspec — the validator autoloads only when
14
+ ActiveModel is defined (the onnxruntime/suika soft-dep pattern).
15
+ 2. **RSpec matcher** — `expect_words("helo").to all_be_spelled_correctly`
16
+ style plus a block matcher over a document; ships with proper
17
+ failure messages.
18
+ 3. **Rake task** — `kotoshu/check.rake`: checks changed markdown/
19
+ asciidoc in the repo (uses the same file-selection as plan 88),
20
+ wired via `gem 'kotoshu'` + `require 'kotoshu/tasks'`.
21
+ 4. **Jekyll** — a generator plugin class checking posts/drafts,
22
+ failing the build on new errors (respects baseline if present).
23
+
24
+ Each integration has specs that load the real framework pieces where
25
+ feasible (ActiveModel is loadable standalone in specs without Rails).
26
+
27
+ ## What does NOT ship
28
+
29
+ Engine packaging, `kotoshu-rails` separate gem, Sidekiq batch
30
+ checkers — plan 62 phase 3, demand-gated. (Also not shipped from 62
31
+ phases 1-2: the railtie, rails generator, dry-validation macro,
32
+ Capybara/Minitest helpers, ActionMailer preview checks.)
33
+
34
+ ## Status
35
+
36
+ **Implemented 2026-09-05 (gem PR feat/framework-integrations).**
37
+ All four items shipped inside the gem on the soft-dependency
38
+ pattern (nothing new in kotoshu.gemspec; activemodel and jekyll are
39
+ dev-group test deps only):
40
+
41
+ 1. `Kotoshu::Validators::SpellingValidator` — ActiveModel
42
+ EachValidator; `validates :body, spelling: true` works after the
43
+ documented one-line top-level alias (ActiveModel resolves helper
44
+ validators from Object), `validates_with` works directly.
45
+ language/personal_words options; one error per misspelling with
46
+ the top suggestion. Specs load real ActiveModel standalone.
47
+ 2. `Kotoshu::Rspec::Matchers` — expect_words + all_be_spelled_correctly,
48
+ be_spelled_correctly (word or document), expect_document, with
49
+ suggestion-bearing failure messages.
50
+ 3. `Kotoshu::Tasks::CheckTask` — `require "kotoshu/tasks"` gives
51
+ `rake kotoshu:check` over the repo text files via the plan-88
52
+ selection (known extensions, .gitignore/.ignore, default skips);
53
+ DSL: files/language/fail_on_error/baseline/root.
54
+ 4. `Kotoshu::Jekyll::Generator` — safe generator checking posts and
55
+ drafts, failing the build on new errors, respecting
56
+ .kotoshu-baseline.json in the site source.
57
+
58
+ The rake task is require-only (defining a task on autoload would be
59
+ a surprise side effect); the others autoload from Kotoshu.
@@ -0,0 +1,44 @@
1
+ # 90 — Gem correctness residues found by the site flip
2
+
3
+ > Found 2026-09-05 while engine-verifying the 13 new specimen pages.
4
+ > The available-languages gate (the worst of the three) was fixed the
5
+ > same night in passing; the two below remain.
6
+
7
+ ## Fixed same-night (context)
8
+
9
+ `LanguageCache::AVAILABLE_LANGUAGES` hardcoded the original six, so
10
+ `kotoshu setup it` (and every other new module language) 404'd at the
11
+ pin — blocking the entire wave-2 module work at the last step. Now the
12
+ 19 module languages, with a comment explaining the module-boundary
13
+ rule (staged dictionaries join when their module lands).
14
+
15
+ ## Remaining
16
+
17
+ ### A — AffReader crashes on the Swedish aff
18
+
19
+ `Readers::AffReader` raises `RegexpError: unmatched close parenthesis`
20
+ on the staged Swedish `.aff` (dictionaries pin 1829a3e). Verified by
21
+ the site agent: the file loads fine under hunspell 1.7.2, so this is
22
+ our parser, not the data. Likely an unescaped `)` inside a condition
23
+ or MAP/BREAK pattern reaching `Regexp.new` without escaping. Fix the
24
+ escaping site (not the fixture); spec with the real sv aff slice.
25
+
26
+ ### B — Spellchecker word detection is ASCII-only
27
+
28
+ `Spellchecker#word_char?` (and whatever tokenizer feeds it) treats
29
+ only ASCII as word characters, so el/uk (and any non-Latin script)
30
+ words are not extracted on the CLI path — the site agent verified the
31
+ new Greek/Ukrainian specimens via dictionary lookup because the CLI
32
+ path could not see the words at all. Extend word-char detection to
33
+ Unicode letters (per-language script knowledge already exists in the
34
+ tokenizers); guard behavior with specs on el/uk/ja text and confirm no
35
+ regression on the ASCII languages (the whole conformance corpus is
36
+ ASCII — it must stay byte-identical).
37
+
38
+ ## Owner gates
39
+
40
+ None.
41
+
42
+ ## Status
43
+
44
+ **Pending.**
@@ -0,0 +1,47 @@
1
+ # 91 — Executes plan 90: Unicode word detection + the Swedish aff parser
2
+
3
+ ## Track A — Unicode word detection (the bigger one)
4
+
5
+ `Spellchecker#word_char?` (spellchecker.rb:313) accepts only ASCII
6
+ letters, so Greek/Ukrainian/any non-Latin script words are never
7
+ extracted on the CLI path — el/uk users cannot use the checker at all
8
+ (found by the site-flip agent; specimens could only be verified via
9
+ raw dictionary lookup).
10
+
11
+ - Extend word-char detection to Unicode letters. The per-language
12
+ tokenizers (wave-2 added Greek/Cyrillic ones) already know their
13
+ scripts — the fix should respect the language's tokenizer/script
14
+ rather than widening a global ASCII set (wrong-script words must
15
+ not suddenly become checkable).
16
+ - **Non-negotiable**: the frozen conformance corpus is ASCII — every
17
+ vector must stay byte-identical. Guard with the full conformance
18
+ run plus new el/uk/ja extraction specs.
19
+ - End-to-end verify: `echo "<greek typo>" | bundle exec exe/kotoshu
20
+ check -l el` flags it; same for uk; en/de regression-clean.
21
+
22
+ ## Track B — AffReader RegexpError on the Swedish aff
23
+
24
+ `Readers::AffReader` raises `RegexpError: unmatched close parenthesis`
25
+ on the staged Swedish .aff (pin 1829a3e); hunspell 1.7.2 loads it
26
+ fine. Find the unescaped-`)` site (likely condition/MAP/BREAK pattern
27
+ compilation) and fix the escaping — never the fixture. Spec with a
28
+ minimal slice of the real sv aff.
29
+
30
+ ## Track C — setup end-to-end spot-check
31
+
32
+ After PR #127's gate widening, run `kotoshu setup` + check for it,
33
+ el, tr, sv (sv will hit Track B until fixed — fix first). Record
34
+ which languages verified end-to-end through the CLI.
35
+
36
+ ## Status
37
+
38
+ **Implemented (2026-09-05, gem PR #130 + residue PR #131).** Track B:
39
+ COMPOUNDRULE )k - the sv aff uses ) as a flag char; flag atoms now
40
+ Regexp.escape'd (full 152k-word sv dictionary loads). Track A:
41
+ Tokenizer::Base#spellcheck_word_regex with Greek/Cyrillic/Latin
42
+ overrides, digits excluded, ASCII default byte-identical (2630 frozen
43
+ vectors 0 failures); plus the bundle-language-priority fix that made
44
+ `check -l el` extract zero words. Track C: remote setup for staged
45
+ languages was the LAST gap - PR #131 added the flat-layout fallback
46
+ (verified live: setup it + check -l it, probva -> prova). el/tr/sv/it
47
+ all verified end-to-end. Suite 3701/0/27.