kotoshu 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.kotoshu-baseline.json +1 -0
- data/README.adoc +30 -19
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/lib/kotoshu/baseline/store.rb +23 -0
- data/lib/kotoshu/cli/baseline_command.rb +49 -11
- data/lib/kotoshu/cli/directory_check.rb +10 -5
- data/lib/kotoshu/cli.rb +23 -5
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +16 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +3 -3
- data/lib/kotoshu/keyboard/registry.rb +9 -5
- data/lib/kotoshu/keyboard.rb +4 -0
- data/lib/kotoshu/languages/sr/language.rb +2 -1
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages.rb +4 -0
- data/lib/kotoshu/resource_manager.rb +12 -1
- data/lib/kotoshu/spellchecker.rb +14 -4
- data/lib/kotoshu/tasks/check_task.rb +9 -3
- data/lib/kotoshu/version.rb +1 -1
- data/lib/kotoshu.rb +2 -2
- metadata +11 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b024cec75cd870973f2993b7eac7b4dba110867803896bff14588513b9a75cb4
|
|
4
|
+
data.tar.gz: aa4a76db758b89a7ae14ac24f59905216be4a3fdc5e523a97c4c3275013da706
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e882b3899dedf60569ca598741be502d6606964616186de31e57ab3c2150e0886f1b445be7147167fc69c880eb603b9c5d9a1b67e20d1f5c801884a0f6083f19
|
|
7
|
+
data.tar.gz: 0ebadb70fffa35c2847268871ce0b030b9626989ad980ff8de02ab02bc53f62ee9f395b98c2e37d9ec4caab13bc91d7e1c5b69120cdc901c861c04d9dd02c91a
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":1,"entries":[{"file":"CHANGELOG.md","line":167,"word":"'","count":1},{"file":"CHANGELOG.md","line":789,"word":"'KOTOSHU","count":1},{"file":"CHANGELOG.md","line":530,"word":"'Multi","count":1},{"file":"CHANGELOG.md","line":488,"word":"'protocols","count":1},{"file":"CHANGELOG.md","line":261,"word":"'s","count":2},{"file":"CHANGELOG.md","line":528,"word":"AZERTY","count":1},{"file":"CHANGELOG.md","line":182,"word":"ActiveModel","count":2},{"file":"CHANGELOG.md","line":172,"word":"AffReader","count":3},{"file":"CHANGELOG.md","line":373,"word":"ArgumentError","count":2},{"file":"CHANGELOG.md","line":238,"word":"AsciiDoc","count":3},{"file":"CHANGELOG.md","line":777,"word":"Asciidoctor","count":1},{"file":"CHANGELOG.md","line":324,"word":"AuditLog","count":4},{"file":"CHANGELOG.md","line":658,"word":"AutoSetup","count":2},{"file":"CHANGELOG.md","line":69,"word":"BACKEND","count":2},{"file":"CHANGELOG.md","line":82,"word":"BDS","count":1},{"file":"CHANGELOG.md","line":137,"word":"BaseCache","count":6},{"file":"CHANGELOG.md","line":540,"word":"Behaviour","count":1},{"file":"CHANGELOG.md","line":389,"word":"BloomFilter","count":1},{"file":"CHANGELOG.md","line":282,"word":"CHECKCOMPOUNDPATTERN","count":2},{"file":"CHANGELOG.md","line":513,"word":"CHECKSHARPS","count":1},{"file":"CHANGELOG.md","line":510,"word":"CIRCUMFIX","count":1},{"file":"CHANGELOG.md","line":731,"word":"CJK","count":3},{"file":"CHANGELOG.md","line":165,"word":"CLI","count":21},{"file":"CHANGELOG.md","line":174,"word":"COMPOUNDRULE","count":2},{"file":"CHANGELOG.md","line":666,"word":"CONFIG","count":2},{"file":"CHANGELOG.md","line":391,"word":"CSpell","count":3},{"file":"CHANGELOG.md","line":555,"word":"CacheCommand","count":1},{"file":"CHANGELOG.md","line":1,"word":"Changelog","count":2},{"file":"CHANGELOG.md","line":197,"word":"CheckTask","count":1},{"file":"CHANGELOG.md","line":350,"word":"Cli","count":7},{"file":"CHANGELOG.md","line":149,"word":"CommonWordsLoader","count":1},{"file":"CHANGELOG.md","line":511,"word":"ConvTable","count":1},{"file":"CHANGELOG.md","line":51,"word":"DIC","count":1},{"file":"CHANGELOG.md","line":524,"word":"DRY'd","count":1},{"file":"CHANGELOG.md","line":108,"word":"Damerau","count":2},{"file":"CHANGELOG.md","line":297,"word":"DictionaryNotFoundError","count":1},{"file":"CHANGELOG.md","line":242,"word":"DocumentResult","count":2},{"file":"CHANGELOG.md","line":465,"word":"DoubleNegativeMatcher","count":1},{"file":"CHANGELOG.md","line":16,"word":"Dubeolsik","count":1},{"file":"CHANGELOG.md","line":145,"word":"ENV","count":3},{"file":"CHANGELOG.md","line":183,"word":"EachValidator","count":1},{"file":"CHANGELOG.md","line":113,"word":"EditDistance","count":2},{"file":"CHANGELOG.md","line":101,"word":"EditDistanceStrategy","count":2},{"file":"CHANGELOG.md","line":428,"word":"EmbeddingPipeline","count":1},{"file":"CHANGELOG.md","line":292,"word":"Embeddings","count":11},{"file":"CHANGELOG.md","line":573,"word":"EmptyLine","count":1},{"file":"CHANGELOG.md","line":309,"word":"EvictionPolicy","count":1},{"file":"CHANGELOG.md","line":572,"word":"ExtraSpacing","count":1},{"file":"CHANGELOG.md","line":628,"word":"FastText","count":4},{"file":"CHANGELOG.md","line":443,"word":"FrequencyCache","count":7},{"file":"CHANGELOG.md","line":141,"word":"FrequencyProvider","count":1},{"file":"CHANGELOG.md","line":441,"word":"FrozenError","count":1},{"file":"CHANGELOG.md","line":511,"word":"ICONV","count":3},{"file":"CHANGELOG.md","line":516,"word":"IJ","count":1},{"file":"CHANGELOG.md","line":15,"word":"IME","count":1},{"file":"CHANGELOG.md","line":283,"word":"INITCAP","count":1},{"file":"CHANGELOG.md","line":82,"word":"ISIRI","count":1},{"file":"CHANGELOG.md","line":21,"word":"InScript","count":1},{"file":"CHANGELOG.md","line":295,"word":"InferenceSession","count":1},{"file":"CHANGELOG.md","line":338,"word":"IntegrityError","count":3},{"file":"CHANGELOG.md","line":38,"word":"JCUKEN","count":4},{"file":"CHANGELOG.md","line":213,"word":"JSON","count":7},{"file":"CHANGELOG.md","line":513,"word":"KEEPCASE","count":1},{"file":"CHANGELOG.md","line":51,"word":"KOTOSHU","count":21},{"file":"CHANGELOG.md","line":48,"word":"Kotoshu","count":46},{"file":"CHANGELOG.md","line":777,"word":"Kramdown","count":1},{"file":"CHANGELOG.md","line":289,"word":"LFS","count":1},{"file":"CHANGELOG.md","line":311,"word":"LRU","count":1},{"file":"CHANGELOG.md","line":52,"word":"LSP","count":2},{"file":"CHANGELOG.md","line":30,"word":"LanguageCache","count":7},{"file":"CHANGELOG.md","line":745,"word":"LanguageResolver","count":1},{"file":"CHANGELOG.md","line":80,"word":"LatinBase","count":1},{"file":"CHANGELOG.md","line":113,"word":"Levenshtein","count":3},{"file":"CHANGELOG.md","line":71,"word":"LidModel","count":1},{"file":"CHANGELOG.md","line":489,"word":"LoadError","count":1},{"file":"CHANGELOG.md","line":393,"word":"LookupBuilder","count":1},{"file":"CHANGELOG.md","line":366,"word":"LruCache","count":1},{"file":"CHANGELOG.md","line":189,"word":"Matchers","count":1},{"file":"CHANGELOG.md","line":691,"word":"MiB","count":1},{"file":"CHANGELOG.md","line":334,"word":"ModelCache","count":2},{"file":"CHANGELOG.md","line":568,"word":"MultiWord","count":1},{"file":"CHANGELOG.md","line":139,"word":"NetHTTP","count":1},{"file":"CHANGELOG.md","line":429,"word":"NoMethodError","count":1},{"file":"CHANGELOG.md","line":436,"word":"NotImplementedError","count":1},{"file":"CHANGELOG.md","line":43,"word":"Nynorsk","count":1},{"file":"CHANGELOG.md","line":511,"word":"OCONV","count":3},{"file":"CHANGELOG.md","line":62,"word":"ONNX","count":15},{"file":"CHANGELOG.md","line":789,"word":"ONNX'","count":1},{"file":"CHANGELOG.md","line":474,"word":"OOV","count":4},{"file":"CHANGELOG.md","line":294,"word":"OnnxModel","count":3},{"file":"CHANGELOG.md","line":426,"word":"OnnxRuntimeModel","count":2},{"file":"CHANGELOG.md","line":705,"word":"OnnxUnavailable","count":1},{"file":"CHANGELOG.md","line":348,"word":"PersonalDictionary","count":1},{"file":"CHANGELOG.md","line":769,"word":"Phonet","count":1},{"file":"CHANGELOG.md","line":459,"word":"PhonetSuggest","count":2},{"file":"CHANGELOG.md","line":458,"word":"PhonetTable","count":1},{"file":"CHANGELOG.md","line":122,"word":"PlainText","count":5},{"file":"CHANGELOG.md","line":689,"word":"ProgressReporter","count":2},{"file":"CHANGELOG.md","line":83,"word":"QWERTZ","count":2},{"file":"CHANGELOG.md","line":257,"word":"README","count":3},{"file":"CHANGELOG.md","line":662,"word":"REPOS","count":1},{"file":"CHANGELOG.md","line":188,"word":"RSpec","count":1},{"file":"CHANGELOG.md","line":78,"word":"RTL","count":7},{"file":"CHANGELOG.md","line":173,"word":"RegexpError","count":1},{"file":"CHANGELOG.md","line":743,"word":"ResourceBundle","count":1},{"file":"CHANGELOG.md","line":141,"word":"ResourceManager","count":6},{"file":"CHANGELOG.md","line":139,"word":"ResourceNotCachedError","count":1},{"file":"CHANGELOG.md","line":656,"word":"ResourceNotSetupError","count":3},{"file":"CHANGELOG.md","line":147,"word":"ResourceUnavailable","count":1},{"file":"CHANGELOG.md","line":752,"word":"Ribose","count":2},{"file":"CHANGELOG.md","line":23,"word":"Romanized","count":1},{"file":"CHANGELOG.md","line":319,"word":"RotationPolicy","count":1},{"file":"CHANGELOG.md","line":189,"word":"Rspec","count":1},{"file":"CHANGELOG.md","line":570,"word":"RuboCop","count":1},{"file":"CHANGELOG.md","line":213,"word":"SARIF","count":5},{"file":"CHANGELOG.md","line":335,"word":"SHA","count":2},{"file":"CHANGELOG.md","line":347,"word":"SUBCOMMAND","count":3},{"file":"CHANGELOG.md","line":474,"word":"SemanticAnalyzer","count":1},{"file":"CHANGELOG.md","line":385,"word":"SemanticStrategy","count":3},{"file":"CHANGELOG.md","line":744,"word":"SetupResult","count":1},{"file":"CHANGELOG.md","line":370,"word":"SimilarityEngine","count":1},{"file":"CHANGELOG.md","line":601,"word":"SimilaritySearch","count":2},{"file":"CHANGELOG.md","line":659,"word":"SourceRegistry","count":2},{"file":"CHANGELOG.md","line":182,"word":"SpellingValidator","count":1},{"file":"CHANGELOG.md","line":31,"word":"StagedLanguages","count":1},{"file":"CHANGELOG.md","line":745,"word":"StatusReport","count":1},{"file":"CHANGELOG.md","line":598,"word":"StringReader","count":1},{"file":"CHANGELOG.md","line":594,"word":"SuggestionSet","count":2},{"file":"CHANGELOG.md","line":239,"word":"Suppressions","count":1},{"file":"CHANGELOG.md","line":624,"word":"SymSpell","count":1},{"file":"CHANGELOG.md","line":522,"word":"TODO","count":21},{"file":"CHANGELOG.md","line":737,"word":"TODOs","count":1},{"file":"CHANGELOG.md","line":773,"word":"TTLs","count":1},{"file":"CHANGELOG.md","line":686,"word":"TTY","count":3},{"file":"CHANGELOG.md","line":103,"word":"Teh","count":1},{"file":"CHANGELOG.md","line":440,"word":"Trie","count":1},{"file":"CHANGELOG.md","line":359,"word":"TypeError","count":2},{"file":"CHANGELOG.md","line":123,"word":"UnixWords","count":3},{"file":"CHANGELOG.md","line":350,"word":"UsageError","count":1},{"file":"CHANGELOG.md","line":182,"word":"Validators","count":1},{"file":"CHANGELOG.md","line":592,"word":"WordResult","count":3},{"file":"CHANGELOG.md","line":664,"word":"XDG","count":5},{"file":"CHANGELOG.md","line":564,"word":"YAML","count":1},{"file":"CHANGELOG.md","line":223,"word":"ZM","count":1},{"file":"CHANGELOG.md","line":512,"word":"ZWJ","count":1},{"file":"CHANGELOG.md","line":97,"word":"ZWNJ","count":2},{"file":"CHANGELOG.md","line":205,"word":"adoc","count":1},{"file":"CHANGELOG.md","line":172,"word":"aff","count":4},{"file":"CHANGELOG.md","line":78,"word":"ar","count":3},{"file":"CHANGELOG.md","line":394,"word":"args","count":1},{"file":"CHANGELOG.md","line":205,"word":"asciidoc","count":1},{"file":"CHANGELOG.md","line":371,"word":"attr","count":3},{"file":"CHANGELOG.md","line":359,"word":"autoload","count":7},{"file":"CHANGELOG.md","line":495,"word":"autoloads","count":1},{"file":"CHANGELOG.md","line":551,"word":"backend's","count":1},{"file":"CHANGELOG.md","line":767,"word":"backends","count":1},{"file":"CHANGELOG.md","line":202,"word":"baselined","count":1},{"file":"CHANGELOG.md","line":451,"word":"basename","count":1},{"file":"CHANGELOG.md","line":551,"word":"behaviour","count":1},{"file":"CHANGELOG.md","line":78,"word":"bg","count":2},{"file":"CHANGELOG.md","line":517,"word":"breakdefault","count":1},{"file":"CHANGELOG.md","line":296,"word":"camelCase","count":1},{"file":"CHANGELOG.md","line":666,"word":"cfg","count":1},{"file":"CHANGELOG.md","line":804,"word":"checksummed","count":1},{"file":"CHANGELOG.md","line":729,"word":"circumfix","count":2},{"file":"CHANGELOG.md","line":733,"word":"cjk","count":2},{"file":"CHANGELOG.md","line":697,"word":"cli","count":2},{"file":"CHANGELOG.md","line":51,"word":"config","count":5},{"file":"CHANGELOG.md","line":785,"word":"csv","count":1},{"file":"CHANGELOG.md","line":220,"word":"da","count":1},{"file":"CHANGELOG.md","line":20,"word":"danda","count":1},{"file":"CHANGELOG.md","line":170,"word":"de","count":3},{"file":"CHANGELOG.md","line":111,"word":"deduplicated","count":1},{"file":"CHANGELOG.md","line":104,"word":"definately","count":1},{"file":"CHANGELOG.md","line":51,"word":"dic","count":3},{"file":"CHANGELOG.md","line":411,"word":"docstring","count":1},{"file":"CHANGELOG.md","line":222,"word":"dotless","count":3},{"file":"CHANGELOG.md","line":565,"word":"ecke","count":1},{"file":"CHANGELOG.md","line":166,"word":"el","count":2},{"file":"CHANGELOG.md","line":487,"word":"embeddings","count":4},{"file":"CHANGELOG.md","line":14,"word":"eojeol","count":1},{"file":"CHANGELOG.md","line":683,"word":"errorCount","count":1},{"file":"CHANGELOG.md","line":514,"word":"eszet","count":1},{"file":"CHANGELOG.md","line":78,"word":"et","count":2},{"file":"CHANGELOG.md","line":22,"word":"eval","count":7},{"file":"CHANGELOG.md","line":788,"word":"exe","count":1},{"file":"CHANGELOG.md","line":45,"word":"fallbacks","count":1},{"file":"CHANGELOG.md","line":63,"word":"fastText","count":1},{"file":"CHANGELOG.md","line":661,"word":"fasttext","count":3},{"file":"CHANGELOG.md","line":517,"word":"forbiddenword","count":1},{"file":"CHANGELOG.md","line":18,"word":"fortis","count":1},{"file":"CHANGELOG.md","line":70,"word":"gemspec","count":2},{"file":"CHANGELOG.md","line":98,"word":"geresh","count":1},{"file":"CHANGELOG.md","line":98,"word":"gershayim","count":1},{"file":"CHANGELOG.md","line":196,"word":"gitignore","count":2},{"file":"CHANGELOG.md","line":316,"word":"hardcoded","count":1},{"file":"CHANGELOG.md","line":190,"word":"helo","count":1},{"file":"CHANGELOG.md","line":519,"word":"honoured","count":3},{"file":"CHANGELOG.md","line":6,"word":"html","count":1},{"file":"CHANGELOG.md","line":5,"word":"https","count":2},{"file":"CHANGELOG.md","line":220,"word":"hu","count":1},{"file":"CHANGELOG.md","line":637,"word":"hunspell","count":3},{"file":"CHANGELOG.md","line":522,"word":"impl","count":21},{"file":"CHANGELOG.md","line":530,"word":"implemented'","count":1},{"file":"CHANGELOG.md","line":246,"word":"init","count":1},{"file":"CHANGELOG.md","line":566,"word":"instantiation","count":1},{"file":"CHANGELOG.md","line":284,"word":"integrational","count":1},{"file":"CHANGELOG.md","line":179,"word":"integrations","count":1},{"file":"CHANGELOG.md","line":392,"word":"ivars","count":1},{"file":"CHANGELOG.md","line":223,"word":"izmir","count":1},{"file":"CHANGELOG.md","line":797,"word":"ja","count":1},{"file":"CHANGELOG.md","line":14,"word":"jamo","count":2},{"file":"CHANGELOG.md","line":157,"word":"jekyll","count":1},{"file":"CHANGELOG.md","line":201,"word":"json","count":9},{"file":"CHANGELOG.md","line":5,"word":"keepachangelog","count":1},{"file":"CHANGELOG.md","line":661,"word":"kelly","count":2},{"file":"CHANGELOG.md","line":405,"word":"klass","count":5},{"file":"CHANGELOG.md","line":13,"word":"ko","count":2},{"file":"CHANGELOG.md","line":25,"word":"kotoshu","count":80},{"file":"CHANGELOG.md","line":457,"word":"kwarg","count":1},{"file":"CHANGELOG.md","line":361,"word":"kwargs","count":2},{"file":"CHANGELOG.md","line":154,"word":"lang","count":4},{"file":"CHANGELOG.md","line":18,"word":"lenis","count":1},{"file":"CHANGELOG.md","line":326,"word":"lockfile","count":1},{"file":"CHANGELOG.md","line":78,"word":"lt","count":2},{"file":"CHANGELOG.md","line":248,"word":"lutaml","count":5},{"file":"CHANGELOG.md","line":78,"word":"lv","count":2},{"file":"CHANGELOG.md","line":105,"word":"ly","count":1},{"file":"CHANGELOG.md","line":70,"word":"magnus","count":1},{"file":"CHANGELOG.md","line":469,"word":"matcher","count":1},{"file":"CHANGELOG.md","line":188,"word":"matchers","count":1},{"file":"CHANGELOG.md","line":19,"word":"matras","count":1},{"file":"CHANGELOG.md","line":159,"word":"md","count":23},{"file":"CHANGELOG.md","line":205,"word":"mdx","count":1},{"file":"CHANGELOG.md","line":462,"word":"metaphone","count":1},{"file":"CHANGELOG.md","line":488,"word":"model'","count":1},{"file":"CHANGELOG.md","line":56,"word":"mtime","count":1},{"file":"CHANGELOG.md","line":677,"word":"mtimes","count":1},{"file":"CHANGELOG.md","line":302,"word":"namespace","count":5},{"file":"CHANGELOG.md","line":538,"word":"namespace's","count":1},{"file":"CHANGELOG.md","line":352,"word":"namespaces","count":2},{"file":"CHANGELOG.md","line":18,"word":"ne","count":2},{"file":"CHANGELOG.md","line":235,"word":"nestable","count":1},{"file":"CHANGELOG.md","line":283,"word":"ngram","count":3},{"file":"CHANGELOG.md","line":220,"word":"nl","count":1},{"file":"CHANGELOG.md","line":43,"word":"nn","count":1},{"file":"CHANGELOG.md","line":221,"word":"normalizer","count":3},{"file":"CHANGELOG.md","line":508,"word":"num","count":1},{"file":"CHANGELOG.md","line":582,"word":"onnx","count":6},{"file":"CHANGELOG.md","line":70,"word":"onnxruntime","count":8},{"file":"CHANGELOG.md","line":381,"word":"op'ing","count":1},{"file":"CHANGELOG.md","line":517,"word":"opentaal","count":1},{"file":"CHANGELOG.md","line":501,"word":"phonet","count":2},{"file":"CHANGELOG.md","line":445,"word":"positionally","count":1},{"file":"CHANGELOG.md","line":132,"word":"pre","count":6},{"file":"CHANGELOG.md","line":615,"word":"queryable","count":1},{"file":"CHANGELOG.md","line":716,"word":"quickstart","count":1},{"file":"CHANGELOG.md","line":231,"word":"qwertz","count":1},{"file":"CHANGELOG.md","line":229,"word":"rb","count":12},{"file":"CHANGELOG.md","line":22,"word":"repo","count":9},{"file":"CHANGELOG.md","line":267,"word":"repo's","count":1},{"file":"CHANGELOG.md","line":659,"word":"repos'","count":1},{"file":"CHANGELOG.md","line":280,"word":"rerank","count":1},{"file":"CHANGELOG.md","line":758,"word":"reranking","count":1},{"file":"CHANGELOG.md","line":220,"word":"ro","count":1},{"file":"CHANGELOG.md","line":261,"word":"roadmap","count":1},{"file":"CHANGELOG.md","line":188,"word":"rspec","count":2},{"file":"CHANGELOG.md","line":205,"word":"rst","count":1},{"file":"CHANGELOG.md","line":733,"word":"rtl","count":2},{"file":"CHANGELOG.md","line":798,"word":"ru","count":1},{"file":"CHANGELOG.md","line":570,"word":"rubocop","count":1},{"file":"CHANGELOG.md","line":681,"word":"sarif","count":2},{"file":"CHANGELOG.md","line":6,"word":"semver","count":1},{"file":"CHANGELOG.md","line":65,"word":"sha","count":3},{"file":"CHANGELOG.md","line":78,"word":"sk","count":2},{"file":"CHANGELOG.md","line":78,"word":"sl","count":2},{"file":"CHANGELOG.md","line":55,"word":"spellcheckable","count":1},{"file":"CHANGELOG.md","line":78,"word":"sr","count":2},{"file":"CHANGELOG.md","line":514,"word":"ss","count":1},{"file":"CHANGELOG.md","line":223,"word":"stanbul","count":1},{"file":"CHANGELOG.md","line":216,"word":"stdin","count":1},{"file":"CHANGELOG.md","line":401,"word":"subclasses","count":1},{"file":"CHANGELOG.md","line":348,"word":"subcommand","count":1},{"file":"CHANGELOG.md","line":383,"word":"subcommand's","count":1},{"file":"CHANGELOG.md","line":330,"word":"subcommands","count":1},{"file":"CHANGELOG.md","line":155,"word":"sublayout","count":1},{"file":"CHANGELOG.md","line":181,"word":"suika","count":2},{"file":"CHANGELOG.md","line":120,"word":"suleman","count":4},{"file":"CHANGELOG.md","line":676,"word":"summarises","count":1},{"file":"CHANGELOG.md","line":245,"word":"suppressedCount","count":1},{"file":"CHANGELOG.md","line":245,"word":"suppressedErrors","count":1},{"file":"CHANGELOG.md","line":251,"word":"suppressions","count":1},{"file":"CHANGELOG.md","line":220,"word":"sv","count":1},{"file":"CHANGELOG.md","line":624,"word":"symspell","count":2},{"file":"CHANGELOG.md","line":448,"word":"tmp","count":3},{"file":"CHANGELOG.md","line":355,"word":"tmpdir","count":1},{"file":"CHANGELOG.md","line":526,"word":"tokenization","count":1},{"file":"CHANGELOG.md","line":13,"word":"tokenizer","count":10},{"file":"CHANGELOG.md","line":36,"word":"tokenizers","count":4},{"file":"CHANGELOG.md","line":722,"word":"toolchain","count":1},{"file":"CHANGELOG.md","line":440,"word":"trie","count":1},{"file":"CHANGELOG.md","line":205,"word":"txt","count":1},{"file":"CHANGELOG.md","line":167,"word":"uk","count":2},{"file":"CHANGELOG.md","line":169,"word":"uncheckable","count":1},{"file":"CHANGELOG.md","line":683,"word":"uniqueErrorCount","count":1},{"file":"CHANGELOG.md","line":138,"word":"url","count":1},{"file":"CHANGELOG.md","line":768,"word":"usr","count":1},{"file":"CHANGELOG.md","line":508,"word":"utf","count":1},{"file":"CHANGELOG.md","line":120,"word":"uzair","count":4},{"file":"CHANGELOG.md","line":30,"word":"vendored","count":1},{"file":"CHANGELOG.md","line":19,"word":"virama","count":1},{"file":"CHANGELOG.md","line":683,"word":"wordCount","count":1},{"file":"CHANGELOG.md","line":254,"word":"yaml","count":3},{"file":"CHANGELOG.md","line":564,"word":"yml","count":1},{"file":"CHANGELOG.md","line":329,"word":"zsh","count":2},{"file":"CODE_OF_CONDUCT.md","line":122,"word":"CoC","count":2},{"file":"CODE_OF_CONDUCT.md","line":125,"word":"faq","count":2},{"file":"CODE_OF_CONDUCT.md","line":130,"word":"github","count":1},{"file":"CODE_OF_CONDUCT.md","line":119,"word":"html","count":2},{"file":"CODE_OF_CONDUCT.md","line":119,"word":"https","count":8},{"file":"CODE_OF_CONDUCT.md","line":130,"word":"mozilla","count":1},{"file":"CODE_OF_CONDUCT.md","line":34,"word":"others'","count":1},{"file":"CODE_OF_CONDUCT.md","line":30,"word":"sexualized","count":1},{"file":"CODE_OF_CONDUCT.md","line":8,"word":"socio","count":1},{"file":"CODE_OF_CONDUCT.md","line":119,"word":"www","count":7},{"file":"SECURITY.md","line":79,"word":"'HOME'","count":1},{"file":"SECURITY.md","line":79,"word":"AuditLog","count":1},{"file":"SECURITY.md","line":87,"word":"CLI","count":1},{"file":"SECURITY.md","line":53,"word":"CLI's","count":1},{"file":"SECURITY.md","line":79,"word":"ENV","count":1},{"file":"SECURITY.md","line":22,"word":"FastText","count":1},{"file":"SECURITY.md","line":24,"word":"HTTPS","count":1},{"file":"SECURITY.md","line":37,"word":"IntegrityError","count":1},{"file":"SECURITY.md","line":41,"word":"JSON","count":2},{"file":"SECURITY.md","line":60,"word":"KOTOSHU","count":2},{"file":"SECURITY.md","line":15,"word":"Kotoshu","count":5},{"file":"SECURITY.md","line":22,"word":"ONNX","count":1},{"file":"SECURITY.md","line":86,"word":"ResourceNotCachedError","count":1},{"file":"SECURITY.md","line":31,"word":"SHA","count":2},{"file":"SECURITY.md","line":6,"word":"acknowledgement","count":1},{"file":"SECURITY.md","line":22,"word":"fasttext","count":1},{"file":"SECURITY.md","line":24,"word":"githubusercontent","count":1},{"file":"SECURITY.md","line":72,"word":"jq","count":1},{"file":"SECURITY.md","line":30,"word":"json","count":1},{"file":"SECURITY.md","line":21,"word":"kelly","count":1},{"file":"SECURITY.md","line":5,"word":"kotoshu","count":14},{"file":"SECURITY.md","line":53,"word":"lang","count":1},{"file":"SECURITY.md","line":68,"word":"logrotate","count":1},{"file":"SECURITY.md","line":22,"word":"onnx","count":1},{"file":"SECURITY.md","line":93,"word":"onnxruntime","count":1},{"file":"SECURITY.md","line":87,"word":"pre","count":1},{"file":"SECURITY.md","line":18,"word":"repo","count":2},{"file":"SECURITY.md","line":26,"word":"repos","count":2},{"file":"SECURITY.md","line":93,"word":"suika","count":1},{"file":"SECURITY.md","line":61,"word":"umask","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"ActiveModelUnavailable","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"AffixRuleError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":46,"word":"Autoloaded","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":69,"word":"CLI","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":45,"word":"CONFIG","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":91,"word":"CacheCommand","count":4},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"CheckCommand","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":57,"word":"CheckTask","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":69,"word":"Cli","count":10},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"ConfigurationError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":67,"word":"ConformanceExporter","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":66,"word":"ConformanceRunner","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":48,"word":"DebugLogger","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":89,"word":"DictCommand","count":6},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"DictionaryNotFoundError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":120,"word":"DocumentResult","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":40,"word":"ENV","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":49,"word":"FluentChecker","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"IntegrityError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"InvalidDictionaryFormatError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":221,"word":"JS","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":78,"word":"JSON","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":58,"word":"JekyllUnavailable","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":40,"word":"KOTOSHU","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":38,"word":"Kotoshu","count":50},{"file":"docs/STABILITY-1.0-AUDIT.md","line":110,"word":"KotoshuLid","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"KotoshuModel","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":106,"word":"KotoshuWasm","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":47,"word":"LanguageCache","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":47,"word":"LanguageIdentifier","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":56,"word":"Matcher","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":56,"word":"Matchers","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":48,"word":"MetricsCollector","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":47,"word":"ModelCache","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":92,"word":"ModelCommand","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":49,"word":"MultiLanguageChecker","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"NameError","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":46,"word":"Namespaces","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":100,"word":"NativeBackend","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":141,"word":"PRs","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":223,"word":"PyPI","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":227,"word":"RBS","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":23,"word":"README","count":10},{"file":"docs/STABILITY-1.0-AUDIT.md","line":9,"word":"Reachability","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":43,"word":"ResourceBundle","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":42,"word":"ResourceManager","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"ResourceNotCachedError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"ResourceNotSetupError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"ResourceResolutionError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":56,"word":"Rspec","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":78,"word":"SARIF","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":221,"word":"SDK","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":47,"word":"SemanticAnalyzer","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":41,"word":"SourceRegistry","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"SpellChecker","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"SpellcheckError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"SpellingValidator","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"SuikaUnavailable","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":154,"word":"Unwired","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"Validators","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":120,"word":"WordError","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":45,"word":"XDG","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":29,"word":"accessor","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":40,"word":"accessors","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":248,"word":"activemodel","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":137,"word":"additionalProperties","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":23,"word":"adoc","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":76,"word":"aff","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":106,"word":"affSource","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"arg","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":41,"word":"attrs","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"autoload","count":9},{"file":"docs/STABILITY-1.0-AUDIT.md","line":44,"word":"autoloaded","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":174,"word":"autoloads","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":100,"word":"backend","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"bucketsBytes","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":186,"word":"changelog","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":73,"word":"cli","count":21},{"file":"docs/STABILITY-1.0-AUDIT.md","line":38,"word":"config","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":135,"word":"const","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"csv","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":39,"word":"deprecations","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":51,"word":"deps","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":110,"word":"detectLanguage","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":76,"word":"dic","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":106,"word":"dicSource","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":197,"word":"docstrings","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":249,"word":"envs","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":10,"word":"exe","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":98,"word":"extconf","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":6,"word":"fac","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":92,"word":"fasttext","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":65,"word":"ffi","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":91,"word":"gemspec","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":208,"word":"gitignored","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"hunspell","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":81,"word":"init","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":191,"word":"inlined","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":51,"word":"integrations","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":58,"word":"jekyll","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":221,"word":"js","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":74,"word":"json","count":8},{"file":"docs/STABILITY-1.0-AUDIT.md","line":64,"word":"jsonl","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":6,"word":"kotoshu","count":119},{"file":"docs/STABILITY-1.0-AUDIT.md","line":38,"word":"kwargs","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":136,"word":"lang","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":177,"word":"lexically","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":110,"word":"loadLid","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"loadModel","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":7,"word":"md","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"modelBytes","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":29,"word":"namespace","count":6},{"file":"docs/STABILITY-1.0-AUDIT.md","line":46,"word":"namespaces","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":112,"word":"npm","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":92,"word":"onnx","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":110,"word":"onnxBytes","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":248,"word":"onnxruntime","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":118,"word":"openapi","count":6},{"file":"docs/STABILITY-1.0-AUDIT.md","line":127,"word":"prewarm","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":92,"word":"py","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":6,"word":"rb","count":98},{"file":"docs/STABILITY-1.0-AUDIT.md","line":227,"word":"rbs","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":164,"word":"repo","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":41,"word":"repos","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":108,"word":"rerank","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":56,"word":"rspec","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":74,"word":"sarif","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":120,"word":"schemas","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":109,"word":"semanticSuggest","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":137,"word":"sha","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":176,"word":"sig","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":106,"word":"src","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"subcommand","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":79,"word":"subcommands","count":3},{"file":"docs/STABILITY-1.0-AUDIT.md","line":248,"word":"suika","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":38,"word":"tokenize","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":30,"word":"trie","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":92,"word":"untracked","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":41,"word":"url","count":5},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"validator","count":1},{"file":"docs/STABILITY-1.0-AUDIT.md","line":55,"word":"validators","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":107,"word":"vocabBytes","count":2},{"file":"docs/STABILITY-1.0-AUDIT.md","line":65,"word":"wasm","count":19},{"file":"docs/STABILITY-1.0-AUDIT.md","line":90,"word":"yaml","count":8},{"file":"docs/STABILITY-1.0-AUDIT.md","line":77,"word":"yml","count":4},{"file":"docs/STABILITY-1.0-AUDIT.md","line":82,"word":"zsh","count":1},{"file":"docs/STABILITY-1.0.md","line":102,"word":"CHANGELOG","count":1},{"file":"docs/STABILITY-1.0.md","line":22,"word":"CLI","count":3},{"file":"docs/STABILITY-1.0.md","line":22,"word":"ENV","count":3},{"file":"docs/STABILITY-1.0.md","line":46,"word":"JS","count":1},{"file":"docs/STABILITY-1.0.md","line":31,"word":"JSON","count":2},{"file":"docs/STABILITY-1.0.md","line":121,"word":"KOTOSHU","count":1},{"file":"docs/STABILITY-1.0.md","line":1,"word":"Kotoshu","count":1},{"file":"docs/STABILITY-1.0.md","line":49,"word":"KotoshuLid","count":1},{"file":"docs/STABILITY-1.0.md","line":49,"word":"KotoshuModel","count":1},{"file":"docs/STABILITY-1.0.md","line":47,"word":"KotoshuWasm","count":2},{"file":"docs/STABILITY-1.0.md","line":31,"word":"SARIF","count":1},{"file":"docs/STABILITY-1.0.md","line":9,"word":"Semver","count":1},{"file":"docs/STABILITY-1.0.md","line":117,"word":"autoload","count":1},{"file":"docs/STABILITY-1.0.md","line":116,"word":"autoloads","count":1},{"file":"docs/STABILITY-1.0.md","line":54,"word":"bucketsBytes","count":1},{"file":"docs/STABILITY-1.0.md","line":122,"word":"changelog","count":1},{"file":"docs/STABILITY-1.0.md","line":31,"word":"cli","count":2},{"file":"docs/STABILITY-1.0.md","line":25,"word":"config","count":2},{"file":"docs/STABILITY-1.0.md","line":48,"word":"detectLanguage","count":1},{"file":"docs/STABILITY-1.0.md","line":82,"word":"fasttext","count":1},{"file":"docs/STABILITY-1.0.md","line":44,"word":"ffi","count":2},{"file":"docs/STABILITY-1.0.md","line":37,"word":"gemspec","count":1},{"file":"docs/STABILITY-1.0.md","line":43,"word":"json","count":3},{"file":"docs/STABILITY-1.0.md","line":23,"word":"jsonl","count":2},{"file":"docs/STABILITY-1.0.md","line":11,"word":"kotoshu","count":26},{"file":"docs/STABILITY-1.0.md","line":48,"word":"loadLid","count":1},{"file":"docs/STABILITY-1.0.md","line":48,"word":"loadModel","count":2},{"file":"docs/STABILITY-1.0.md","line":5,"word":"md","count":2},{"file":"docs/STABILITY-1.0.md","line":82,"word":"onnx","count":1},{"file":"docs/STABILITY-1.0.md","line":62,"word":"openapi","count":2},{"file":"docs/STABILITY-1.0.md","line":24,"word":"rb","count":11},{"file":"docs/STABILITY-1.0.md","line":104,"word":"repo","count":1},{"file":"docs/STABILITY-1.0.md","line":48,"word":"rerank","count":1},{"file":"docs/STABILITY-1.0.md","line":77,"word":"sarif","count":1},{"file":"docs/STABILITY-1.0.md","line":85,"word":"schemas","count":1},{"file":"docs/STABILITY-1.0.md","line":48,"word":"semanticSuggest","count":1},{"file":"docs/STABILITY-1.0.md","line":44,"word":"src","count":2},{"file":"docs/STABILITY-1.0.md","line":119,"word":"stderr","count":1},{"file":"docs/STABILITY-1.0.md","line":27,"word":"subclasses","count":1},{"file":"docs/STABILITY-1.0.md","line":110,"word":"url","count":3},{"file":"docs/STABILITY-1.0.md","line":41,"word":"wasm","count":5},{"file":"docs/STABILITY-1.0.md","line":62,"word":"yaml","count":2},{"file":"docs/STABILITY-1.0.md","line":76,"word":"yml","count":3},{"file":"docs/cache-architecture.md","line":378,"word":"'","count":1},{"file":"docs/cache-architecture.md","line":428,"word":"'Download","count":1},{"file":"docs/cache-architecture.md","line":428,"word":"'download","count":1},{"file":"docs/cache-architecture.md","line":188,"word":"'en","count":10},{"file":"docs/cache-architecture.md","line":433,"word":"'mc'","count":2},{"file":"docs/cache-architecture.md","line":433,"word":"'mycache'","count":2},{"file":"docs/cache-architecture.md","line":389,"word":"'wb'","count":1},{"file":"docs/cache-architecture.md","line":10,"word":"BaseCache","count":8},{"file":"docs/cache-architecture.md","line":364,"word":"CDN","count":2},{"file":"docs/cache-architecture.md","line":46,"word":"CLI","count":4},{"file":"docs/cache-architecture.md","line":40,"word":"EditDistance","count":1},{"file":"docs/cache-architecture.md","line":54,"word":"Embeddings","count":1},{"file":"docs/cache-architecture.md","line":128,"word":"FastText","count":7},{"file":"docs/cache-architecture.md","line":27,"word":"FrequencyCache","count":5},{"file":"docs/cache-architecture.md","line":428,"word":"GitHub'","count":1},{"file":"docs/cache-architecture.md","line":393,"word":"GzipReader","count":1},{"file":"docs/cache-architecture.md","line":180,"word":"IntegrityError","count":1},{"file":"docs/cache-architecture.md","line":246,"word":"JSON","count":5},{"file":"docs/cache-architecture.md","line":170,"word":"KOTOSHU","count":1},{"file":"docs/cache-architecture.md","line":1,"word":"Kotoshu","count":7},{"file":"docs/cache-architecture.md","line":25,"word":"LanguageCache","count":4},{"file":"docs/cache-architecture.md","line":183,"word":"Lifecycle","count":1},{"file":"docs/cache-architecture.md","line":26,"word":"ModelCache","count":7},{"file":"docs/cache-architecture.md","line":314,"word":"MyCache","count":2},{"file":"docs/cache-architecture.md","line":131,"word":"ONNX","count":3},{"file":"docs/cache-architecture.md","line":17,"word":"OOP","count":1},{"file":"docs/cache-architecture.md","line":279,"word":"ParserError","count":1},{"file":"docs/cache-architecture.md","line":500,"word":"Pre","count":1},{"file":"docs/cache-architecture.md","line":428,"word":"RESOURCE'","count":1},{"file":"docs/cache-architecture.md","line":457,"word":"RSpec","count":1},{"file":"docs/cache-architecture.md","line":172,"word":"ResourceNotSetupError","count":1},{"file":"docs/cache-architecture.md","line":278,"word":"RuntimeError","count":1},{"file":"docs/cache-architecture.md","line":166,"word":"SHA","count":4},{"file":"docs/cache-architecture.md","line":40,"word":"SpellChecker","count":1},{"file":"docs/cache-architecture.md","line":5,"word":"TTL","count":6},{"file":"docs/cache-architecture.md","line":388,"word":"URI","count":1},{"file":"docs/cache-architecture.md","line":393,"word":"Zlib","count":1},{"file":"docs/cache-architecture.md","line":115,"word":"aff","count":3},{"file":"docs/cache-architecture.md","line":117,"word":"ar","count":1},{"file":"docs/cache-architecture.md","line":501,"word":"backends","count":1},{"file":"docs/cache-architecture.md","line":166,"word":"bytes'","count":2},{"file":"docs/cache-architecture.md","line":425,"word":"cli","count":1},{"file":"docs/cache-architecture.md","line":118,"word":"de","count":3},{"file":"docs/cache-architecture.md","line":428,"word":"desc","count":1},{"file":"docs/cache-architecture.md","line":12,"word":"deserialization","count":1},{"file":"docs/cache-architecture.md","line":76,"word":"dest","count":12},{"file":"docs/cache-architecture.md","line":114,"word":"dic","count":3},{"file":"docs/cache-architecture.md","line":79,"word":"dir","count":2},{"file":"docs/cache-architecture.md","line":365,"word":"dl","count":3},{"file":"docs/cache-architecture.md","line":56,"word":"fasttext","count":15},{"file":"docs/cache-architecture.md","line":365,"word":"fbaipublicfiles","count":3},{"file":"docs/cache-architecture.md","line":251,"word":"githubusercontent","count":1},{"file":"docs/cache-architecture.md","line":386,"word":"gz","count":9},{"file":"docs/cache-architecture.md","line":378,"word":"gz'","count":1},{"file":"docs/cache-architecture.md","line":377,"word":"gzip","count":2},{"file":"docs/cache-architecture.md","line":406,"word":"gzipped","count":1},{"file":"docs/cache-architecture.md","line":251,"word":"https","count":4},{"file":"docs/cache-architecture.md","line":113,"word":"json","count":14},{"file":"docs/cache-architecture.md","line":263,"word":"kelly","count":1},{"file":"docs/cache-architecture.md","line":46,"word":"kotoshu","count":14},{"file":"docs/cache-architecture.md","line":136,"word":"lang","count":3},{"file":"docs/cache-architecture.md","line":501,"word":"memcached","count":1},{"file":"docs/cache-architecture.md","line":434,"word":"mycache","count":2},{"file":"docs/cache-architecture.md","line":57,"word":"onnx","count":14},{"file":"docs/cache-architecture.md","line":425,"word":"rb","count":5},{"file":"docs/cache-architecture.md","line":117,"word":"ru","count":3},{"file":"docs/cache-architecture.md","line":188,"word":"spelling'","count":10},{"file":"docs/cache-architecture.md","line":75,"word":"subclasses","count":2},{"file":"docs/cache-architecture.md","line":101,"word":"ttl","count":3},{"file":"docs/cache-architecture.md","line":168,"word":"unparsable","count":1},{"file":"docs/cache-architecture.md","line":388,"word":"uri","count":2},{"file":"docs/cache-architecture.md","line":86,"word":"url","count":14},{"file":"docs/cache-architecture.md","line":179,"word":"urls","count":2},{"file":"docs/cache-architecture.md","line":406,"word":"vec","count":2},{"file":"docs/verification-evidence/correct3/pr_body.md","line":12,"word":"'","count":4},{"file":"docs/verification-evidence/correct3/pr_body.md","line":19,"word":"AffReader","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":26,"word":"CLI","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":19,"word":"COMPOUNDRULE","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":19,"word":"CompoundRule","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":37,"word":"RBS","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":17,"word":"RegexpError","count":2},{"file":"docs/verification-evidence/correct3/pr_body.md","line":11,"word":"Tokenizer","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":17,"word":"aff","count":2},{"file":"docs/verification-evidence/correct3/pr_body.md","line":14,"word":"config","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":15,"word":"de","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":7,"word":"el","count":3},{"file":"docs/verification-evidence/correct3/pr_body.md","line":7,"word":"kotoshu","count":3},{"file":"docs/verification-evidence/correct3/pr_body.md","line":28,"word":"lang","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":25,"word":"rubocop","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":19,"word":"sv","count":3},{"file":"docs/verification-evidence/correct3/pr_body.md","line":14,"word":"tokenizer","count":3},{"file":"docs/verification-evidence/correct3/pr_body.md","line":11,"word":"tokenizers","count":2},{"file":"docs/verification-evidence/correct3/pr_body.md","line":15,"word":"whitespace","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":31,"word":"xqzz","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":32,"word":"xxqqz","count":1},{"file":"docs/verification-evidence/correct3/pr_body.md","line":33,"word":"zzqqxvv","count":1}]}
|
data/README.adoc
CHANGED
|
@@ -17,7 +17,7 @@ directory layout, SHA-256 integrity verification, SARIF output, an
|
|
|
17
17
|
* `Kotoshu.check(text, language: "en")` / `Kotoshu.suggest("helo")` — full document check and suggestions
|
|
18
18
|
* `Kotoshu.spellchecker_for(lang)` — per-language checker resolved from the cache (`strict:` is a `Kotoshu.setup` option)
|
|
19
19
|
* `kotoshu check FILE` CLI with these flags:
|
|
20
|
-
** `--language
|
|
20
|
+
** `--language LANG|auto` — any set-up language code, script variants included (`sr-Latn`) (default: `auto`)
|
|
21
21
|
** `--format text|json|sarif`
|
|
22
22
|
** `--offline` — use only cached resources, never download
|
|
23
23
|
** `--strict` — exit 3 if any optional resource (frequency, model) can't load
|
|
@@ -218,7 +218,13 @@ typo-heavy text.
|
|
|
218
218
|
[[multi-language]]
|
|
219
219
|
== Multi-Language Support
|
|
220
220
|
|
|
221
|
-
Kotoshu
|
|
221
|
+
Kotoshu serves every language staged in the dictionaries manifest —
|
|
222
|
+
36 full-feature codes with per-language modules (script-aware
|
|
223
|
+
tokenizer, national keyboard grid, engine-verified specimens) and the
|
|
224
|
+
script-fallback basic tier for the rest; see the
|
|
225
|
+
<<language-matrix, Language Support Matrix>> for the breakdown.
|
|
226
|
+
Semantic reranking adds an ONNX model where the models registry pins
|
|
227
|
+
one (50+ languages), for example:
|
|
222
228
|
|
|
223
229
|
* *de* - German (Deutsch)
|
|
224
230
|
* *en* - English
|
|
@@ -937,28 +943,29 @@ script:
|
|
|
937
943
|
kotoshu completions languages
|
|
938
944
|
----
|
|
939
945
|
|
|
946
|
+
[[language-matrix]]
|
|
940
947
|
=== Language Support Matrix
|
|
941
948
|
|
|
942
|
-
Kotoshu
|
|
949
|
+
Kotoshu serves languages in tiers. Every language staged in the
|
|
950
|
+
kotoshu/dictionaries manifest downloads through `kotoshu setup` and
|
|
951
|
+
checks through the Hunspell engine. Region variants fold onto their
|
|
952
|
+
base dictionary (`en-GB` resolves `en`, `es-MX` resolves `es`);
|
|
953
|
+
script variants keep a dictionary of their own (`sr-Latn` stages the
|
|
954
|
+
Latin Serbian dictionary alongside the Cyrillic `sr`).
|
|
943
955
|
|
|
944
|
-
.
|
|
945
|
-
[width="100%",cols="^
|
|
956
|
+
.Language Support Matrix
|
|
957
|
+
[width="100%",cols="^2,^3,^3",options="header"]
|
|
946
958
|
|=========================================
|
|
947
|
-
|
|
|
959
|
+
| Tier | Languages | What it carries |
|
|
948
960
|
|=========================================
|
|
949
|
-
|
|
|
950
|
-
|
|
|
951
|
-
|
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
| el (Greek) | ✗ | ✗ | ✓ | ✗ | ✗ | Kelly frequency only |
|
|
958
|
-
| it (Italian) | ✗ | ✗ | ✓ | ✗ | ✗ | Kelly frequency only |
|
|
959
|
-
| no (Norwegian) | ✗ | ✗ | ✓ | ✗ | ✗ | Kelly frequency only |
|
|
960
|
-
| sv (Swedish) | ✗ | ✗ | ✓ | ✗ | ✗ | Kelly frequency only |
|
|
961
|
-
|=========================================]
|
|
961
|
+
| Full feature (35 languages, 36 codes) | ar bg ca cs da de el en es et fa fr he hr hu id it ko lt lv nb ne nl nn pl pt ro ru sk sl sr sr-Latn sv tr uk vi | Per-language module: staged dictionary + Hunspell affix rules, script-aware tokenizer, national keyboard grid for typo-proximity suggestions, engine-verified specimens |
|
|
962
|
+
| Basic (26 languages) | br cy eo eu fo fur fy ga gd gl hy hyw ia ie is ka la lb ltg mk mn nds oc rw tk tlh | Staged dictionary + script-fallback word extraction and the generic keyboard |
|
|
963
|
+
| Kelly frequency (8 languages) | ar el en it no ru sv zh | Top-1000 frequency list wired into suggestion ranking, on top of either tier |
|
|
964
|
+
|=========================================+
|
|
965
|
+
|
|
966
|
+
ONNX semantic models (opt-in, `--model`) resolve per the registry the
|
|
967
|
+
link:https://github.com/kotoshu/models-fasttext-onnx[models repo]
|
|
968
|
+
pins — 50+ languages across the tiers, in full/fluency/mini sizes.
|
|
962
969
|
|
|
963
970
|
.Dictionary Sources
|
|
964
971
|
[width="100%",cols="^1,^1,^1,^1",options="header"]
|
|
@@ -973,6 +980,10 @@ Kotoshu provides multi-language support with varying feature availability.
|
|
|
973
980
|
| ru (Russian) | 146,269 | BSD-style | Alexander Lebedev |
|
|
974
981
|
|=========================================+
|
|
975
982
|
|
|
983
|
+
Staged dictionaries for the other tiers carry their provenance in the
|
|
984
|
+
dictionaries repo manifest (`LICENSE` and source per file; LibreOffice
|
|
985
|
+
/ wooorm-dictionaries derivations dominate).
|
|
986
|
+
|
|
976
987
|
.Kelly Frequency Lists
|
|
977
988
|
[width="100%",cols="^1,^1,^1",options="header"]
|
|
978
989
|
|=========================================
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Plan 110 — sr-Latn wiring and the long-tail polish list
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
Plan 107 wired nn and deferred sr-Latn (the Latin script variant of Serbian)
|
|
5
|
+
as follow-up. With the basic tier live, this is the last named language gap
|
|
6
|
+
in the corpus; everything after it is small-polish.
|
|
7
|
+
|
|
8
|
+
## Work (gem repo)
|
|
9
|
+
1. sr-Latn: register as a full-feature module reusing the Serbian dictionary
|
|
10
|
+
with the Latin tokenizer + the Croatian/Slovenian QWERTZ grid (the
|
|
11
|
+
scripts share the Latin layout), or map sr-Latn -> the staged sr-Latn
|
|
12
|
+
dictionary if one exists in the manifest — follow the data, document the
|
|
13
|
+
choice. Specimens engine-verified.
|
|
14
|
+
2. Polish sweep: any spellcheck_word_regex script set lacking a staged
|
|
15
|
+
language (inventory + one-line table in the PR body); README language
|
|
16
|
+
table refresh to the 35+ tiers.
|
|
17
|
+
3. Frequency-list inventory (no sourcing): list which staged languages have
|
|
18
|
+
published frequency data wired vs not — record only, sourcing is a
|
|
19
|
+
separate owner decision.
|
|
20
|
+
|
|
21
|
+
## Verification
|
|
22
|
+
Live setup + check -l sr-Latn round trip; suite green; conformance
|
|
23
|
+
untouched.
|
|
24
|
+
|
|
25
|
+
## Status
|
|
26
|
+
Executed 2026-09-07 — the data chose the distinct staged dictionary:
|
|
27
|
+
sr-Latn is a full-feature module (Languages::SerbianLatin, Latin
|
|
28
|
+
tokenizer, Serbian-Latin QWERTZ grid sharing the hrsl key table), and
|
|
29
|
+
the sr-Latn code now survives ResourceManager normalization
|
|
30
|
+
(script subtags kept, region subtags still fold) so the staged Latin
|
|
31
|
+
files are reachable — previously sr-Latn collapsed onto the Cyrillic
|
|
32
|
+
sr dictionary. Script-set and frequency inventories recorded in the
|
|
33
|
+
PR body (record-only). Suite green, conformance untouched.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Plan 111 — The embedding bake-off: is fastText still the right model?
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
Our gates prove int8 near-lossless RELATIVE to fastText — but fastText is
|
|
5
|
+
2017-era, context-blind, and our own numbers cap intended-top-5 near 0.21
|
|
6
|
+
on hard English typos even with buckets. The eval harness (keyboard-aware
|
|
7
|
+
grids, typo corpora, frozen gates) is the asset; use it to test successors.
|
|
8
|
+
|
|
9
|
+
## Work (models repo; eval only — no registry release without a verdict)
|
|
10
|
+
1. Baseline: current mini/fluency+buckets through the existing gates.
|
|
11
|
+
2. Candidate B: fastText + small cross-encoder rerank over (context,
|
|
12
|
+
candidate) pairs — ONNX-exportable distill class, 15-30 MB int8.
|
|
13
|
+
3. Candidate C: purpose-trained typo bi-encoder over the vendored
|
|
14
|
+
github-typo-corpus (train/eval split strictly by repo).
|
|
15
|
+
4. Candidate D: ModernBERT-class small encoder, same size budget.
|
|
16
|
+
5. Report: gates table per candidate per language (en de ru es minimum),
|
|
17
|
+
size, latency-per-suggest, and a ship/reject verdict with the same
|
|
18
|
+
discipline as int4 (reject on data is a complete outcome).
|
|
19
|
+
|
|
20
|
+
## Verification
|
|
21
|
+
Every number from the existing harness, unweakened; frozen corpora; no
|
|
22
|
+
registry changes in this plan.
|
|
23
|
+
|
|
24
|
+
## Status
|
|
25
|
+
Executed 2026-09-09 — models PR #25 merged. VERDICT: fastText stays.
|
|
26
|
+
B (MiniLM cross-encoder rerank) REJECT — loses 4/4 languages (en 0.194 vs
|
|
27
|
+
fluency 0.252 top-5; slate recall caps en at 0.43; context ablation
|
|
28
|
+
negative). D (ModernBERT-base) REJECT — representation failure (top-5
|
|
29
|
+
0.03-0.24), 5x size cap. C (0.5 MB char-BiGRU typo bi-encoder, 5.2
|
|
30
|
+
ms/suggest) REJECT as drop-in but PROMISING — beats the full tier on the
|
|
31
|
+
strict repo-clean subset for en and de; open thread needing non-en data,
|
|
32
|
+
a bigger clean bench, and a top-1 story. Full ladders in
|
|
33
|
+
eval/reports/bakeoff-v1.md.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Plan 112 — Deployability and CI gating
|
|
2
|
+
|
|
3
|
+
## Work (kotoshu-rs; site rides a follow-up)
|
|
4
|
+
1. Prebuilt worker on npm: @kotoshu/worker exposing the playground's
|
|
5
|
+
engine worker (load/check/suggest-batch/semantic/detect protocol) so
|
|
6
|
+
JS users embed the engine without reimplementing the worker; versioned
|
|
7
|
+
with the wasm package.
|
|
8
|
+
2. CI latency gate: the multi-language sweep bench (rs PR #23 harness)
|
|
9
|
+
runs on CI with budgets (en < 120 ms, pt < 700 ms avg) failing red.
|
|
10
|
+
3. Wasm memory ceiling test: dict + mini tier + buckets resident under a
|
|
11
|
+
stated budget per language class.
|
|
12
|
+
4. Language pack (design note, owner decision): one fetch for
|
|
13
|
+
dict+tier+buckets — additive registry field sketch in the PR body.
|
|
14
|
+
|
|
15
|
+
## Verification
|
|
16
|
+
Node smoke for the worker package; budgets tuned to measured p50s, not
|
|
17
|
+
aspirations; all existing CI green.
|
|
18
|
+
|
|
19
|
+
## Status
|
|
20
|
+
Executed 2026-09-09 — kotoshu-rs PR #28 merged. @kotoshu/worker package
|
|
21
|
+
(createEngine full protocol, ./engine-worker entry, semantic-merge pure
|
|
22
|
+
half; publish pending owner npm registration); latency.yml gate with
|
|
23
|
+
measured budgets (en 56ms avg vs 120 gate, pt 346 vs 700 — 2x headroom);
|
|
24
|
+
wasm32 memory ceilings frozen (en 46.9/64 MB, pt 165/192 MB) with
|
|
25
|
+
behavior probes; language-pack kotoshu://packs/{lang} sketch in the PR
|
|
26
|
+
body awaiting owner decision.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Plan 113 — Language packs: one fetch for a whole language
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
Loading a language today is 3+ round trips (dict aff+dic, tier, buckets,
|
|
5
|
+
sometimes registry) - the rs PR #28 sketch showed a single length-prefixed
|
|
6
|
+
pack is simpler for SDK users and cuts playground cold loads.
|
|
7
|
+
|
|
8
|
+
## Work
|
|
9
|
+
1. models repo: pack builder (dict sources + tier + buckets concatenated,
|
|
10
|
+
length-prefixed sections; kotoshu://packs/{lang} additive registry
|
|
11
|
+
entries, versioned with their parts); v1.6.0 when validated.
|
|
12
|
+
2. kotoshu-rs: wasm loadPack(bytes) -> {dictionary, model} handle; worker
|
|
13
|
+
package gains pack mode; keep per-artifact loading as fallback.
|
|
14
|
+
3. Site: playground opt-in pack path with one progress bar.
|
|
15
|
+
|
|
16
|
+
## Verification
|
|
17
|
+
Pack bytes == concatenation of parts (checksums per section); parity
|
|
18
|
+
specs per language; playground CDP cold-load comparison (requests and ms).
|
|
19
|
+
|
|
20
|
+
## Status
|
|
21
|
+
Executed 2026-09-09 — models PR #26 + kotoshu-rs PR #32, both merged.
|
|
22
|
+
KPK1 format (length+tag framed sections, sha256 footers, golden-sha parity
|
|
23
|
+
across implementations); en/de/pt packs LFS-committed with live mirrors;
|
|
24
|
+
additive kotoshu://packs/{lang} entries; wasm loadPack + worker pack mode
|
|
25
|
+
with per-artifact fallback. Registry v1.6.0 cut awaiting owner; site
|
|
26
|
+
playground flip optional (worker package already serves pack mode).
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Plan 114 — The typo bi-encoder thread (bake-off follow-up)
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
Bake-off candidate C (0.5 MB char-BiGRU, 5.2 ms/suggest) beat the FULL
|
|
5
|
+
tier on repo-clean en/de pairs - the only successor that did. It needs:
|
|
6
|
+
non-en training data, a bigger clean bench, and a top-1 story.
|
|
7
|
+
|
|
8
|
+
## Work (models repo)
|
|
9
|
+
1. Extend training data beyond en (de ru es typo corpora; synth via the
|
|
10
|
+
existing generator where corpora are thin).
|
|
11
|
+
2. Grow the repo-clean bench (target 2000+ pairs/lang); freeze as the
|
|
12
|
+
C-benchmark.
|
|
13
|
+
3. Train v2; measure against fastText tiers on the same bench; ship as
|
|
14
|
+
an OPT-IN registry resource only if it wins en+de clearly and degrades
|
|
15
|
+
nowhere - reject honestly otherwise.
|
|
16
|
+
|
|
17
|
+
## Status
|
|
18
|
+
Executed 2026-09-09 — models PR #27 merged. VERDICT: REJECT, honest on
|
|
19
|
+
data; fastText tiers stay, no registry change, no tag. Frozen C-benchmark:
|
|
20
|
+
en 2509 real repo-clean pairs (target 2000+ met with real data only),
|
|
21
|
+
de/ru/es real clean pairs are 14/27/20 — the corpus holds only 212/551/186
|
|
22
|
+
unique pairs for those languages — plus a labeled 2000-pair noise.py synth
|
|
23
|
+
component per language, disjoint from all training sources. C v2 (same
|
|
24
|
+
0.481 MB architecture, training data the one changed variable) beats the
|
|
25
|
+
full tier on 2509 real unseen en pairs (+7.5 pp top-5, CI +5.9 to +9.1)
|
|
26
|
+
and fixed the top-1 gap (0.128 vs 0.103), but fails the pre-declared rule:
|
|
27
|
+
de real clears no paired CI at n=14, and es real regresses 10 pp top-5 at
|
|
28
|
+
n=20 (three independent es slices agree). Synth dominance (+40 to +48 pp)
|
|
29
|
+
is generator-domain and does not transfer to real es. Surviving thread for
|
|
30
|
+
a future plan: the C-retrieve + fastText-rescore hybrid beats the full
|
|
31
|
+
tier top-1 and top-5 on all four real components, but needs the
|
|
32
|
+
per-language vocab matrix cost priced and a real es/de typo corpus.
|
|
33
|
+
Full ladder: models eval/reports/biencoder-v2.md.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Plan 115 — The hybrid retrieval thread: price it before believing it
|
|
2
|
+
|
|
3
|
+
## Why
|
|
4
|
+
Plan 114's reject left one survivor on record: C-retrieve (0.5 MB bi-encoder)
|
|
5
|
+
+ fastText-rescore beat the FULL tier on top-1 AND top-5 across all four
|
|
6
|
+
real bench components. Before any ship decision, its costs and its
|
|
7
|
+
dependence on synthetic-heavy slices must be priced honestly.
|
|
8
|
+
|
|
9
|
+
## Work (models repo)
|
|
10
|
+
1. Price the retrieval half: per-language 100k x 256 vocab matrix (fp16 +
|
|
11
|
+
int8 variants) — build for en, measure size, build time, top-k
|
|
12
|
+
retrieval latency, and quality vs the brute-force sweep on the frozen
|
|
13
|
+
C-benchmark.
|
|
14
|
+
2. Quality stress: score the hybrid ONLY on real-pair components
|
|
15
|
+
(en 2509; de/ru/es real n are tiny — report with CIs, no synth pooling
|
|
16
|
+
for the verdict).
|
|
17
|
+
3. Verdict per the plan-114 rule: ship as opt-in registry resource only
|
|
18
|
+
on clear wins with no regression; honest reject otherwise. No registry
|
|
19
|
+
changes on reject.
|
|
20
|
+
|
|
21
|
+
## Status
|
|
22
|
+
Pending
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Plan 116 — Baseline-aware suggestion skip in `kotoshu check`
|
|
2
|
+
|
|
3
|
+
Status: executed (gem PR #175, 2026-09-10)
|
|
4
|
+
|
|
5
|
+
Measured on this repo's own dogfood gate (prose baseline, 640 entries /
|
|
6
|
+
1,905 covered occurrences): check wall time 13m+ -> 13.7s, identical
|
|
7
|
+
output. Full default suite 4003/0.
|
|
8
|
+
|
|
9
|
+
Depends on: plan 82 (baselines), plan 113 (packs — unrelated but same file surfaces)
|
|
10
|
+
Blocks: nothing; targets the 0.11.x line before 1.0
|
|
11
|
+
|
|
12
|
+
## Problem
|
|
13
|
+
|
|
14
|
+
`kotoshu check --baseline` in CI regenerates the suggestion sweep for
|
|
15
|
+
every error it is about to suppress. On the kotoshu repo itself the
|
|
16
|
+
prose baseline holds 1,905 occurrences across 7 files; the check phase
|
|
17
|
+
of the dogfood gate spends ~10 CPU-minutes producing suggestions that
|
|
18
|
+
the baseline then throws away. The site repo's spellcheck job runs
|
|
19
|
+
6m48s for the same reason. Suggestions only matter for errors that
|
|
20
|
+
SURVIVE the baseline — those are the rows a human reads.
|
|
21
|
+
|
|
22
|
+
`kotoshu baseline init` already fixed this for itself
|
|
23
|
+
(`Spellchecker#check(text, suggestions: false)`, plan-executed in
|
|
24
|
+
"fix(cli): make baseline init usable on real repositories"). `check`
|
|
25
|
+
still pays full price.
|
|
26
|
+
|
|
27
|
+
## Design
|
|
28
|
+
|
|
29
|
+
The budget is knowable before checking: the baseline entry list for the
|
|
30
|
+
file under check says exactly which words will be absorbed (and how
|
|
31
|
+
many times). So:
|
|
32
|
+
|
|
33
|
+
1. `Cli::DirectoryCheck` (and the single-file check path) computes
|
|
34
|
+
`covered = baseline.entries_for(path)` → a word→count budget hash,
|
|
35
|
+
before invoking the spellcheck callback.
|
|
36
|
+
2. `Spellchecker#check` gains an optional `suggestions_filter:` hook —
|
|
37
|
+
a callable receiving the word, returning whether to generate
|
|
38
|
+
suggestions. Default nil = generate for everything (current
|
|
39
|
+
behavior). The loop already has the `suggestions:` branch; this
|
|
40
|
+
generalizes it without touching result shape.
|
|
41
|
+
- Occurrence counting must be honored: the budget for `wrold: 3`
|
|
42
|
+
means the first 3 occurrences skip, the 4th (a NEW error)
|
|
43
|
+
generates. The filter therefore closes over a mutable budget
|
|
44
|
+
hash in the caller, decremented per consumed occurrence.
|
|
45
|
+
3. Errors suppressed by the baseline keep their (empty) suggestion set
|
|
46
|
+
— unchanged from today, since suppressed rows render without
|
|
47
|
+
suggestions anyway.
|
|
48
|
+
4. New errors — beyond budget or unknown words — generate suggestions
|
|
49
|
+
exactly as today. SARIF, text, and JSON output shapes unchanged.
|
|
50
|
+
5. Inline-suppressed words: keep generating suggestions today (they
|
|
51
|
+
are dropped at assembly regardless) — fold into the same skip by
|
|
52
|
+
treating `kotoshu:disable` regions as covered when the suppression
|
|
53
|
+
scan already knows the word/line. Out of scope if the scan order
|
|
54
|
+
makes this awkward; the baseline case is the measured cost.
|
|
55
|
+
|
|
56
|
+
## Surfaces touched
|
|
57
|
+
|
|
58
|
+
- `lib/kotoshu/spellchecker.rb` — `check(text, suggestions:,
|
|
59
|
+
suggestions_filter:)`
|
|
60
|
+
- `lib/kotoshu/cli/directory_check.rb` — budget construction from the
|
|
61
|
+
loaded baseline, filter closure
|
|
62
|
+
- `lib/kotoshu/cli.rb` — single-file `--baseline` path, same wiring
|
|
63
|
+
- `spec/kotoshu/cli/baseline_cli_spec.rb` — new cases: covered word
|
|
64
|
+
beyond budget generates suggestions; covered occurrence does not;
|
|
65
|
+
no baseline = unchanged
|
|
66
|
+
|
|
67
|
+
## Non-goals
|
|
68
|
+
|
|
69
|
+
- No output format changes; suppressed rows stay suggestion-less.
|
|
70
|
+
- No change to `Baseline::Store` semantics (apply stays the authority
|
|
71
|
+
on pass/fail); the filter is purely a compute optimization whose
|
|
72
|
+
visible effect is empty suggestions on rows that were going to be
|
|
73
|
+
suppressed.
|
|
74
|
+
|
|
75
|
+
## Acceptance
|
|
76
|
+
|
|
77
|
+
- Dogfood gate on the kotoshu repo: check phase wall time drops from
|
|
78
|
+
~10 min to under ~2 min (the sweep itself, minus 1,905 suggestion
|
|
79
|
+
generations).
|
|
80
|
+
- All existing baseline specs green; new filter specs green.
|
|
81
|
+
- Site repo spellcheck job measurably faster after its gem bump.
|
|
@@ -79,6 +79,29 @@ module Kotoshu
|
|
|
79
79
|
self
|
|
80
80
|
end
|
|
81
81
|
|
|
82
|
+
# A decrementing budget view of +file+'s entries for the check
|
|
83
|
+
# path (plan 116): the callable answers false for the first
|
|
84
|
+
# +count+ occurrences of each entry word — exactly the
|
|
85
|
+
# occurrences {#apply} will absorb, whose suggestions nobody
|
|
86
|
+
# reads — and true for anything beyond, so new errors keep
|
|
87
|
+
# their suggestions. Occurrence order is check order, the same
|
|
88
|
+
# position order #apply absorbs in.
|
|
89
|
+
#
|
|
90
|
+
# @param file [String] path the text being checked came from
|
|
91
|
+
# @return [#call] word -> boolean
|
|
92
|
+
def suggestions_filter_for(file)
|
|
93
|
+
budget = Hash.new(0)
|
|
94
|
+
entries_for(file).each { |entry| budget[entry.word] += entry.count }
|
|
95
|
+
lambda do |word|
|
|
96
|
+
if budget[word].positive?
|
|
97
|
+
budget[word] -= 1
|
|
98
|
+
false
|
|
99
|
+
else
|
|
100
|
+
true
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
82
105
|
# Entries recorded for +file+. Paths compare canonically — a
|
|
83
106
|
# baseline recorded as `docs/a.md` matches a check run over
|
|
84
107
|
# `./docs/a.md` and vice versa — so walking `.` and explicit
|
|
@@ -17,17 +17,20 @@ module Kotoshu
|
|
|
17
17
|
|
|
18
18
|
desc "init FILE [FILE ...]", "Write a baseline of the current findings"
|
|
19
19
|
long_desc <<~DESC
|
|
20
|
-
Checks each FILE
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
reformatting.
|
|
20
|
+
Checks each FILE (directories and globs expand exactly like
|
|
21
|
+
`kotoshu check` directory mode; inline ignore directives apply)
|
|
22
|
+
and writes the canonical baseline JSON: one entry per (file,
|
|
23
|
+
word) with the misspelling count, stable across reformatting.
|
|
24
|
+
The personal dictionary is NOT consulted - baselines freeze what
|
|
25
|
+
every machine would see, and CI has no personal dictionary.
|
|
26
|
+
Suggestions are skipped: a baseline only needs positions.
|
|
24
27
|
|
|
25
28
|
The baseline only covers real errors present right now; new
|
|
26
29
|
errors still fail `kotoshu check --baseline`.
|
|
27
30
|
|
|
28
31
|
Examples:
|
|
29
32
|
|
|
30
|
-
kotoshu baseline init
|
|
33
|
+
kotoshu baseline init . # directories and globs expand like check
|
|
31
34
|
kotoshu baseline init src/*.txt --output .kotoshu-baseline.json
|
|
32
35
|
DESC
|
|
33
36
|
method_option :output,
|
|
@@ -40,12 +43,24 @@ module Kotoshu
|
|
|
40
43
|
"Give at least one file: kotoshu baseline init FILE [FILE ...]"
|
|
41
44
|
end
|
|
42
45
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
missing = targets.reject do |target|
|
|
47
|
+
File.directory?(target) || File.file?(target) || !Dir[target].empty?
|
|
48
|
+
end
|
|
49
|
+
unless missing.empty?
|
|
50
|
+
raise Errors::UsageError, "File not found: #{missing.join(' ')}"
|
|
51
|
+
end
|
|
46
52
|
|
|
47
|
-
|
|
48
|
-
|
|
53
|
+
files = targets.flat_map { |t| expand_target(t) }
|
|
54
|
+
if files.empty?
|
|
55
|
+
raise Errors::UsageError,
|
|
56
|
+
"No text files found (known extensions: " \
|
|
57
|
+
"#{Cli::DirectoryWalker::TEXT_EXTENSIONS.join(' ')})"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
checks = {}
|
|
61
|
+
files.sort.each do |file|
|
|
62
|
+
text = File.read(file, encoding: Kotoshu.configuration.encoding).scrub
|
|
63
|
+
checks[file] = [run_check(text), text]
|
|
49
64
|
end
|
|
50
65
|
|
|
51
66
|
store = Kotoshu::Baseline::Store.from_checks(checks)
|
|
@@ -58,13 +73,36 @@ module Kotoshu
|
|
|
58
73
|
end
|
|
59
74
|
|
|
60
75
|
no_commands do
|
|
76
|
+
# Directories expand through the same walker directory mode uses
|
|
77
|
+
# (gitignore-aware, known text extensions); globs expand via
|
|
78
|
+
# Dir; plain files pass through unchanged.
|
|
79
|
+
def expand_target(target)
|
|
80
|
+
if File.directory?(target)
|
|
81
|
+
DirectoryWalker.new.files(target)
|
|
82
|
+
elsif File.file?(target)
|
|
83
|
+
[target]
|
|
84
|
+
else
|
|
85
|
+
Dir[target].select { |f| File.file?(f) }
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Baselines must be machine-independent: the personal dictionary
|
|
90
|
+
# of whichever machine runs init must never leak into the frozen
|
|
91
|
+
# set, or CI (which has none) flags words the baseline never
|
|
92
|
+
# recorded. Disabled for the whole init run.
|
|
61
93
|
def run_check(text)
|
|
94
|
+
previous = Kotoshu.configuration.personal_dictionary
|
|
95
|
+
Kotoshu.configuration.personal_dictionary = false
|
|
96
|
+
Kotoshu.reset_spellchecker
|
|
62
97
|
language = resolve_language(text)
|
|
63
|
-
Kotoshu.spellchecker_for(language).check(text)
|
|
98
|
+
Kotoshu.spellchecker_for(language).check(text, suggestions: false)
|
|
64
99
|
rescue Kotoshu::ResourceNotSetupError => e
|
|
65
100
|
raise Errors::ResourceUnavailable, e.message
|
|
66
101
|
rescue Kotoshu::DictionaryNotFoundError => e
|
|
67
102
|
raise Errors::ResourceUnavailable, e.message
|
|
103
|
+
ensure
|
|
104
|
+
Kotoshu.configuration.personal_dictionary = previous
|
|
105
|
+
Kotoshu.reset_spellchecker
|
|
68
106
|
end
|
|
69
107
|
|
|
70
108
|
def resolve_language(text)
|
|
@@ -18,9 +18,10 @@ module Kotoshu
|
|
|
18
18
|
attr_reader :outcomes
|
|
19
19
|
|
|
20
20
|
# @param roots [Array<String>] files and directories to check
|
|
21
|
-
# @param check [#call] callable taking the file text and
|
|
22
|
-
#
|
|
23
|
-
# file-mode
|
|
21
|
+
# @param check [#call] callable taking the file text and an
|
|
22
|
+
# optional suggestions_filter: keyword and returning a
|
|
23
|
+
# Models::Result::DocumentResult (the CLI file-mode
|
|
24
|
+
# pipeline, including inline suppressions)
|
|
24
25
|
# @param baseline_path [String, nil] --baseline file
|
|
25
26
|
# @param include_globs [Array<String>] --include globs
|
|
26
27
|
# @param exclude_globs [Array<String>] --exclude globs
|
|
@@ -111,8 +112,12 @@ module Kotoshu
|
|
|
111
112
|
# @param path [String] file path
|
|
112
113
|
# @return [FileOutcome]
|
|
113
114
|
def check_file(path)
|
|
114
|
-
text = File.read(path, encoding: Kotoshu.configuration.encoding)
|
|
115
|
-
|
|
115
|
+
text = File.read(path, encoding: Kotoshu.configuration.encoding).scrub
|
|
116
|
+
# Baseline-covered occurrences skip the suggestion sweep
|
|
117
|
+
# (plan 116): the filter answers false for exactly the
|
|
118
|
+
# occurrences apply_baseline will absorb.
|
|
119
|
+
filter = @baseline_path ? baseline_store.suggestions_filter_for(path) : nil
|
|
120
|
+
result = @check.call(text, suggestions_filter: filter)
|
|
116
121
|
application = apply_baseline(result, path)
|
|
117
122
|
result = application.result if application
|
|
118
123
|
FileOutcome.new(path: path, result: result, application: application)
|
data/lib/kotoshu/cli.rb
CHANGED
|
@@ -446,18 +446,36 @@ module Kotoshu
|
|
|
446
446
|
# semantics; see Kotoshu::Baseline). Returns nil when no baseline
|
|
447
447
|
# was given.
|
|
448
448
|
def apply_baseline(result, source)
|
|
449
|
+
store = check_baseline_store
|
|
450
|
+
return nil unless store
|
|
451
|
+
|
|
452
|
+
store.apply(result, file: source)
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
# The --baseline file's store, loaded once per invocation.
|
|
456
|
+
def check_baseline_store
|
|
449
457
|
path = options[:baseline]
|
|
450
458
|
return nil unless path
|
|
451
459
|
|
|
452
|
-
|
|
460
|
+
@check_baseline_store ||= begin
|
|
461
|
+
raise Errors::UsageError, "Baseline file not found: #{path}" unless File.exist?(path)
|
|
453
462
|
|
|
454
|
-
|
|
463
|
+
Kotoshu::Baseline::Store.load(path)
|
|
464
|
+
end
|
|
455
465
|
rescue Errors::UsageError
|
|
456
466
|
raise
|
|
457
467
|
rescue StandardError => e
|
|
458
468
|
raise Errors::UsageError, "Invalid baseline file #{path}: #{e.message}"
|
|
459
469
|
end
|
|
460
470
|
|
|
471
|
+
# Baseline-covered occurrences in the single-file path skip the
|
|
472
|
+
# suggestion sweep too (plan 116) — same budget the directory
|
|
473
|
+
# runner builds per file.
|
|
474
|
+
def single_file_filter(source)
|
|
475
|
+
store = check_baseline_store
|
|
476
|
+
store&.suggestions_filter_for(source)
|
|
477
|
+
end
|
|
478
|
+
|
|
461
479
|
def read_target(target)
|
|
462
480
|
if target.nil?
|
|
463
481
|
[$stdin.read, "<stdin>"]
|
|
@@ -468,10 +486,10 @@ module Kotoshu
|
|
|
468
486
|
end
|
|
469
487
|
end
|
|
470
488
|
|
|
471
|
-
def run_check(text)
|
|
489
|
+
def run_check(text, suggestions_filter: nil)
|
|
472
490
|
language = resolve_language(text)
|
|
473
491
|
spellchecker = Kotoshu.spellchecker_for(language)
|
|
474
|
-
spellchecker.check(text)
|
|
492
|
+
spellchecker.check(text, suggestions_filter: suggestions_filter)
|
|
475
493
|
rescue Kotoshu::DictionaryNotFoundError => e
|
|
476
494
|
raise Errors::ResourceUnavailable, e.message
|
|
477
495
|
end
|
|
@@ -768,7 +786,7 @@ module Kotoshu
|
|
|
768
786
|
# unchanged).
|
|
769
787
|
def check_single_file(target)
|
|
770
788
|
text, source = read_target(target)
|
|
771
|
-
result = run_check(text)
|
|
789
|
+
result = run_check(text, suggestions_filter: single_file_filter(source))
|
|
772
790
|
application = apply_baseline(result, source)
|
|
773
791
|
result = application.result if application
|
|
774
792
|
display_result(result, source, application: application)
|
|
@@ -73,6 +73,22 @@ module Kotoshu
|
|
|
73
73
|
)
|
|
74
74
|
end
|
|
75
75
|
end
|
|
76
|
+
|
|
77
|
+
# Serbian Latin QWERTZ (the srp driver table — the same
|
|
78
|
+
# South-Slavic physical arrangement as Croatian and Slovenian;
|
|
79
|
+
# plan 110).
|
|
80
|
+
#
|
|
81
|
+
# Languages: sr-Latn
|
|
82
|
+
class SerbianLatin < Layout
|
|
83
|
+
# Initialize the Serbian Latin layout.
|
|
84
|
+
def initialize
|
|
85
|
+
super(
|
|
86
|
+
name: 'Serbian-Latin-QWERTZ',
|
|
87
|
+
language_codes: %w[sr-Latn],
|
|
88
|
+
key_positions: HrSlQwertz::KEY_POSITIONS
|
|
89
|
+
)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
76
92
|
end
|
|
77
93
|
end
|
|
78
94
|
end
|
|
@@ -9,9 +9,9 @@ module Kotoshu
|
|
|
9
9
|
# The Serbian-specific letters љ њ ђ ћ џ ј are real keys: љ њ
|
|
10
10
|
# take the q/w faces, ђ sits where QWERTY carries the right
|
|
11
11
|
# bracket, and џ ц в б н м fill the bottom row. Serbian Latin
|
|
12
|
-
# (sr-Latn) types on
|
|
13
|
-
# by
|
|
14
|
-
#
|
|
12
|
+
# (sr-Latn) types on the South-Slavic QWERTZ grid and is
|
|
13
|
+
# claimed by its own layout (Layouts::SerbianLatin, plan 110),
|
|
14
|
+
# not this one.
|
|
15
15
|
#
|
|
16
16
|
# The key grid is mirrored from the models repo eval harness
|
|
17
17
|
# (kotoshu/models-fasttext-onnx eval/noise.py _SR_CYR, plan 83
|
|
@@ -150,11 +150,15 @@ module Kotoshu
|
|
|
150
150
|
register(Layouts::QWERTY)
|
|
151
151
|
register(Layouts::QWERTZ)
|
|
152
152
|
register(Layouts::AZERTY)
|
|
153
|
-
#
|
|
154
|
-
#
|
|
155
|
-
#
|
|
156
|
-
#
|
|
157
|
-
#
|
|
153
|
+
# Serbian Latin registers before Serbian Cyrillic: the Cyrillic
|
|
154
|
+
# layout claims sr, and base-language matching would fold
|
|
155
|
+
# sr-Latn onto it; the Latin variant needs its own grid (plan
|
|
156
|
+
# 110). Ukrainian, Bulgarian and Serbian register before JCUKEN:
|
|
157
|
+
# the base JCUKEN layout blanket-claims uk bg, but typists of
|
|
158
|
+
# those languages get their dedicated grids (plan 84 uk, plan
|
|
159
|
+
# 100 bg/sr). Lookup is first-registered-wins, so order here is
|
|
160
|
+
# load-bearing for uk bg sr sr-Latn.
|
|
161
|
+
register(Layouts::SerbianLatin)
|
|
158
162
|
register(Layouts::Ukrainian)
|
|
159
163
|
register(Layouts::BulgarianBds)
|
|
160
164
|
register(Layouts::SerbianCyrillic)
|
data/lib/kotoshu/keyboard.rb
CHANGED
|
@@ -41,6 +41,10 @@ module Kotoshu
|
|
|
41
41
|
autoload :SerbianCyrillic, "kotoshu/keyboard/layouts/serbian"
|
|
42
42
|
autoload :Croatian, "kotoshu/keyboard/layouts/hrsl"
|
|
43
43
|
autoload :Slovenian, "kotoshu/keyboard/layouts/hrsl"
|
|
44
|
+
# Serbian Latin (plan 110): the South-Slavic QWERTZ grid the hr
|
|
45
|
+
# and sl layouts share, claiming sr-Latn so it does not fold
|
|
46
|
+
# onto the Serbian Cyrillic grid.
|
|
47
|
+
autoload :SerbianLatin, "kotoshu/keyboard/layouts/hrsl"
|
|
44
48
|
# ko and ne layouts (plan 108): Dubeolsik cites the KS X 5002
|
|
45
49
|
# layout standard (the eval harness models ko slips as jamo
|
|
46
50
|
# confusion pairs, not a key grid); Devanagari InScript is
|
|
@@ -9,7 +9,8 @@ module Kotoshu
|
|
|
9
9
|
# primary and the script the staged dictionary carries — the
|
|
10
10
|
# module registers sr for the Cyrillic grid
|
|
11
11
|
# (Keyboard::Layouts::SerbianCyrillic); Serbian Latin (sr-Latn)
|
|
12
|
-
#
|
|
12
|
+
# has its own module over the distinct staged Latin dictionary
|
|
13
|
+
# (plan 110, Kotoshu::Languages::SerbianLatin). The Serbian
|
|
13
14
|
# letters љ њ ђ ћ џ ј are real keys on that grid.
|
|
14
15
|
class Serbian < LatinBase
|
|
15
16
|
register "sr"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Serbian Latin implementation (plan 110).
|
|
6
|
+
#
|
|
7
|
+
# The Latin script variant of Serbian, the last named gap in the
|
|
8
|
+
# corpus after plan 107 opened the staged manifest. The data
|
|
9
|
+
# decided the wiring: the dictionaries manifest stages a distinct
|
|
10
|
+
# sr-Latn dictionary (LibreOffice, wooorm/dictionaries), so
|
|
11
|
+
# sr-Latn resolves its own download rather than riding the
|
|
12
|
+
# Cyrillic sr dictionary — and because sr and sr-Latn differ by
|
|
13
|
+
# script, not vocabulary, a script-subtag code must survive
|
|
14
|
+
# ResourceManager language normalization (plan 110) for the
|
|
15
|
+
# staged Latin files to be reachable at all.
|
|
16
|
+
#
|
|
17
|
+
# Thin composition like its Croatian sibling: shared Latin
|
|
18
|
+
# tokenizer (\p{Latin} covers š đ č ć ž), base normalizer,
|
|
19
|
+
# registry entry. The five Latin digraph letters are real keys on
|
|
20
|
+
# the South-Slavic QWERTZ grid
|
|
21
|
+
# (Keyboard::Layouts::SerbianLatin, the physical arrangement the
|
|
22
|
+
# hr and sl modules share).
|
|
23
|
+
class SerbianLatin < LatinBase
|
|
24
|
+
register "sr-Latn"
|
|
25
|
+
|
|
26
|
+
def initialize(code: "sr-Latn", name: "Serbian (Latin)")
|
|
27
|
+
super
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def default_dictionary_paths
|
|
31
|
+
["/usr/share/dict/serbian"]
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
data/lib/kotoshu/languages.rb
CHANGED
|
@@ -56,5 +56,9 @@ module Kotoshu
|
|
|
56
56
|
# tokenizer + InScript grid.
|
|
57
57
|
autoload :Korean, "kotoshu/languages/ko/language"
|
|
58
58
|
autoload :Nepali, "kotoshu/languages/ne/language"
|
|
59
|
+
# Serbian Latin module (plan 110): full-feature wiring over the
|
|
60
|
+
# distinct staged sr-Latn dictionary — Latin tokenizer, the
|
|
61
|
+
# South-Slavic QWERTZ grid shared with hr and sl.
|
|
62
|
+
autoload :SerbianLatin, "kotoshu/languages/sr-Latn/language"
|
|
59
63
|
end
|
|
60
64
|
end
|
|
@@ -555,8 +555,19 @@ module Kotoshu
|
|
|
555
555
|
end
|
|
556
556
|
end
|
|
557
557
|
|
|
558
|
+
# Collapse a user-given code to its cache key: lowercase base
|
|
559
|
+
# language, plus the script subtag when one is present. Region
|
|
560
|
+
# variants share their base dictionary (en-GB -> en, sv-FI -> sv),
|
|
561
|
+
# but a BCP-47 script subtag (four letters: Latn in sr-Latn and
|
|
562
|
+
# tlh-Latn) names a dictionary the manifest stages separately, so
|
|
563
|
+
# it survives normalization (plan 110).
|
|
558
564
|
def normalize_language(code)
|
|
559
|
-
code.to_s.split(
|
|
565
|
+
parts = code.to_s.split(/[-_]/)
|
|
566
|
+
base = parts.first.downcase
|
|
567
|
+
subtag = parts[1]
|
|
568
|
+
return base unless subtag&.match?(/\A[A-Za-z]{4}\z/)
|
|
569
|
+
|
|
570
|
+
"#{base}-#{subtag.downcase.capitalize}"
|
|
560
571
|
end
|
|
561
572
|
|
|
562
573
|
def spelling_cache_for(_lang = nil, config: nil)
|
data/lib/kotoshu/spellchecker.rb
CHANGED
|
@@ -184,13 +184,21 @@ module Kotoshu
|
|
|
184
184
|
# `kotoshu check --no-personal`).
|
|
185
185
|
#
|
|
186
186
|
# @param text [String] The text to check
|
|
187
|
+
# @param suggestions [Boolean] Generate suggestions for errors
|
|
188
|
+
# (default true; false yields errors with empty suggestion
|
|
189
|
+
# sets — positions and counts are unaffected)
|
|
190
|
+
# @param suggestions_filter [#call, nil] Optional word predicate;
|
|
191
|
+
# when it answers false for a misspelled word, that occurrence
|
|
192
|
+
# skips suggestion generation. The CI baseline uses this to
|
|
193
|
+
# skip the sweep for occurrences it is about to absorb
|
|
194
|
+
# (plan 116) while new errors keep their suggestions.
|
|
187
195
|
# @return [Models::Result::DocumentResult] The check result
|
|
188
196
|
#
|
|
189
197
|
# @example
|
|
190
198
|
# result = spellchecker.check("Hello wrold")
|
|
191
199
|
# result.success? # => false
|
|
192
200
|
# result.errors.map(&:word) # => ["wrold"]
|
|
193
|
-
def check(text)
|
|
201
|
+
def check(text, suggestions: true, suggestions_filter: nil)
|
|
194
202
|
return Models::Result::DocumentResult.success if text.nil? || text.empty?
|
|
195
203
|
|
|
196
204
|
# Inline ignore directives (plan 82): suppressed errors move into
|
|
@@ -205,15 +213,17 @@ module Kotoshu
|
|
|
205
213
|
words.each do |word, pos|
|
|
206
214
|
next if personal.include?(word.downcase)
|
|
207
215
|
|
|
208
|
-
|
|
209
|
-
next if
|
|
216
|
+
correct = correct?(word)
|
|
217
|
+
next if correct
|
|
210
218
|
|
|
219
|
+
want_suggestions =
|
|
220
|
+
suggestions && (!suggestions_filter || suggestions_filter.call(word))
|
|
211
221
|
line = Documents::SourcePosition.line_for_offset(text, pos)
|
|
212
222
|
suppressed = suppressions.any? { |s| s.applies_to?(line, word: word) }
|
|
213
223
|
entry = Models::Result::WordResult.new(
|
|
214
224
|
word: word,
|
|
215
225
|
correct: false,
|
|
216
|
-
suggestions:
|
|
226
|
+
suggestions: (suggest(word) if want_suggestions) || Suggestions::SuggestionSet.empty,
|
|
217
227
|
position: pos,
|
|
218
228
|
suppressed: suppressed,
|
|
219
229
|
suppressed_by: (Models::Result::WordResult::SUPPRESSED_BY_INLINE if suppressed)
|
|
@@ -62,7 +62,7 @@ module Kotoshu
|
|
|
62
62
|
def run
|
|
63
63
|
runner = Cli::DirectoryCheck.new(
|
|
64
64
|
target_files,
|
|
65
|
-
check: ->(text) { check_text(text) },
|
|
65
|
+
check: ->(text, suggestions_filter: nil) { check_text(text, suggestions_filter: suggestions_filter) },
|
|
66
66
|
baseline_path: baseline_path
|
|
67
67
|
)
|
|
68
68
|
runner.run
|
|
@@ -76,9 +76,15 @@ module Kotoshu
|
|
|
76
76
|
# Check one text through the configured language.
|
|
77
77
|
#
|
|
78
78
|
# @param text [String] file contents
|
|
79
|
+
# @param suggestions_filter [#call, nil] baseline budget filter
|
|
80
|
+
# (plan 116) — see Baseline::Store#suggestions_filter_for
|
|
79
81
|
# @return [Models::Result::DocumentResult]
|
|
80
|
-
def check_text(text)
|
|
81
|
-
|
|
82
|
+
def check_text(text, suggestions_filter: nil)
|
|
83
|
+
if language
|
|
84
|
+
Kotoshu.spellchecker_for(language).check(text, suggestions_filter: suggestions_filter)
|
|
85
|
+
else
|
|
86
|
+
Kotoshu.check(text, suggestions_filter: suggestions_filter)
|
|
87
|
+
end
|
|
82
88
|
end
|
|
83
89
|
|
|
84
90
|
# Print one line per file with errors plus a summary.
|
data/lib/kotoshu/version.rb
CHANGED
data/lib/kotoshu.rb
CHANGED
|
@@ -323,9 +323,9 @@ module Kotoshu
|
|
|
323
323
|
# Kotoshu.setup(:en)
|
|
324
324
|
# result = Kotoshu.check("Hello wrold")
|
|
325
325
|
# result.errors.map(&:word) # => ["wrold"]
|
|
326
|
-
def self.check(text, language: nil, **_options)
|
|
326
|
+
def self.check(text, language: nil, suggestions_filter: nil, **_options)
|
|
327
327
|
checker = language ? spellchecker_for(language) : spellchecker
|
|
328
|
-
checker.check(text)
|
|
328
|
+
checker.check(text, suggestions_filter: suggestions_filter)
|
|
329
329
|
end
|
|
330
330
|
|
|
331
331
|
# Check a file for spelling errors. Hot path.
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kotoshu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.11.
|
|
4
|
+
version: 0.11.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -77,6 +77,7 @@ extensions:
|
|
|
77
77
|
- ext/kotoshu_native/extconf.rb
|
|
78
78
|
extra_rdoc_files: []
|
|
79
79
|
files:
|
|
80
|
+
- ".kotoshu-baseline.json"
|
|
80
81
|
- ".pre-commit-hooks.yaml"
|
|
81
82
|
- ".rspec"
|
|
82
83
|
- ".rubocop.yml"
|
|
@@ -114,6 +115,13 @@ files:
|
|
|
114
115
|
- TODO.impl/108-ko-ne-full-feature.md
|
|
115
116
|
- TODO.impl/109-ten-audit.md
|
|
116
117
|
- TODO.impl/11-release-v1.md
|
|
118
|
+
- TODO.impl/110-sr-latn-and-polish.md
|
|
119
|
+
- TODO.impl/111-embedding-bakeoff.md
|
|
120
|
+
- TODO.impl/112-deployability-and-gating.md
|
|
121
|
+
- TODO.impl/113-language-packs.md
|
|
122
|
+
- TODO.impl/114-typo-biencoder-thread.md
|
|
123
|
+
- TODO.impl/115-hybrid-retrieval-pricing.md
|
|
124
|
+
- TODO.impl/116-baseline-suggestion-skip.md
|
|
117
125
|
- TODO.impl/20-encapsulation-cleanup.md
|
|
118
126
|
- TODO.impl/21-autoload-migration.md
|
|
119
127
|
- TODO.impl/22-url-pin-single-source-of-truth.md
|
|
@@ -434,6 +442,7 @@ files:
|
|
|
434
442
|
- lib/kotoshu/languages/ru/language.rb
|
|
435
443
|
- lib/kotoshu/languages/sk/language.rb
|
|
436
444
|
- lib/kotoshu/languages/sl/language.rb
|
|
445
|
+
- lib/kotoshu/languages/sr-Latn/language.rb
|
|
437
446
|
- lib/kotoshu/languages/sr/language.rb
|
|
438
447
|
- lib/kotoshu/languages/sv/language.rb
|
|
439
448
|
- lib/kotoshu/languages/tr/language.rb
|