kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cache
|
|
7
|
+
# Parsed view of `registry.json` from kotoshu/models-fasttext-onnx
|
|
8
|
+
# (Resource Spec v1, TODO.impl/07 of that repo; gem side: plan 67 M1).
|
|
9
|
+
#
|
|
10
|
+
# Shape:
|
|
11
|
+
#
|
|
12
|
+
# {
|
|
13
|
+
# "spec": "kotoshu.resources/v1",
|
|
14
|
+
# "registry_version": 1,
|
|
15
|
+
# "generated_at": "2026-09-02T22:16:05Z",
|
|
16
|
+
# "release_tag": "v1.0.0",
|
|
17
|
+
# "resources": {
|
|
18
|
+
# "kotoshu://models/en/mini": {
|
|
19
|
+
# "type": "model",
|
|
20
|
+
# "language": "en",
|
|
21
|
+
# "tier": { "name": "mini", "dims": 300, ... },
|
|
22
|
+
# "version": "1.0.0",
|
|
23
|
+
# "urls": { "primary": "...", "mirror": "..." },
|
|
24
|
+
# "vocab_url": "...",
|
|
25
|
+
# "sha256": "...", "size_bytes": 3040752,
|
|
26
|
+
# "license": "CC-BY-SA-3.0",
|
|
27
|
+
# "min_engine_version": "0.7",
|
|
28
|
+
# "eval_ref": "eval/reports/en.mini.json"
|
|
29
|
+
# }
|
|
30
|
+
# }
|
|
31
|
+
# }
|
|
32
|
+
#
|
|
33
|
+
# Ids are stable forever (`kotoshu://models/{lang}/{tier}`); content
|
|
34
|
+
# changes bump `version` and the release tag, never the id.
|
|
35
|
+
#
|
|
36
|
+
# Serialized via lutaml-model: use {ModelRegistry.from_json} /
|
|
37
|
+
# {Resource.from_hash} for the wire forms — no hand-rolled
|
|
38
|
+
# `to_h` / `from_h` on the model (see CLAUDE.md serialization rule).
|
|
39
|
+
class ModelRegistry < Lutaml::Model::Serializable
|
|
40
|
+
# One downloadable model artifact, keyed by its stable
|
|
41
|
+
# `kotoshu://models/{lang}/{tier}` id in the parent registry.
|
|
42
|
+
class Resource < Lutaml::Model::Serializable
|
|
43
|
+
# Download URLs. `primary` is the GitHub Release asset;
|
|
44
|
+
# `mirror` points at the git tree and is a fallback only.
|
|
45
|
+
class Urls < Lutaml::Model::Serializable
|
|
46
|
+
attribute :primary, :string
|
|
47
|
+
attribute :mirror, :string
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Tier metadata block. `quantization` is null for the full tier.
|
|
51
|
+
class Tier < Lutaml::Model::Serializable
|
|
52
|
+
attribute :name, :string
|
|
53
|
+
attribute :dims, :integer
|
|
54
|
+
attribute :vocab_size, :integer
|
|
55
|
+
attribute :quantization, :string
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
attribute :type, :string
|
|
59
|
+
attribute :language, :string
|
|
60
|
+
attribute :tier, Tier
|
|
61
|
+
attribute :version, :string
|
|
62
|
+
attribute :urls, Urls
|
|
63
|
+
attribute :vocab_url, :string
|
|
64
|
+
attribute :sha256, :string
|
|
65
|
+
attribute :size_bytes, :integer
|
|
66
|
+
attribute :license, :string
|
|
67
|
+
attribute :min_engine_version, :string
|
|
68
|
+
attribute :eval_ref, :string
|
|
69
|
+
|
|
70
|
+
# Stable registry id for a (language, tier) pair. This is the key
|
|
71
|
+
# under which the resource appears in {ModelRegistry#resources}.
|
|
72
|
+
#
|
|
73
|
+
# @param language [String] ISO 639-1 code
|
|
74
|
+
# @param tier [String, Symbol] tier name ("mini", "fluency", "full")
|
|
75
|
+
# @return [String] e.g. "kotoshu://models/en/mini"
|
|
76
|
+
def self.id_for(language, tier)
|
|
77
|
+
"kotoshu://models/#{language}/#{tier}"
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
attribute :spec, :string
|
|
82
|
+
attribute :registry_version, :integer
|
|
83
|
+
attribute :generated_at, :string
|
|
84
|
+
attribute :release_tag, :string
|
|
85
|
+
# Id => per-resource hash. Values are deserialized on lookup via
|
|
86
|
+
# {Resource.from_hash} (the framework API); lutaml-model 0.8 does
|
|
87
|
+
# not support typed values inside a `:hash` attribute.
|
|
88
|
+
attribute :resources, :hash
|
|
89
|
+
|
|
90
|
+
# Typed lookup of a model resource by language and tier.
|
|
91
|
+
#
|
|
92
|
+
# @param language [String] ISO 639-1 code
|
|
93
|
+
# @param tier [String, Symbol] tier name
|
|
94
|
+
# @return [Resource, nil] the entry, or nil when the registry has
|
|
95
|
+
# no such (language, tier) combination
|
|
96
|
+
def find(language, tier)
|
|
97
|
+
raw = resources[Resource.id_for(language, tier)]
|
|
98
|
+
raw && Resource.from_hash(raw)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# All languages the registry knows about (derived from ids).
|
|
102
|
+
#
|
|
103
|
+
# @return [Array<String>] sorted, unique language codes
|
|
104
|
+
def languages
|
|
105
|
+
resources.keys.map { |id| id.split("/")[3] }.compact.uniq.sort
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Cache
|
|
5
|
+
# Offline-safe snapshot of the languages staged in the
|
|
6
|
+
# kotoshu/dictionaries manifest (plan 107).
|
|
7
|
+
#
|
|
8
|
+
# `manifest.json` in the dictionaries repo is the single source of
|
|
9
|
+
# truth for what `kotoshu setup` can download; this module vendors
|
|
10
|
+
# its language codes so the hot path never needs the network to
|
|
11
|
+
# answer "is this language downloadable?" (resolve is cache-only
|
|
12
|
+
# by contract, and the spelling gate runs inside #available?).
|
|
13
|
+
#
|
|
14
|
+
# The list is regenerated by
|
|
15
|
+
#
|
|
16
|
+
# rake kotoshu:staged_languages:sync
|
|
17
|
+
#
|
|
18
|
+
# which fetches the manifest at the SourceRegistry pin, extracts
|
|
19
|
+
# the distinct language codes from its resources, and rewrites
|
|
20
|
+
# CODES in place. Edit the dictionaries repo, not this file.
|
|
21
|
+
#
|
|
22
|
+
# Two tiers of support (plan 107 vocabulary):
|
|
23
|
+
# - full feature: a language module under Kotoshu::Languages
|
|
24
|
+
# (keyboard layout + verified specimens + tokenizer care) —
|
|
25
|
+
# see LanguageCache.full_feature_languages;
|
|
26
|
+
# - basic: manifest-present but module-less — the script fallback
|
|
27
|
+
# tokenizer and generic keyboard still serve check/suggest.
|
|
28
|
+
module StagedLanguages
|
|
29
|
+
# Language codes carried by the dictionaries manifest at its
|
|
30
|
+
# SourceRegistry pin; refreshed by the sync rake task above.
|
|
31
|
+
CODES = %w[
|
|
32
|
+
ar bg br ca ca-valencia cs cy da de de-AT
|
|
33
|
+
de-CH el el-polyton en en-AU en-CA en-GB en-ZA eo es
|
|
34
|
+
es-AR es-BO es-CL es-CO es-CR es-CU es-DO es-EC es-GT es-HN
|
|
35
|
+
es-MX es-NI es-PA es-PE es-PH es-PR es-PY es-SV es-US es-UY
|
|
36
|
+
es-VE et eu fa fo fr fur fy ga gd
|
|
37
|
+
gl he hr hu hy hyw ia id ie is
|
|
38
|
+
it ka ko la lb lt ltg lv mk mn
|
|
39
|
+
nb nds ne nl nn oc pl pt pt-BR pt-PT
|
|
40
|
+
ro ru rw sk sl sr sr-Latn sv sv-FI tk
|
|
41
|
+
tlh tlh-Latn tr uk vi
|
|
42
|
+
].freeze
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# LookupCache autoloaded via Kotoshu::Cache
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cache
|
|
7
|
+
# LRU cache specifically for suggestion results.
|
|
8
|
+
#
|
|
9
|
+
# Extends LookupCache with suggestion-specific features like
|
|
10
|
+
# caching by word + max_results combination.
|
|
11
|
+
#
|
|
12
|
+
# @example Caching suggestions
|
|
13
|
+
# cache = SuggestionCache.new(max_size: 5000)
|
|
14
|
+
# cache.write("helo", ["hello", "help"], max_results: 10)
|
|
15
|
+
# cache.read("helo", max_results: 10) # => ["hello", "help"]
|
|
16
|
+
class SuggestionCache < LookupCache
|
|
17
|
+
# Default maximum cache size for suggestions
|
|
18
|
+
DEFAULT_MAX_SIZE = 5000
|
|
19
|
+
|
|
20
|
+
# Create a new suggestion cache.
|
|
21
|
+
#
|
|
22
|
+
# @param max_size [Integer] Maximum number of entries (default: 5000)
|
|
23
|
+
def initialize(max_size: DEFAULT_MAX_SIZE)
|
|
24
|
+
super
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Write suggestions to cache.
|
|
28
|
+
#
|
|
29
|
+
# @param word [String] The misspelled word
|
|
30
|
+
# @param suggestions [Array<String>] Suggested words
|
|
31
|
+
# @param max_results [Integer] Max results used for this query
|
|
32
|
+
# @return [Array<String>] The stored suggestions
|
|
33
|
+
def write(word, suggestions, max_results: 10)
|
|
34
|
+
cache_key = cache_key_for(word, max_results)
|
|
35
|
+
super(cache_key, suggestions)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Read suggestions from cache.
|
|
39
|
+
#
|
|
40
|
+
# @param word [String] The misspelled word
|
|
41
|
+
# @param max_results [Integer] Max results used for this query
|
|
42
|
+
# @return [Array<String>, nil] Cached suggestions or nil
|
|
43
|
+
def read(word, max_results: 10)
|
|
44
|
+
cache_key = cache_key_for(word, max_results)
|
|
45
|
+
super(cache_key)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Fetch suggestions from cache or compute them.
|
|
49
|
+
#
|
|
50
|
+
# @param word [String] The misspelled word
|
|
51
|
+
# @param max_results [Integer] Max results for this query
|
|
52
|
+
# @yield Block to compute suggestions on cache miss
|
|
53
|
+
# @return [Array<String>] Cached or computed suggestions
|
|
54
|
+
def fetch(word, max_results: 10)
|
|
55
|
+
cache_key = cache_key_for(word, max_results)
|
|
56
|
+
|
|
57
|
+
if @data.key?(cache_key)
|
|
58
|
+
record_hit
|
|
59
|
+
@access_order += 1
|
|
60
|
+
@data[cache_key][1] = @access_order # Update access order
|
|
61
|
+
@data[cache_key][0] # Return value
|
|
62
|
+
else
|
|
63
|
+
record_miss
|
|
64
|
+
suggestions = yield
|
|
65
|
+
write(word, suggestions, max_results: max_results)
|
|
66
|
+
suggestions
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Delete suggestions from cache.
|
|
71
|
+
#
|
|
72
|
+
# @param word [String] The misspelled word
|
|
73
|
+
# @param max_results [Integer] Max results for this query
|
|
74
|
+
# @return [Array<String>, nil] Deleted suggestions or nil
|
|
75
|
+
def delete(word, max_results: 10)
|
|
76
|
+
cache_key = cache_key_for(word, max_results)
|
|
77
|
+
super(cache_key)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Check if suggestions are cached for this word.
|
|
81
|
+
#
|
|
82
|
+
# @param word [String] The misspelled word
|
|
83
|
+
# @param max_results [Integer] Max results for this query
|
|
84
|
+
# @return [Boolean] True if cached
|
|
85
|
+
def key?(word, max_results: 10)
|
|
86
|
+
cache_key = cache_key_for(word, max_results)
|
|
87
|
+
super(cache_key)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Invalidate all cached suggestions for a word.
|
|
91
|
+
#
|
|
92
|
+
# @param word [String] The word to invalidate
|
|
93
|
+
# @return [self] Self for chaining
|
|
94
|
+
def invalidate_word(word)
|
|
95
|
+
# Find and delete all cache entries for this word
|
|
96
|
+
keys_to_delete = @data.keys.select { |key| key.start_with?("#{word}|") }
|
|
97
|
+
keys_to_delete.each { |key| @data.delete(key) }
|
|
98
|
+
self
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
# Generate cache key for word + max_results.
|
|
104
|
+
#
|
|
105
|
+
# @param word [String] The word
|
|
106
|
+
# @param max_results [Integer] Max results
|
|
107
|
+
# @return [String] Cache key
|
|
108
|
+
def cache_key_for(word, max_results)
|
|
109
|
+
"#{word.downcase}|#{max_results}"
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Cache module for Kotoshu
|
|
5
|
+
#
|
|
6
|
+
# This module provides access to various cache implementations for
|
|
7
|
+
# dictionaries, models, and other resources.
|
|
8
|
+
#
|
|
9
|
+
# @example Using the language cache
|
|
10
|
+
# cache = Kotoshu::Cache::LanguageCache.new
|
|
11
|
+
# dict = cache.get_spelling('en')
|
|
12
|
+
# # => { dic_path: "~/.cache/kotoshu/languages/en/spelling/index.dic",
|
|
13
|
+
# # aff_path: "~/.cache/kotoshu/languages/en/spelling/index.aff",
|
|
14
|
+
# # metadata: { ... } }
|
|
15
|
+
#
|
|
16
|
+
module Cache
|
|
17
|
+
autoload :BaseCache, "kotoshu/cache/base_cache"
|
|
18
|
+
autoload :Cache, "kotoshu/cache/cache"
|
|
19
|
+
autoload :EvictionPolicy, "kotoshu/cache/eviction_policy"
|
|
20
|
+
autoload :FrequencyCache, "kotoshu/cache/frequency_cache"
|
|
21
|
+
autoload :LanguageCache, "kotoshu/cache/language_cache"
|
|
22
|
+
autoload :LookupCache, "kotoshu/cache/lookup_cache"
|
|
23
|
+
autoload :ModelCache, "kotoshu/cache/model_cache"
|
|
24
|
+
autoload :ModelRegistry, "kotoshu/cache/model_registry"
|
|
25
|
+
autoload :SuggestionCache, "kotoshu/cache/suggestion_cache"
|
|
26
|
+
# Vendored snapshot of the dictionaries manifest languages
|
|
27
|
+
# (plan 107); refreshed by kotoshu:staged_languages:sync.
|
|
28
|
+
autoload :StagedLanguages, "kotoshu/cache/staged_languages"
|
|
29
|
+
|
|
30
|
+
class << self
|
|
31
|
+
# Create a new language cache instance
|
|
32
|
+
#
|
|
33
|
+
# @param cache_path [String] optional custom cache directory
|
|
34
|
+
# @param url_base [String] optional custom GitHub URL
|
|
35
|
+
# @return [LanguageCache] new language cache instance
|
|
36
|
+
def language_cache(cache_path: nil, url_base: nil)
|
|
37
|
+
LanguageCache.new(cache_path: cache_path, url_base: url_base)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Create a new model cache instance
|
|
41
|
+
#
|
|
42
|
+
# @param cache_path [String] optional custom cache directory
|
|
43
|
+
# @return [ModelCache] new model cache instance
|
|
44
|
+
def model_cache(cache_path: nil)
|
|
45
|
+
ModelCache.new(cache_path: cache_path)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "io/console"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cli
|
|
7
|
+
# Interactive prompt that wraps the strict two-stage setup/resolve flow
|
|
8
|
+
# for the human-facing CLI.
|
|
9
|
+
#
|
|
10
|
+
# The library API (`Kotoshu.correct?`, `Kotoshu.suggest`) still raises
|
|
11
|
+
# `ResourceNotSetupError` strictly — no surprise downloads on metered
|
|
12
|
+
# networks. This class catches that error in the CLI dispatcher, asks
|
|
13
|
+
# the user once, and retries the original command. Programmatic users
|
|
14
|
+
# never see it.
|
|
15
|
+
#
|
|
16
|
+
# Non-TTY contexts (pipes, CI) and offline mode never prompt. The caller
|
|
17
|
+
# decides how to surface a nil result — the CLI dispatcher raises
|
|
18
|
+
# Errors::ResourceUnavailable so scripts see stable exit codes.
|
|
19
|
+
class AutoSetup
|
|
20
|
+
# @param input [IO] Stdin (or override for tests)
|
|
21
|
+
# @param output [IO] Stderr (or override for tests)
|
|
22
|
+
def initialize(input: $stdin, output: $stderr)
|
|
23
|
+
@input = input
|
|
24
|
+
@output = output
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Prompt the user to set up the missing language.
|
|
28
|
+
#
|
|
29
|
+
# @param error [Kotoshu::ResourceNotSetupError] The error raised by resolve
|
|
30
|
+
# @param want [Array<Symbol>] Resource types to fetch (default [:spelling])
|
|
31
|
+
# @return [String, nil] Language code on success; nil when non-TTY,
|
|
32
|
+
# offline, or user declined.
|
|
33
|
+
def call(error, want: %i[spelling])
|
|
34
|
+
language = error.language
|
|
35
|
+
return nil if skip_prompt?
|
|
36
|
+
|
|
37
|
+
@output.puts prompt_message(language, error.resource_type, want)
|
|
38
|
+
answer = @input.gets&.strip&.downcase
|
|
39
|
+
return nil unless affirmative?(answer)
|
|
40
|
+
|
|
41
|
+
Kotoshu.setup(language, want: want)
|
|
42
|
+
language
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def skip_prompt?
|
|
48
|
+
Kotoshu.configuration.offline || !@input.tty?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def prompt_message(language, resource, want)
|
|
52
|
+
size_hint = size_hint_for(want)
|
|
53
|
+
"Language '#{language}' is not set up (missing #{resource}).\n" \
|
|
54
|
+
"Download now (~#{size_hint} from github.com/kotoshu/dictionaries)? [Y/n]"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def size_hint_for(want)
|
|
58
|
+
case want
|
|
59
|
+
when %i[spelling] then "5 MB"
|
|
60
|
+
when %i[spelling frequency] then "6 MB"
|
|
61
|
+
when %i[spelling frequency model] then "120 MB"
|
|
62
|
+
else "unknown size"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def affirmative?(answer)
|
|
67
|
+
answer.nil? || answer.empty? || answer.start_with?("y")
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "thor"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cli
|
|
7
|
+
# `kotoshu baseline` — CI baseline management (plan 82, Track B),
|
|
8
|
+
# wired as a subcommand in cli.rb.
|
|
9
|
+
class BaselineCommand < Thor
|
|
10
|
+
namespace :baseline
|
|
11
|
+
|
|
12
|
+
class_option :language,
|
|
13
|
+
type: :string,
|
|
14
|
+
default: "auto",
|
|
15
|
+
desc: "Language code (auto, de, en, es, fr, pt, ru)",
|
|
16
|
+
aliases: ["-l"]
|
|
17
|
+
|
|
18
|
+
desc "init FILE [FILE ...]", "Write a baseline of the current findings"
|
|
19
|
+
long_desc <<~DESC
|
|
20
|
+
Checks each FILE (directories and globs expand exactly like
|
|
21
|
+
`kotoshu check` directory mode; inline ignore directives apply)
|
|
22
|
+
and writes the canonical baseline JSON: one entry per (file,
|
|
23
|
+
word) with the misspelling count, stable across reformatting.
|
|
24
|
+
The personal dictionary is NOT consulted - baselines freeze what
|
|
25
|
+
every machine would see, and CI has no personal dictionary.
|
|
26
|
+
Suggestions are skipped: a baseline only needs positions.
|
|
27
|
+
|
|
28
|
+
The baseline only covers real errors present right now; new
|
|
29
|
+
errors still fail `kotoshu check --baseline`.
|
|
30
|
+
|
|
31
|
+
Examples:
|
|
32
|
+
|
|
33
|
+
kotoshu baseline init . # directories and globs expand like check
|
|
34
|
+
kotoshu baseline init src/*.txt --output .kotoshu-baseline.json
|
|
35
|
+
DESC
|
|
36
|
+
method_option :output,
|
|
37
|
+
type: :string,
|
|
38
|
+
default: Kotoshu::Baseline::Store::DEFAULT_FILENAME,
|
|
39
|
+
desc: "Baseline file to write (default: #{Kotoshu::Baseline::Store::DEFAULT_FILENAME})"
|
|
40
|
+
def init(*targets)
|
|
41
|
+
if targets.empty?
|
|
42
|
+
raise Errors::UsageError,
|
|
43
|
+
"Give at least one file: kotoshu baseline init FILE [FILE ...]"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
missing = targets.reject do |target|
|
|
47
|
+
File.directory?(target) || File.file?(target) || !Dir[target].empty?
|
|
48
|
+
end
|
|
49
|
+
unless missing.empty?
|
|
50
|
+
raise Errors::UsageError, "File not found: #{missing.join(' ')}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
files = targets.flat_map { |t| expand_target(t) }
|
|
54
|
+
if files.empty?
|
|
55
|
+
raise Errors::UsageError,
|
|
56
|
+
"No text files found (known extensions: " \
|
|
57
|
+
"#{Cli::DirectoryWalker::TEXT_EXTENSIONS.join(' ')})"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
checks = {}
|
|
61
|
+
files.sort.each do |file|
|
|
62
|
+
text = File.read(file, encoding: Kotoshu.configuration.encoding).scrub
|
|
63
|
+
checks[file] = [run_check(text), text]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
store = Kotoshu::Baseline::Store.from_checks(checks)
|
|
67
|
+
store.save(options[:output])
|
|
68
|
+
total = store.entries.sum(&:count)
|
|
69
|
+
noun = store.entries.size == 1 ? "entry" : "entries"
|
|
70
|
+
error_noun = total == 1 ? "error" : "errors"
|
|
71
|
+
puts "Wrote #{options[:output]}: #{store.entries.size} #{noun} " \
|
|
72
|
+
"(#{total} #{error_noun} baselined)"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
no_commands do
|
|
76
|
+
# Directories expand through the same walker directory mode uses
|
|
77
|
+
# (gitignore-aware, known text extensions); globs expand via
|
|
78
|
+
# Dir; plain files pass through unchanged.
|
|
79
|
+
def expand_target(target)
|
|
80
|
+
if File.directory?(target)
|
|
81
|
+
DirectoryWalker.new.files(target)
|
|
82
|
+
elsif File.file?(target)
|
|
83
|
+
[target]
|
|
84
|
+
else
|
|
85
|
+
Dir[target].select { |f| File.file?(f) }
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Baselines must be machine-independent: the personal dictionary
|
|
90
|
+
# of whichever machine runs init must never leak into the frozen
|
|
91
|
+
# set, or CI (which has none) flags words the baseline never
|
|
92
|
+
# recorded. Disabled for the whole init run.
|
|
93
|
+
def run_check(text)
|
|
94
|
+
previous = Kotoshu.configuration.personal_dictionary
|
|
95
|
+
Kotoshu.configuration.personal_dictionary = false
|
|
96
|
+
Kotoshu.reset_spellchecker
|
|
97
|
+
language = resolve_language(text)
|
|
98
|
+
Kotoshu.spellchecker_for(language).check(text, suggestions: false)
|
|
99
|
+
rescue Kotoshu::ResourceNotSetupError => e
|
|
100
|
+
raise Errors::ResourceUnavailable, e.message
|
|
101
|
+
rescue Kotoshu::DictionaryNotFoundError => e
|
|
102
|
+
raise Errors::ResourceUnavailable, e.message
|
|
103
|
+
ensure
|
|
104
|
+
Kotoshu.configuration.personal_dictionary = previous
|
|
105
|
+
Kotoshu.reset_spellchecker
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def resolve_language(text)
|
|
109
|
+
result = LanguageResolver.new(
|
|
110
|
+
flag_value: options[:language],
|
|
111
|
+
default_language: Kotoshu.configuration.default_language
|
|
112
|
+
).resolve(text: text)
|
|
113
|
+
warn "# #{result.note}" if result.note
|
|
114
|
+
result.language
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|