kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,600 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Two-stage resource resolution.
|
|
5
|
+
#
|
|
6
|
+
# Stage 1 — setup (slow, network-required, explicit):
|
|
7
|
+
# Kotoshu.setup(:en) # download from kotoshu/dictionaries
|
|
8
|
+
# Kotoshu.setup(:en, want: %i[spelling frequency])
|
|
9
|
+
# Kotoshu.setup(:en, aff: "/path/to.en.aff", dic: "/path/to/en.dic") # local files
|
|
10
|
+
# Kotoshu.setup(:en, from: "/path/to/dict/dir/") # local directory
|
|
11
|
+
#
|
|
12
|
+
# Stage 2 — resolve (instant, cache-only, raises on miss):
|
|
13
|
+
# bundle = Kotoshu::ResourceManager.resolve(language: "en")
|
|
14
|
+
# bundle.dictionary # => #<Dictionary::Hunspell ...>
|
|
15
|
+
#
|
|
16
|
+
# The hot path (Kotoshu.correct?, .check, .suggest, .spellchecker_for) calls
|
|
17
|
+
# resolve and lets ResourceNotSetupError propagate. Setup is never implicit.
|
|
18
|
+
class ResourceManager
|
|
19
|
+
# Default `want:` value: spelling only. Everything else (frequency,
|
|
20
|
+
# model) is an explicit opt-in.
|
|
21
|
+
DEFAULT_WANT = %i[spelling].freeze
|
|
22
|
+
|
|
23
|
+
# The Stage-1 result: per-resource status for one language's
|
|
24
|
+
# setup run. A resource that was not requested is nil; one that
|
|
25
|
+
# was requested but failed is :unavailable.
|
|
26
|
+
#
|
|
27
|
+
# Statuses: :downloaded | :local | :cached | :unavailable | nil.
|
|
28
|
+
# The most common ones, as a quick reference:
|
|
29
|
+
#
|
|
30
|
+
# result = Kotoshu.setup(:en, want: %i[spelling frequency model])
|
|
31
|
+
# result.spelling # => :downloaded (or :cached on re-run)
|
|
32
|
+
# result.frequency # => :downloaded | :unavailable (soft dep)
|
|
33
|
+
# result.model # => :downloaded | :cached | :unavailable
|
|
34
|
+
# result.model_tier # => :full | :fluency | :mini | nil
|
|
35
|
+
# result.source # => :kotoshu | :local
|
|
36
|
+
#
|
|
37
|
+
# See also {ResourceBundle} — the Stage-2 result.
|
|
38
|
+
SetupResult = Struct.new(
|
|
39
|
+
:language, # [String] Language code that was set up
|
|
40
|
+
:spelling, # :downloaded | :local | :cached | nil
|
|
41
|
+
:frequency, # :downloaded | :local | :cached | :unavailable | nil
|
|
42
|
+
:model, # :downloaded | :cached | :unavailable | nil
|
|
43
|
+
:model_tier, # :full | :fluency | :mini | nil (tier actually set up)
|
|
44
|
+
:source, # :kotoshu | :local
|
|
45
|
+
keyword_init: true
|
|
46
|
+
) do
|
|
47
|
+
# Whether the setup run produced anything usable: at least one
|
|
48
|
+
# of spelling/frequency succeeded. A nil everywhere means the
|
|
49
|
+
# run failed outright (network, integrity) — with `strict:`,
|
|
50
|
+
# such failures raise instead.
|
|
51
|
+
#
|
|
52
|
+
# @return [Boolean]
|
|
53
|
+
def success?
|
|
54
|
+
!spelling.nil? || !frequency.nil?
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
class << self
|
|
59
|
+
# Stage 1, class-level convenience. Delegates to a new instance;
|
|
60
|
+
# see the instance method for parameters and semantics.
|
|
61
|
+
#
|
|
62
|
+
# @param language [String, Symbol]
|
|
63
|
+
# @param want [Array<Symbol>] Resource types (default: [:spelling])
|
|
64
|
+
# @param opts [Hash] Forwarded to {#setup} (force:, strict:,
|
|
65
|
+
# tier:, aff:, dic:, from:, frequency:)
|
|
66
|
+
# @return [SetupResult]
|
|
67
|
+
# @see #setup
|
|
68
|
+
# @see Kotoshu.setup
|
|
69
|
+
def setup(language, want: DEFAULT_WANT, force: false, strict: false, **opts)
|
|
70
|
+
new.setup(language: language, want: want, force: force, strict: strict, **opts)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Stage 1 from local files, class-level convenience. Delegates to
|
|
74
|
+
# a new instance; see the instance method.
|
|
75
|
+
#
|
|
76
|
+
# @param language [String, Symbol]
|
|
77
|
+
# @param aff [String] Path to the local .aff file
|
|
78
|
+
# @param dic [String] Path to the local .dic file
|
|
79
|
+
# @param frequency [String, nil] Optional path to frequency.json
|
|
80
|
+
# @param force [Boolean] Re-register even if already cached
|
|
81
|
+
# @return [SetupResult]
|
|
82
|
+
# @raise [ArgumentError] when a file is missing
|
|
83
|
+
# @see #setup_from_local
|
|
84
|
+
def setup_from_local(language:, aff:, dic:, frequency: nil, force: false)
|
|
85
|
+
new.setup_from_local(language: language, aff: aff, dic: dic, frequency: frequency, force: force)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Stage 2, class-level convenience. Delegates to a new instance;
|
|
89
|
+
# see the instance method for the tier contract.
|
|
90
|
+
#
|
|
91
|
+
# @param language [String, Symbol]
|
|
92
|
+
# @param want [Array<Symbol>] Resource types (default: [:spelling])
|
|
93
|
+
# @param tier [Symbol, nil] Model tier to resolve
|
|
94
|
+
# @return [ResourceBundle]
|
|
95
|
+
# @raise [ResourceNotSetupError] when a wanted resource is not cached
|
|
96
|
+
# @see #resolve
|
|
97
|
+
def resolve(language:, want: DEFAULT_WANT, tier: nil)
|
|
98
|
+
new.resolve(language: language, want: want, tier: tier)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Predicate, class-level convenience. See {#setup?}.
|
|
102
|
+
#
|
|
103
|
+
# @param language [String, Symbol]
|
|
104
|
+
# @param resource [Symbol, nil] :spelling, :frequency, :model, nil for any
|
|
105
|
+
# @param tier [Symbol, nil] Model tier to check (:any = any tier)
|
|
106
|
+
# @return [Boolean]
|
|
107
|
+
def setup?(language, resource: nil, tier: nil)
|
|
108
|
+
new.setup?(language, resource: resource, tier: tier)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Languages with a cached spelling dictionary, sorted.
|
|
112
|
+
#
|
|
113
|
+
# @return [Array<String>]
|
|
114
|
+
# @see Kotoshu.languages_setup
|
|
115
|
+
def languages_setup
|
|
116
|
+
new.languages_setup
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# ---- Stage 1: setup ----
|
|
121
|
+
|
|
122
|
+
# Download or register resources for one language (slow,
|
|
123
|
+
# network-required when remote, explicit). Idempotent: re-running
|
|
124
|
+
# with the same args is a no-op unless `force:` is true.
|
|
125
|
+
#
|
|
126
|
+
# Pass `aff:`/`dic:` or `from:` to register local Hunspell files
|
|
127
|
+
# instead of downloading (single-language only). See
|
|
128
|
+
# {Kotoshu.setup} — the facade takes the same options and fans
|
|
129
|
+
# them out over multiple languages.
|
|
130
|
+
#
|
|
131
|
+
# @param language [String, Symbol] Language code
|
|
132
|
+
# @param want [Array<Symbol>] Resource types (default: [:spelling])
|
|
133
|
+
# @param force [Boolean] Re-fetch even if already cached
|
|
134
|
+
# @param strict [Boolean] Re-raise on optional-resource failure
|
|
135
|
+
# (frequency/model); without it, failures degrade to
|
|
136
|
+
# :unavailable with a warning
|
|
137
|
+
# @param tier [Symbol, String, nil] Model tier when want includes
|
|
138
|
+
# :model (:full default, :fluency, :mini)
|
|
139
|
+
# @param aff [String, nil] Path to local .aff file
|
|
140
|
+
# @param dic [String, nil] Path to local .dic file
|
|
141
|
+
# @param from [String, nil] Directory containing lang.aff/lang.dic
|
|
142
|
+
# @param frequency [String, nil] Path to local frequency.json
|
|
143
|
+
# @return [SetupResult]
|
|
144
|
+
# @raise [ArgumentError] when local files are named but missing
|
|
145
|
+
def setup(language:, want: DEFAULT_WANT, force: false, strict: false, tier: nil,
|
|
146
|
+
aff: nil, dic: nil, from: nil, frequency: nil)
|
|
147
|
+
lang = normalize_language(language)
|
|
148
|
+
|
|
149
|
+
if aff || dic || from
|
|
150
|
+
setup_from_local(language: lang, aff: aff, dic: dic, from: from,
|
|
151
|
+
frequency: frequency, force: force)
|
|
152
|
+
else
|
|
153
|
+
setup_from_remote(lang, want: want, force: force, strict: strict, tier: tier)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Register local Hunspell files as the language's cached spelling
|
|
158
|
+
# dictionary (no network). `from:` looks for `lang.aff` and
|
|
159
|
+
# `lang.dic` inside the directory; explicit `aff:`/`dic:` paths win.
|
|
160
|
+
#
|
|
161
|
+
# @param language [String, Symbol] Language code
|
|
162
|
+
# @param aff [String, nil] Path to the .aff file
|
|
163
|
+
# @param dic [String, nil] Path to the .dic file
|
|
164
|
+
# @param from [String, nil] Directory to find lang.aff/lang.dic in
|
|
165
|
+
# @param frequency [String, nil] Optional path to frequency.json
|
|
166
|
+
# @param force [Boolean] Re-register even if already cached
|
|
167
|
+
# @return [SetupResult]
|
|
168
|
+
# @raise [ArgumentError] when a file is missing
|
|
169
|
+
#
|
|
170
|
+
# @example
|
|
171
|
+
# ResourceManager.setup_from_local(
|
|
172
|
+
# language: "en",
|
|
173
|
+
# aff: "/usr/share/hunspell/en_US.aff",
|
|
174
|
+
# dic: "/usr/share/hunspell/en_US.dic"
|
|
175
|
+
# ).source # => :local
|
|
176
|
+
def setup_from_local(language:, aff:, dic:, from: nil, frequency: nil, force: false)
|
|
177
|
+
lang = normalize_language(language)
|
|
178
|
+
|
|
179
|
+
aff_path, dic_path = resolve_local_paths(lang, aff: aff, dic: dic, from: from)
|
|
180
|
+
raise ArgumentError, "aff file not found: #{aff_path}" unless File.exist?(aff_path)
|
|
181
|
+
raise ArgumentError, "dic file not found: #{dic_path}" unless File.exist?(dic_path)
|
|
182
|
+
|
|
183
|
+
spelling_cache = spelling_cache_for(lang)
|
|
184
|
+
spelling_cache.install_local(lang, aff: aff_path, dic: dic_path, force: force)
|
|
185
|
+
spelling_status = :local
|
|
186
|
+
|
|
187
|
+
frequency_status = nil
|
|
188
|
+
if frequency
|
|
189
|
+
raise ArgumentError, "frequency file not found: #{frequency}" unless File.exist?(frequency)
|
|
190
|
+
|
|
191
|
+
freq_cache = frequency_cache_for
|
|
192
|
+
freq_cache.install_local(lang, path: frequency, force: force)
|
|
193
|
+
frequency_status = :local
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
SetupResult.new(
|
|
197
|
+
language: lang,
|
|
198
|
+
spelling: spelling_status,
|
|
199
|
+
frequency: frequency_status,
|
|
200
|
+
model: nil,
|
|
201
|
+
source: :local
|
|
202
|
+
)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# ---- Stage 2: resolve (cache-only) ----
|
|
206
|
+
#
|
|
207
|
+
# `tier:` selects which cached model tier to resolve. Default: the
|
|
208
|
+
# configured `model_tier` ("fluency" — owner decision 2026-09-04).
|
|
209
|
+
# A missing tier raises ResourceNotSetupError exactly like any
|
|
210
|
+
# other unset resource — resolve never downloads.
|
|
211
|
+
#
|
|
212
|
+
# Tier-less fallback (legacy-cache bridge): when `tier:` is NOT
|
|
213
|
+
# given and the configured tier is not cached, exactly one cached
|
|
214
|
+
# tier satisfies the request. Caches written before the fluency
|
|
215
|
+
# default hold the legacy `full` layout; without this bridge a
|
|
216
|
+
# tier-less resolve would raise for them. An explicit `tier:`
|
|
217
|
+
# never falls back, and zero or multiple cached tiers raise the
|
|
218
|
+
# missing-tier error for the configured tier.
|
|
219
|
+
#
|
|
220
|
+
# `tier: :any` (explicit opt-in only) maps to the single cached
|
|
221
|
+
# tier; with zero or multiple cached tiers it raises.
|
|
222
|
+
#
|
|
223
|
+
# @param language [String, Symbol] Language code
|
|
224
|
+
# @param want [Array<Symbol>] Resource types to resolve
|
|
225
|
+
# (default: [:spelling])
|
|
226
|
+
# @param tier [Symbol, nil] Model tier to resolve (:full, :fluency,
|
|
227
|
+
# :mini, or :any for "the single cached tier")
|
|
228
|
+
# @return [ResourceBundle] Bundle with a member per resolved
|
|
229
|
+
# resource; unrequested resources are nil
|
|
230
|
+
# @raise [ResourceNotSetupError] when a wanted resource is not
|
|
231
|
+
# cached — resolve never downloads
|
|
232
|
+
# @raise [ResourceResolutionError] when multiple model tiers are
|
|
233
|
+
# cached and `tier: :any` cannot disambiguate
|
|
234
|
+
def resolve(language:, want: DEFAULT_WANT, tier: nil)
|
|
235
|
+
lang = normalize_language(language)
|
|
236
|
+
effective_tier = effective_tier(tier)
|
|
237
|
+
|
|
238
|
+
spelling_dict = want.include?(:spelling) ? resolve_spelling_cached(lang) : nil
|
|
239
|
+
frequency_data = want.include?(:frequency) ? resolve_frequency_cached(lang) : nil
|
|
240
|
+
model = if want.include?(:model)
|
|
241
|
+
resolve_model_cached(lang, tier: effective_tier, from_default: tier.nil?)
|
|
242
|
+
else
|
|
243
|
+
nil
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
ResourceBundle.new(
|
|
247
|
+
language: lang,
|
|
248
|
+
dictionary: spelling_dict,
|
|
249
|
+
frequency: frequency_data,
|
|
250
|
+
model: model,
|
|
251
|
+
rules: nil,
|
|
252
|
+
cached: true,
|
|
253
|
+
source_urls: []
|
|
254
|
+
)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# ---- Predicates ----
|
|
258
|
+
|
|
259
|
+
# Whether a language (or a specific resource for it) is already
|
|
260
|
+
# cached and resolvable. Mirrors the tier-less resolve contract
|
|
261
|
+
# for models: with no `tier:`, the configured tier counts plus a
|
|
262
|
+
# single cached tier of any kind.
|
|
263
|
+
#
|
|
264
|
+
# @param language [String, Symbol] Language code
|
|
265
|
+
# @param resource [Symbol, nil] :spelling (default), :frequency,
|
|
266
|
+
# :model, or nil for "spelling"
|
|
267
|
+
# @param tier [Symbol, nil] Model tier to check (:full, :fluency,
|
|
268
|
+
# :mini, or :any); only meaningful with resource: :model
|
|
269
|
+
# @return [Boolean]
|
|
270
|
+
#
|
|
271
|
+
# @example
|
|
272
|
+
# ResourceManager.setup?(:en) # spelling
|
|
273
|
+
# ResourceManager.setup?(:en, resource: :frequency) # Kelly list
|
|
274
|
+
# ResourceManager.setup?(:en, resource: :model, tier: :mini)
|
|
275
|
+
def setup?(language, resource: nil, tier: nil)
|
|
276
|
+
lang = normalize_language(language)
|
|
277
|
+
case resource&.to_sym
|
|
278
|
+
when nil, :spelling
|
|
279
|
+
spelling_cache_for(lang).available?("#{lang}:spelling")
|
|
280
|
+
when :frequency
|
|
281
|
+
fc = frequency_cache_for
|
|
282
|
+
fc.supports_resource?(lang) && fc.available?(lang)
|
|
283
|
+
when :model
|
|
284
|
+
cache = model_cache_for
|
|
285
|
+
if tier.nil?
|
|
286
|
+
# Mirrors the tier-less resolve contract: the configured
|
|
287
|
+
# tier counts, and so does exactly one cached tier of any
|
|
288
|
+
# kind (legacy single-tier caches).
|
|
289
|
+
cache.available?(cache.tier_resource_id(lang, effective_tier(nil))) ||
|
|
290
|
+
cache.cached_tiers(lang).size == 1
|
|
291
|
+
elsif tier.to_sym == :any
|
|
292
|
+
cache.cached_tiers(lang).any?
|
|
293
|
+
else
|
|
294
|
+
cache.available?(cache.tier_resource_id(lang, tier))
|
|
295
|
+
end
|
|
296
|
+
else
|
|
297
|
+
false
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Languages with a cached spelling dictionary, sorted.
|
|
302
|
+
#
|
|
303
|
+
# @return [Array<String>] e.g. ["de", "en", "fr"]
|
|
304
|
+
def languages_setup
|
|
305
|
+
spelling_cache_for(nil).cached_resources
|
|
306
|
+
.map { |r| r.to_s.split(":").first }
|
|
307
|
+
.uniq
|
|
308
|
+
.sort
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
private
|
|
312
|
+
|
|
313
|
+
def setup_from_remote(lang, want:, force:, strict:, tier: nil)
|
|
314
|
+
config = Configuration.instance
|
|
315
|
+
spelling_status = nil
|
|
316
|
+
frequency_status = nil
|
|
317
|
+
model_status = nil
|
|
318
|
+
model_tier = nil
|
|
319
|
+
|
|
320
|
+
if want.include?(:spelling)
|
|
321
|
+
cache = spelling_cache_for(lang, config: config)
|
|
322
|
+
was_cached = cache.available?("#{lang}:spelling")
|
|
323
|
+
if was_cached && !force
|
|
324
|
+
spelling_status = :cached
|
|
325
|
+
else
|
|
326
|
+
warn "[#{lang}] downloading spelling dictionary..." unless quiet?
|
|
327
|
+
cache.get_spelling(lang, force_download: force)
|
|
328
|
+
spelling_status = :downloaded
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
if want.include?(:frequency)
|
|
333
|
+
frequency_status = setup_frequency_remote(lang, force: force, strict: strict, config: config)
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
if want.include?(:model)
|
|
337
|
+
model_tier = effective_tier(tier, config: config)
|
|
338
|
+
model_status = setup_model_remote(lang, want: want, force: force, strict: strict, config: config,
|
|
339
|
+
tier: model_tier, from_default: tier.nil?)
|
|
340
|
+
if model_status == :cached
|
|
341
|
+
model_tier = setup_model_remote_satisfied_tier(lang, config: config, requested: model_tier,
|
|
342
|
+
from_default: tier.nil?)
|
|
343
|
+
end
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
SetupResult.new(
|
|
347
|
+
language: lang,
|
|
348
|
+
spelling: spelling_status,
|
|
349
|
+
frequency: frequency_status,
|
|
350
|
+
model: model_status,
|
|
351
|
+
model_tier: model_tier,
|
|
352
|
+
source: :kotoshu
|
|
353
|
+
)
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def setup_frequency_remote(lang, force:, strict:, config:)
|
|
357
|
+
cache = frequency_cache_for(config: config)
|
|
358
|
+
return :unavailable unless cache.supports_resource?(lang)
|
|
359
|
+
|
|
360
|
+
was_cached = cache.available?(lang)
|
|
361
|
+
return :cached if was_cached && !force
|
|
362
|
+
|
|
363
|
+
warn "[#{lang}] downloading frequency data..." unless quiet?
|
|
364
|
+
cache.get(lang, force_download: force)
|
|
365
|
+
:downloaded
|
|
366
|
+
rescue StandardError => e
|
|
367
|
+
raise if strict
|
|
368
|
+
|
|
369
|
+
warn "[#{lang}] frequency data unavailable: #{e.class} (#{e.message})" unless quiet?
|
|
370
|
+
:unavailable
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# The tier a :cached status actually satisfied: the requested tier
|
|
374
|
+
# when it is cached, else the single cached tier the tier-less
|
|
375
|
+
# bridge matched. SetupResult reports the truth, not the request.
|
|
376
|
+
def setup_model_remote_satisfied_tier(lang, config:, requested:, from_default:)
|
|
377
|
+
cache = model_cache_for(config: config)
|
|
378
|
+
requested = Cache::ModelCache.normalize_tier(requested)
|
|
379
|
+
return requested if cache.available?(cache.tier_resource_id(lang, requested))
|
|
380
|
+
|
|
381
|
+
cached = cache.cached_tiers(lang) if from_default
|
|
382
|
+
cached&.size == 1 ? cached.first : requested
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Set up the ONNX model for `lang` at `tier`.
|
|
386
|
+
#
|
|
387
|
+
# full: today's behavior, byte for byte — AVAILABLE_MODELS gate,
|
|
388
|
+
# git-tree URL, download-or-convert fallback. fluency/mini: the
|
|
389
|
+
# registry-driven path with sha256 verification against the
|
|
390
|
+
# registry entry. In both cases a cached model short-circuits to
|
|
391
|
+
# :cached BEFORE any registry consultation, so offline hosts with
|
|
392
|
+
# a warm cache (but no cached registry) still work.
|
|
393
|
+
def setup_model_remote(lang, want:, force:, strict:, config:, tier: :full, from_default: false)
|
|
394
|
+
tier = Cache::ModelCache.normalize_tier(tier)
|
|
395
|
+
cache = model_cache_for(config: config)
|
|
396
|
+
resource_id = cache.tier_resource_id(lang, tier)
|
|
397
|
+
|
|
398
|
+
if tier == :full && !Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym)
|
|
399
|
+
return :unavailable
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
was_cached = cache.available?(resource_id)
|
|
403
|
+
return :cached if was_cached && !force
|
|
404
|
+
|
|
405
|
+
# Tier-less bridge (mirrors resolve_model_cached): a cache written
|
|
406
|
+
# before the fluency default holds only the legacy full layout;
|
|
407
|
+
# its owner never named a tier. Exactly one cached tier satisfies
|
|
408
|
+
# the request; explicit tiers never fall back.
|
|
409
|
+
if from_default && !force
|
|
410
|
+
cached_tiers = cache.cached_tiers(lang)
|
|
411
|
+
if cached_tiers.size == 1
|
|
412
|
+
legacy_id = cache.tier_resource_id(lang, cached_tiers.first)
|
|
413
|
+
return :cached if cache.available?(legacy_id)
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
if tier != :full && cache.registry_entry_for(lang, tier).nil?
|
|
418
|
+
warn "[#{lang}] no registry entry for tier #{tier}; available: #{tiers_for_language(cache, lang).join(', ')}" unless quiet?
|
|
419
|
+
return :unavailable
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
warn "[#{lang}] downloading ONNX model (#{tier} tier)..." unless quiet?
|
|
423
|
+
if tier == :full
|
|
424
|
+
cache.get(resource_id, force_download: force)
|
|
425
|
+
else
|
|
426
|
+
cache.download_tiered_model(lang, tier: tier, force_download: force)
|
|
427
|
+
end
|
|
428
|
+
:downloaded
|
|
429
|
+
rescue StandardError => e
|
|
430
|
+
raise if strict
|
|
431
|
+
|
|
432
|
+
warn "[#{lang}] ONNX model unavailable: #{e.class} (#{e.message})" unless quiet?
|
|
433
|
+
:unavailable
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def resolve_spelling_cached(lang)
|
|
437
|
+
cache = spelling_cache_for(lang)
|
|
438
|
+
resource_id = "#{lang}:spelling"
|
|
439
|
+
raise ResourceNotSetupError.new(lang, "spelling") unless cache.available?(resource_id)
|
|
440
|
+
|
|
441
|
+
# Cache-only by construction: resolve must never download, so read
|
|
442
|
+
# via load_cached instead of the download-on-miss get.
|
|
443
|
+
result = cache.load_cached(resource_id)
|
|
444
|
+
raise ResourceNotSetupError.new(lang, "spelling") unless result
|
|
445
|
+
|
|
446
|
+
Dictionary::Hunspell.new(
|
|
447
|
+
dic_path: result[:dic_path] || result["dic_path"],
|
|
448
|
+
aff_path: result[:aff_path] || result["aff_path"],
|
|
449
|
+
language_code: lang
|
|
450
|
+
)
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
def resolve_frequency_cached(lang)
|
|
454
|
+
cache = frequency_cache_for
|
|
455
|
+
return nil unless cache.supports_resource?(lang)
|
|
456
|
+
raise ResourceNotSetupError.new(lang, "frequency") unless cache.available?(lang)
|
|
457
|
+
|
|
458
|
+
begin
|
|
459
|
+
cache.load_cached(lang)
|
|
460
|
+
rescue StandardError
|
|
461
|
+
nil
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
# Resolve the cached model for `lang` at `tier` (cache-only —
|
|
466
|
+
# never downloads, never consults the network-side registry).
|
|
467
|
+
#
|
|
468
|
+
# :any maps to the single cached tier; zero or multiple cached
|
|
469
|
+
# tiers raise (deterministic: ambiguity errors list tiers in
|
|
470
|
+
# Cache::ModelCache::TIER_PREFERENCE order — mini, fluency, full —
|
|
471
|
+
# which is a reporting order, not a fallback chain).
|
|
472
|
+
#
|
|
473
|
+
# `from_default` marks a tier-less call (the tier came from the
|
|
474
|
+
# configuration, not an argument). For those calls only, a missing
|
|
475
|
+
# configured tier falls back to the single cached tier: caches
|
|
476
|
+
# written before the fluency default (owner decision 2026-09-04)
|
|
477
|
+
# hold the legacy full layout, and their owners never asked for a
|
|
478
|
+
# tier by name. Zero or multiple cached tiers keep the normal
|
|
479
|
+
# missing-tier error. Explicit tiers and :any never fall back.
|
|
480
|
+
def resolve_model_cached(lang, tier: :full, from_default: false)
|
|
481
|
+
cache = model_cache_for
|
|
482
|
+
return nil unless model_language_supported?(lang, cache)
|
|
483
|
+
|
|
484
|
+
if tier.to_sym == :any
|
|
485
|
+
cached = cache.cached_tiers(lang)
|
|
486
|
+
case cached.size
|
|
487
|
+
when 1
|
|
488
|
+
return resolve_tier_cached(lang, cached.first, cache)
|
|
489
|
+
when 0
|
|
490
|
+
raise ResourceNotSetupError.new(lang, "model (no tier cached)")
|
|
491
|
+
else
|
|
492
|
+
raise ResourceResolutionError.new(
|
|
493
|
+
lang,
|
|
494
|
+
"multiple model tiers cached (#{cached.join(', ')}); " \
|
|
495
|
+
"pass tier: one of #{cached.join('|')} explicitly"
|
|
496
|
+
)
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
requested = Cache::ModelCache.normalize_tier(tier)
|
|
501
|
+
return resolve_tier_cached(lang, requested, cache) if cache.available?(cache.tier_resource_id(lang, requested))
|
|
502
|
+
|
|
503
|
+
if from_default
|
|
504
|
+
cached = cache.cached_tiers(lang)
|
|
505
|
+
return resolve_tier_cached(lang, cached.first, cache) if cached.size == 1
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
resolve_tier_cached(lang, requested, cache)
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
def resolve_tier_cached(lang, tier, cache)
|
|
512
|
+
resource_id = cache.tier_resource_id(lang, tier)
|
|
513
|
+
missing = tier == :full ? "model" : "model tier '#{tier}'"
|
|
514
|
+
raise ResourceNotSetupError.new(lang, missing) unless cache.available?(resource_id)
|
|
515
|
+
|
|
516
|
+
begin
|
|
517
|
+
cache.load_cached(resource_id)
|
|
518
|
+
rescue StandardError
|
|
519
|
+
nil
|
|
520
|
+
end
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# Whether `lang` can have a model at all: listed in AVAILABLE_MODELS
|
|
524
|
+
# (full-tier gate, unchanged) or any tier already cached on disk.
|
|
525
|
+
def model_language_supported?(lang, cache)
|
|
526
|
+
Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym) ||
|
|
527
|
+
cache.cached_tiers(lang).any?
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
# The tier a tier-less call means: an explicit argument wins, then
|
|
531
|
+
# the configured `model_tier` (default "fluency" — owner decision
|
|
532
|
+
# 2026-09-04; "full" and "mini" are explicit opt-ins).
|
|
533
|
+
def effective_tier(tier, config: nil)
|
|
534
|
+
return :any if tier && tier.to_sym == :any
|
|
535
|
+
|
|
536
|
+
Cache::ModelCache.normalize_tier(tier || (config || Configuration.instance).model_tier)
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
# Registry tiers available for a language (for diagnostics; may
|
|
540
|
+
# consult the cached registry, never used on the resolve path).
|
|
541
|
+
def tiers_for_language(cache, lang)
|
|
542
|
+
Cache::ModelCache::TIERS.select do |t|
|
|
543
|
+
t == :full ? Cache::ModelCache::AVAILABLE_MODELS[:onnx].key?(lang.to_sym) : cache.registry_entry_for(lang, t)
|
|
544
|
+
end
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
def resolve_local_paths(lang, aff:, dic:, from:)
|
|
548
|
+
if from
|
|
549
|
+
dir = File.expand_path(from)
|
|
550
|
+
aff_path = aff || File.join(dir, "#{lang}.aff")
|
|
551
|
+
dic_path = dic || File.join(dir, "#{lang}.dic")
|
|
552
|
+
[aff_path, dic_path]
|
|
553
|
+
else
|
|
554
|
+
[File.expand_path(aff), File.expand_path(dic)]
|
|
555
|
+
end
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
# Collapse a user-given code to its cache key: lowercase base
|
|
559
|
+
# language, plus the script subtag when one is present. Region
|
|
560
|
+
# variants share their base dictionary (en-GB -> en, sv-FI -> sv),
|
|
561
|
+
# but a BCP-47 script subtag (four letters: Latn in sr-Latn and
|
|
562
|
+
# tlh-Latn) names a dictionary the manifest stages separately, so
|
|
563
|
+
# it survives normalization (plan 110).
|
|
564
|
+
def normalize_language(code)
|
|
565
|
+
parts = code.to_s.split(/[-_]/)
|
|
566
|
+
base = parts.first.downcase
|
|
567
|
+
subtag = parts[1]
|
|
568
|
+
return base unless subtag&.match?(/\A[A-Za-z]{4}\z/)
|
|
569
|
+
|
|
570
|
+
"#{base}-#{subtag.downcase.capitalize}"
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
def spelling_cache_for(_lang = nil, config: nil)
|
|
574
|
+
cfg = config || Configuration.instance
|
|
575
|
+
Cache::LanguageCache.new(
|
|
576
|
+
cache_path: cfg.cache_path,
|
|
577
|
+
resource_pin: cfg.resource_pin
|
|
578
|
+
)
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
def frequency_cache_for(config: nil)
|
|
582
|
+
cfg = config || Configuration.instance
|
|
583
|
+
Cache::FrequencyCache.new(
|
|
584
|
+
cache_path: cfg.cache_path,
|
|
585
|
+
resource_pin: cfg.resource_pin
|
|
586
|
+
)
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
def model_cache_for(config: nil)
|
|
590
|
+
cfg = config || Configuration.instance
|
|
591
|
+
Cache::ModelCache.new(
|
|
592
|
+
cache_path: cfg.cache_path
|
|
593
|
+
)
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
def quiet?
|
|
597
|
+
!$stderr.tty? || ENV["KOTOSHU_QUIET"] == "1"
|
|
598
|
+
end
|
|
599
|
+
end
|
|
600
|
+
end
|