kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,1147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# base_cache autoloaded via Kotoshu::Cache
|
|
4
|
+
require "zlib"
|
|
5
|
+
require "open-uri"
|
|
6
|
+
require "open3"
|
|
7
|
+
|
|
8
|
+
module Kotoshu
|
|
9
|
+
module Cache
|
|
10
|
+
# Manages embedding model downloads from FastText CDN and GitHub.
|
|
11
|
+
#
|
|
12
|
+
# Extends BaseCache to support FastText .vec files and ONNX models.
|
|
13
|
+
# Downloads FastText models from Facebook's public CDN.
|
|
14
|
+
#
|
|
15
|
+
# @example Downloading a FastText model
|
|
16
|
+
# cache = ModelCache.new
|
|
17
|
+
# vec_file = cache.get_fasttext_model('en')
|
|
18
|
+
# model = FastTextModel.from_file(vec_file)
|
|
19
|
+
#
|
|
20
|
+
# @example Downloading an ONNX model
|
|
21
|
+
# onnx_file = cache.get_onnx_model('en')
|
|
22
|
+
class ModelCache < BaseCache
|
|
23
|
+
# Available models in FastText CDN and models-fasttext-onnx repository
|
|
24
|
+
AVAILABLE_MODELS = {
|
|
25
|
+
# FastText crawl vectors (300D) from Facebook Research
|
|
26
|
+
# https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/
|
|
27
|
+
# Selected high-resource languages
|
|
28
|
+
fasttext: {
|
|
29
|
+
de: { file: "cc.de.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
|
|
30
|
+
en: { file: "cc.en.300.vec.gz", size: 2_000_000, source: "FastText Common Crawl" },
|
|
31
|
+
es: { file: "cc.es.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
|
|
32
|
+
fr: { file: "cc.fr.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
|
|
33
|
+
pt: { file: "cc.pt.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" },
|
|
34
|
+
ru: { file: "cc.ru.300.vec.gz", size: 1_000_000, source: "FastText Common Crawl" }
|
|
35
|
+
},
|
|
36
|
+
# ONNX models (active set) from models-fasttext-onnx repository.
|
|
37
|
+
# Sizes synced with manifest.json in kotoshu/models-fasttext-onnx.
|
|
38
|
+
# The repo holds .onnx for 158 languages but only the 9 below are
|
|
39
|
+
# tracked and exposed — to promote a language, see
|
|
40
|
+
# models-fasttext-onnx/.gitignore and re-sync this constant.
|
|
41
|
+
# https://github.com/kotoshu/models-fasttext-onnx
|
|
42
|
+
onnx: {
|
|
43
|
+
de: { file: "fasttext.de.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
44
|
+
en: { file: "fasttext.en.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
45
|
+
es: { file: "fasttext.es.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
46
|
+
fr: { file: "fasttext.fr.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
47
|
+
pt: { file: "fasttext.pt.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
48
|
+
ru: { file: "fasttext.ru.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
49
|
+
zh: { file: "fasttext.zh.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
50
|
+
ja: { file: "fasttext.ja.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
51
|
+
ko: { file: "fasttext.ko.onnx", size: 120_000_415, source: "models-fasttext-onnx" },
|
|
52
|
+
}
|
|
53
|
+
}.freeze
|
|
54
|
+
|
|
55
|
+
# Model tiers published by kotoshu/models-fasttext-onnx (registry
|
|
56
|
+
# ids `kotoshu://models/{lang}/{tier}`). `full` is today's layout
|
|
57
|
+
# and behavior; `fluency` and `mini` are the tiered tiers resolved
|
|
58
|
+
# through the registry (TODO.impl/67 M1).
|
|
59
|
+
TIERS = %i[full fluency mini].freeze
|
|
60
|
+
|
|
61
|
+
# First line of a Git LFS pointer stub — what raw.githubusercontent.com
|
|
62
|
+
# serves for LFS-tracked files (*.onnx) instead of the object content.
|
|
63
|
+
# Used to detect pointer-stub downloads/caches by content: the legacy
|
|
64
|
+
# download path records the checksum of whatever it downloaded, so a
|
|
65
|
+
# pointer stub passes checksum verification forever.
|
|
66
|
+
LFS_POINTER_PREFIX = "version https://git-lfs.github.com/spec/v1"
|
|
67
|
+
|
|
68
|
+
# Deterministic tier preference order, smallest footprint first.
|
|
69
|
+
# This is NOT a fallback chain — tiers are never silently
|
|
70
|
+
# substituted. It only fixes the ordering in which cached tiers
|
|
71
|
+
# are reported (e.g. ambiguity errors from `tier: :any`), so
|
|
72
|
+
# output is stable across machines and runs.
|
|
73
|
+
TIER_PREFERENCE = %i[mini fluency full].freeze
|
|
74
|
+
|
|
75
|
+
# The language-identification artifact pair (plan 106): registry
|
|
76
|
+
# resource `kotoshu://models/lid/lid-176` — the ONNX-converted
|
|
77
|
+
# lid.176 model (int8-per-row, dim 16) plus its vocab.json
|
|
78
|
+
# sidecar. Lives in the same registry as the embedding tiers but
|
|
79
|
+
# is NOT a language tier: it has no language of its own and the
|
|
80
|
+
# tier machinery (resource ids, TIERS) does not apply. It is
|
|
81
|
+
# stored under a top-level `lid/` directory (not
|
|
82
|
+
# `{lang}/models/onnx/{tier}`) so {#cached_resources} keeps
|
|
83
|
+
# reporting per-language ids only.
|
|
84
|
+
LID_LANGUAGE = "lid"
|
|
85
|
+
LID_TIER = "lid-176"
|
|
86
|
+
LID_REGISTRY_ID = ModelRegistry::Resource.id_for(LID_LANGUAGE, LID_TIER)
|
|
87
|
+
|
|
88
|
+
# Directory (under the cache root) holding the LID artifact pair.
|
|
89
|
+
LID_DIRECTORY = "lid"
|
|
90
|
+
|
|
91
|
+
# Normalize and validate a tier name.
|
|
92
|
+
#
|
|
93
|
+
# @param tier [String, Symbol] "full", "fluency", or "mini"
|
|
94
|
+
# @return [Symbol] the normalized tier symbol
|
|
95
|
+
# @raise [ArgumentError] for any other value (covers config
|
|
96
|
+
# `model_tier` / `KOTOSHU_MODEL_TIER` and caller typos)
|
|
97
|
+
def self.normalize_tier(tier)
|
|
98
|
+
symbol = tier.to_s.downcase.strip.to_sym
|
|
99
|
+
unless TIERS.include?(symbol)
|
|
100
|
+
raise ArgumentError,
|
|
101
|
+
"unknown model tier #{tier.inspect} (expected one of: #{TIERS.join(', ')})"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
symbol
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Get or download FastText model for a language.
|
|
108
|
+
#
|
|
109
|
+
# @param language_code [String] ISO 639-1 language code
|
|
110
|
+
# @param force_download [Boolean] Force re-download
|
|
111
|
+
# @return [String, nil] Path to downloaded .vec file
|
|
112
|
+
def get_fasttext_model(language_code, force_download: false)
|
|
113
|
+
resource_id = "#{language_code}:fasttext"
|
|
114
|
+
result = get(resource_id, force_download: force_download)
|
|
115
|
+
|
|
116
|
+
result&.dig(:model_path)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Get or download ONNX model for a language.
|
|
120
|
+
#
|
|
121
|
+
# @param language_code [String] ISO 639-1 language code
|
|
122
|
+
# @param force_download [Boolean] Force re-download
|
|
123
|
+
# @return [String, nil] Path to downloaded .onnx file
|
|
124
|
+
def get_onnx_model(language_code, force_download: false)
|
|
125
|
+
resource_id = "#{language_code}:onnx"
|
|
126
|
+
result = get(resource_id, force_download: force_download)
|
|
127
|
+
|
|
128
|
+
result&.dig(:model_path)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# ---- Tier-aware model resources (TODO.impl/67 M1) ----
|
|
132
|
+
|
|
133
|
+
# Resource identifier for a (language, tier) model.
|
|
134
|
+
#
|
|
135
|
+
# The `full` tier maps to today's two-part id ("{lang}:onnx") and
|
|
136
|
+
# therefore to today's on-disk layout — backwards compatible.
|
|
137
|
+
# `fluency`/`mini` get a third path segment so tiers coexist:
|
|
138
|
+
#
|
|
139
|
+
# models/{lang}/models/onnx/ # full (unchanged)
|
|
140
|
+
# models/{lang}/models/onnx/mini/ # mini
|
|
141
|
+
# models/{lang}/models/onnx/fluency/ # fluency
|
|
142
|
+
#
|
|
143
|
+
# @param language_code [String] ISO 639-1 language code
|
|
144
|
+
# @param tier [String, Symbol] one of {TIERS}
|
|
145
|
+
# @return [String] resource id ("en:onnx" or "en:onnx:mini")
|
|
146
|
+
def tier_resource_id(language_code, tier)
|
|
147
|
+
tier = self.class.normalize_tier(tier)
|
|
148
|
+
lang = language_code.to_s
|
|
149
|
+
tier == :full ? "#{lang}:onnx" : "#{lang}:onnx:#{tier}"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Whether a resource id carries a tier segment ("{lang}:onnx:{tier}").
|
|
153
|
+
#
|
|
154
|
+
# @param resource_id [String] resource identifier
|
|
155
|
+
# @return [Boolean] true for well-formed tiered ids
|
|
156
|
+
def tiered_resource_id?(resource_id)
|
|
157
|
+
parts = resource_id.to_s.split(":")
|
|
158
|
+
return false unless parts.size == 3
|
|
159
|
+
|
|
160
|
+
_lang, type, tier = parts
|
|
161
|
+
type == "onnx" && TIERS.include?(tier.to_sym)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Tier of a tiered resource id ("en:onnx:mini" -> :mini).
|
|
165
|
+
# Returns nil for legacy two-part ids.
|
|
166
|
+
#
|
|
167
|
+
# @param resource_id [String] resource identifier
|
|
168
|
+
# @return [Symbol, nil]
|
|
169
|
+
def tier_from_resource_id(resource_id)
|
|
170
|
+
return nil unless tiered_resource_id?(resource_id)
|
|
171
|
+
|
|
172
|
+
resource_id.split(":")[2].to_sym
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Model tiers currently cached for a language (disk-only lookup;
|
|
176
|
+
# never touches the network, so it is resolve-safe).
|
|
177
|
+
#
|
|
178
|
+
# The legacy layout (`{lang}/models/onnx/`) counts as :full.
|
|
179
|
+
#
|
|
180
|
+
# @param language_code [String] ISO 639-1 language code
|
|
181
|
+
# @return [Array<Symbol>] cached tiers in {TIER_PREFERENCE} order
|
|
182
|
+
def cached_tiers(language_code)
|
|
183
|
+
lang = language_code.to_s
|
|
184
|
+
onnx_dir = File.join(@cache_path, lang, "models", "onnx")
|
|
185
|
+
TIER_PREFERENCE.select do |tier|
|
|
186
|
+
dir = tier == :full ? onnx_dir : File.join(onnx_dir, tier.to_s)
|
|
187
|
+
File.exist?(File.join(dir, "metadata.json"))
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Look up the registry entry for a (language, tier) model.
|
|
192
|
+
#
|
|
193
|
+
# Reads the cached registry.json; downloads it once (per cache
|
|
194
|
+
# instance) when absent or stale — unless offline mode is active,
|
|
195
|
+
# in which case a stale cached copy is reused and a missing one
|
|
196
|
+
# raises instead of ever hitting the network.
|
|
197
|
+
#
|
|
198
|
+
# @param language_code [String] ISO 639-1 language code
|
|
199
|
+
# @param tier [String, Symbol] one of {TIERS}
|
|
200
|
+
# @param force [Boolean] re-fetch the registry first
|
|
201
|
+
# @return [ModelRegistry::Resource, nil] nil when the registry has
|
|
202
|
+
# no entry for the pair
|
|
203
|
+
# @raise [Kotoshu::Error] when offline with no cached registry
|
|
204
|
+
def registry_entry_for(language_code, tier, force: false)
|
|
205
|
+
registry(force: force)&.find(language_code.to_s, tier.to_s)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Download and verify a tiered model from the registry.
|
|
209
|
+
#
|
|
210
|
+
# Downloads `urls.primary`, falling back to `urls.mirror`, plus the
|
|
211
|
+
# vocab sibling; verifies the model's SHA-256 against the registry
|
|
212
|
+
# entry before accepting it into the cache. Unlike {#get}, download
|
|
213
|
+
# failures and checksum mismatches RAISE rather than degrading to
|
|
214
|
+
# nil — setup callers rely on that to report :unavailable.
|
|
215
|
+
#
|
|
216
|
+
# @param language_code [String] ISO 639-1 language code
|
|
217
|
+
# @param tier [String, Symbol] "mini" or "fluency" (":full" is
|
|
218
|
+
# delegated to the legacy download path)
|
|
219
|
+
# @param force_download [Boolean] re-fetch the registry first
|
|
220
|
+
# @return [Hash] { model_path:, metadata:, vocab_path? }
|
|
221
|
+
# @raise [Kotoshu::Error] no registry entry / offline without cache
|
|
222
|
+
# @raise [Kotoshu::IntegrityError] SHA-256 mismatch (corrupt bytes
|
|
223
|
+
# are removed before raising)
|
|
224
|
+
def download_tiered_model(language_code, tier:, force_download: false)
|
|
225
|
+
tier = self.class.normalize_tier(tier)
|
|
226
|
+
lang = language_code.to_s
|
|
227
|
+
return get("#{lang}:onnx", force_download: force_download) if tier == :full
|
|
228
|
+
|
|
229
|
+
entry = registry_entry_for(lang, tier, force: force_download)
|
|
230
|
+
unless entry
|
|
231
|
+
raise Kotoshu::Error,
|
|
232
|
+
"no registry entry for #{ModelRegistry::Resource.id_for(lang, tier)}"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
resource_id = tier_resource_id(lang, tier)
|
|
236
|
+
dest_path = resource_dir_for(resource_id)
|
|
237
|
+
FileUtils.mkdir_p(dest_path)
|
|
238
|
+
|
|
239
|
+
filename = filename_from_url(entry.urls.primary)
|
|
240
|
+
model_file = File.join(dest_path, filename)
|
|
241
|
+
used_url = download_primary_or_mirror!(entry, model_file)
|
|
242
|
+
|
|
243
|
+
verify_registry_sha256!(entry, model_file, resource_id, used_url)
|
|
244
|
+
|
|
245
|
+
vocab_path = download_tiered_vocab(entry, dest_path)
|
|
246
|
+
|
|
247
|
+
metadata = {
|
|
248
|
+
version: entry.version,
|
|
249
|
+
url: used_url,
|
|
250
|
+
language: lang,
|
|
251
|
+
type: "onnx",
|
|
252
|
+
tier: tier.to_s,
|
|
253
|
+
file: filename,
|
|
254
|
+
checksum: entry.sha256,
|
|
255
|
+
registry_id: ModelRegistry::Resource.id_for(lang, tier),
|
|
256
|
+
size_bytes: entry.size_bytes,
|
|
257
|
+
cached_at: Time.now.utc.iso8601,
|
|
258
|
+
source: "registry"
|
|
259
|
+
}
|
|
260
|
+
write_metadata(File.join(dest_path, "metadata.json"), metadata)
|
|
261
|
+
|
|
262
|
+
result = { model_path: model_file, metadata: metadata }
|
|
263
|
+
result[:vocab_path] = vocab_path if vocab_path
|
|
264
|
+
result
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
# ---- LID artifact pair (plan 106) ----
|
|
268
|
+
|
|
269
|
+
# Whether the LID artifact pair is cached and complete.
|
|
270
|
+
#
|
|
271
|
+
# @return [Boolean]
|
|
272
|
+
def lid_cached?
|
|
273
|
+
!load_cached_lid.nil?
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Resolve the LID artifact pair from the cache only — never
|
|
277
|
+
# touches the network (the resolve half of the two-stage model).
|
|
278
|
+
#
|
|
279
|
+
# @return [Hash, nil] { onnx_path:, vocab_path:, metadata } when
|
|
280
|
+
# cached, nil when not set up
|
|
281
|
+
# @raise [Kotoshu::IntegrityError] cached bytes fail their
|
|
282
|
+
# recorded checksum
|
|
283
|
+
def load_cached_lid
|
|
284
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
285
|
+
metadata_path = File.join(dir, "metadata.json")
|
|
286
|
+
return nil unless File.exist?(metadata_path)
|
|
287
|
+
|
|
288
|
+
metadata = read_metadata(metadata_path)
|
|
289
|
+
return nil unless metadata
|
|
290
|
+
|
|
291
|
+
onnx_file = File.join(dir, metadata["file"].to_s)
|
|
292
|
+
vocab_file = File.join(dir, metadata["vocab_file"].to_s)
|
|
293
|
+
return nil unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
294
|
+
return nil unless File.exist?(vocab_file) && File.size(vocab_file).positive?
|
|
295
|
+
|
|
296
|
+
verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
297
|
+
|
|
298
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Download the LID artifact pair from the models registry
|
|
302
|
+
# (`kotoshu://models/lid/lid-176`), verifying the ONNX bytes
|
|
303
|
+
# against the registry sha256 before accepting them (the setup
|
|
304
|
+
# half of the two-stage model; same contract as
|
|
305
|
+
# {#download_tiered_model}).
|
|
306
|
+
#
|
|
307
|
+
# @param force [Boolean] re-fetch the registry first
|
|
308
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
309
|
+
# @raise [Kotoshu::Error] no registry entry, or offline without
|
|
310
|
+
# a cached registry (the registry is fetched once per cache
|
|
311
|
+
# instance; see {#registry})
|
|
312
|
+
# @raise [Kotoshu::IntegrityError] sha256 mismatch (corrupt
|
|
313
|
+
# bytes are removed before raising)
|
|
314
|
+
def download_lid(force: false)
|
|
315
|
+
entry = registry(force: force)&.find(LID_LANGUAGE, LID_TIER)
|
|
316
|
+
unless entry
|
|
317
|
+
raise Kotoshu::Error, "no registry entry for #{LID_REGISTRY_ID}"
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
321
|
+
FileUtils.mkdir_p(dir)
|
|
322
|
+
|
|
323
|
+
onnx_file = File.join(dir, filename_from_url(entry.urls.primary))
|
|
324
|
+
used_url = download_primary_or_mirror!(entry, onnx_file)
|
|
325
|
+
verify_registry_sha256!(entry, onnx_file, LID_REGISTRY_ID, used_url,
|
|
326
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download.")
|
|
327
|
+
|
|
328
|
+
# The vocab sidecar is required, not optional: the reader
|
|
329
|
+
# cannot score without its labels and Huffman counts.
|
|
330
|
+
vocab_file = File.join(dir, filename_from_url(entry.vocab_url))
|
|
331
|
+
begin
|
|
332
|
+
download_file(entry.vocab_url, vocab_file)
|
|
333
|
+
rescue StandardError => e
|
|
334
|
+
File.delete(onnx_file) if File.exist?(onnx_file)
|
|
335
|
+
raise Kotoshu::Error, "LID vocab unavailable at #{entry.vocab_url}: #{e.message}"
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
metadata = {
|
|
339
|
+
version: entry.version,
|
|
340
|
+
url: used_url,
|
|
341
|
+
language: LID_LANGUAGE,
|
|
342
|
+
type: "lid",
|
|
343
|
+
tier: LID_TIER,
|
|
344
|
+
file: File.basename(onnx_file),
|
|
345
|
+
vocab_file: File.basename(vocab_file),
|
|
346
|
+
checksum: entry.sha256,
|
|
347
|
+
registry_id: LID_REGISTRY_ID,
|
|
348
|
+
size_bytes: entry.size_bytes,
|
|
349
|
+
cached_at: Time.now.utc.iso8601,
|
|
350
|
+
source: "registry"
|
|
351
|
+
}
|
|
352
|
+
write_metadata(File.join(dir, "metadata.json"), metadata)
|
|
353
|
+
|
|
354
|
+
{ onnx_path: onnx_file, vocab_path: vocab_file, metadata: metadata }
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Get available model types for a language.
|
|
358
|
+
#
|
|
359
|
+
# @param language_code [String] ISO 639-1 language code
|
|
360
|
+
# @return [Array<Symbol>] Available model types (:fasttext, :onnx)
|
|
361
|
+
def available_models_for(language_code)
|
|
362
|
+
lang = language_code.to_sym
|
|
363
|
+
types = []
|
|
364
|
+
types << :fasttext if AVAILABLE_MODELS[:fasttext][lang]
|
|
365
|
+
types << :onnx if AVAILABLE_MODELS[:onnx][lang]
|
|
366
|
+
types
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# Get model info for a language and type.
|
|
370
|
+
#
|
|
371
|
+
# @param language_code [String] ISO 639-1 language code
|
|
372
|
+
# @param model_type [Symbol] Model type (:fasttext, :onnx)
|
|
373
|
+
# @return [Hash, nil] Model info or nil if not available
|
|
374
|
+
def model_info(language_code, model_type)
|
|
375
|
+
AVAILABLE_MODELS.dig(model_type, language_code.to_sym)
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# List all available models across all languages.
|
|
379
|
+
#
|
|
380
|
+
# @return [Hash] Mapping of language to available model types
|
|
381
|
+
def all_available_models
|
|
382
|
+
AVAILABLE_MODELS
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Check if a resource type is supported.
|
|
386
|
+
#
|
|
387
|
+
# @param resource_id [String] The resource identifier (e.g., "en:fasttext")
|
|
388
|
+
# @return [Boolean] True if supported
|
|
389
|
+
def supports_resource?(resource_id)
|
|
390
|
+
return true if tiered_resource_id?(resource_id)
|
|
391
|
+
|
|
392
|
+
parts = resource_id.split(":")
|
|
393
|
+
return false unless parts.size == 2
|
|
394
|
+
|
|
395
|
+
language, type = parts
|
|
396
|
+
AVAILABLE_MODELS[type.to_sym]&.key?(language.to_sym)
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# List all cached resources.
|
|
400
|
+
#
|
|
401
|
+
# Tiered entries appear as "{lang}:onnx:{tier}"; the registry
|
|
402
|
+
# storage directory is not a model resource and is skipped.
|
|
403
|
+
#
|
|
404
|
+
# @return [Array<String>] List of cached resource identifiers
|
|
405
|
+
def cached_resources
|
|
406
|
+
Dir.glob(File.join(@cache_path, "**", "metadata.json")).filter_map do |path|
|
|
407
|
+
relative = Pathname.new(path).relative_path_from(Pathname.new(@cache_path))
|
|
408
|
+
parts = relative.to_s.split("/")
|
|
409
|
+
next if parts.first == "registry" # registry storage, not a model
|
|
410
|
+
|
|
411
|
+
case parts.length
|
|
412
|
+
when 4 then "#{parts[0]}:#{parts[2]}" # lang/models/type/metadata.json (full)
|
|
413
|
+
when 5 then "#{parts[0]}:#{parts[2]}:#{parts[3]}" # lang/models/type/tier/metadata.json
|
|
414
|
+
end
|
|
415
|
+
end.uniq
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Load cached resource data (implements abstract method).
|
|
419
|
+
#
|
|
420
|
+
# Public — this is the cache-only reader BaseCache declares
|
|
421
|
+
# publicly; resolve paths use it to read without any download.
|
|
422
|
+
#
|
|
423
|
+
# Verifies the cached model file's SHA-256 against the checksum
|
|
424
|
+
# recorded at download time. A mismatch — truncated file, disk
|
|
425
|
+
# corruption, manual edit — raises {Kotoshu::IntegrityError} with
|
|
426
|
+
# a remediation hint pointing at the cache subcommand, matching
|
|
427
|
+
# the Phase C contract of TODO.impl/38-onnx-semantic-gating.md.
|
|
428
|
+
# Caches written before checksums were recorded are accepted as
|
|
429
|
+
# "unverified" (graceful degradation).
|
|
430
|
+
#
|
|
431
|
+
# @param resource_id [String] The resource identifier
|
|
432
|
+
# @return [Hash, nil] Loaded model info
|
|
433
|
+
# @raise [Kotoshu::IntegrityError] if the cached file's checksum
|
|
434
|
+
# does not match the recorded checksum
|
|
435
|
+
def load_cached(resource_id)
|
|
436
|
+
return load_tiered_cached(resource_id) if tiered_resource_id?(resource_id)
|
|
437
|
+
|
|
438
|
+
language = extract_language(resource_id)
|
|
439
|
+
type = extract_type(resource_id)
|
|
440
|
+
return nil unless language && type
|
|
441
|
+
|
|
442
|
+
model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
|
|
443
|
+
return nil unless model_info
|
|
444
|
+
|
|
445
|
+
metadata_path = metadata_path_for(resource_id)
|
|
446
|
+
return nil unless File.exist?(metadata_path)
|
|
447
|
+
|
|
448
|
+
metadata = read_metadata(metadata_path)
|
|
449
|
+
return nil unless metadata
|
|
450
|
+
|
|
451
|
+
# For .gz files, the decompressed version is stored without .gz extension
|
|
452
|
+
filename = model_info[:file].sub('.gz', '')
|
|
453
|
+
model_file = File.join(resource_dir_for(resource_id), filename)
|
|
454
|
+
|
|
455
|
+
return nil unless File.exist?(model_file)
|
|
456
|
+
|
|
457
|
+
verify_cached_integrity!(resource_id, metadata, model_file)
|
|
458
|
+
|
|
459
|
+
{ model_path: model_file, metadata: metadata }
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
protected
|
|
463
|
+
|
|
464
|
+
# Download a specific resource (implements abstract method).
|
|
465
|
+
#
|
|
466
|
+
# @param resource_id [String] The resource identifier
|
|
467
|
+
# @param dest_path [String] Destination directory
|
|
468
|
+
# @return [Hash] Downloaded model info
|
|
469
|
+
def download_resource(resource_id, dest_path)
|
|
470
|
+
if tiered_resource_id?(resource_id)
|
|
471
|
+
download_tiered_model(extract_language(resource_id),
|
|
472
|
+
tier: tier_from_resource_id(resource_id))
|
|
473
|
+
else
|
|
474
|
+
download_legacy_resource(resource_id, dest_path)
|
|
475
|
+
end
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# Download a legacy (untiered) resource: FastText .vec or the
|
|
479
|
+
# full-tier ONNX from the pinned git tree. This is today's
|
|
480
|
+
# behavior, unchanged.
|
|
481
|
+
#
|
|
482
|
+
# @param resource_id [String] The resource identifier
|
|
483
|
+
# @param dest_path [String] Destination directory
|
|
484
|
+
# @return [Hash] Downloaded model info
|
|
485
|
+
def download_legacy_resource(resource_id, dest_path)
|
|
486
|
+
language = extract_language(resource_id)
|
|
487
|
+
type = extract_type(resource_id)
|
|
488
|
+
return nil unless language && type
|
|
489
|
+
|
|
490
|
+
model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
|
|
491
|
+
return nil unless model_info
|
|
492
|
+
|
|
493
|
+
FileUtils.mkdir_p(dest_path)
|
|
494
|
+
|
|
495
|
+
filename = model_info[:file]
|
|
496
|
+
|
|
497
|
+
# Handle ONNX with try-download-first approach
|
|
498
|
+
if type == "onnx"
|
|
499
|
+
download_or_convert_onnx(language, dest_path, filename)
|
|
500
|
+
else
|
|
501
|
+
# Handle FastText download (existing logic)
|
|
502
|
+
url = model_url(language, type, filename)
|
|
503
|
+
|
|
504
|
+
# Remove .gz extension for final storage (we decompress gzip files)
|
|
505
|
+
final_filename = filename.sub('.gz', '')
|
|
506
|
+
model_file = File.join(dest_path, final_filename)
|
|
507
|
+
|
|
508
|
+
# Download (and decompress if needed)
|
|
509
|
+
if url.end_with?('.gz')
|
|
510
|
+
download_and_decompress(url, model_file)
|
|
511
|
+
else
|
|
512
|
+
download_file(url, model_file)
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
# Save metadata
|
|
516
|
+
metadata = build_model_metadata(language, type, final_filename, url, model_file)
|
|
517
|
+
write_metadata(File.join(dest_path, "metadata.json"), metadata)
|
|
518
|
+
|
|
519
|
+
{ model_path: model_file, metadata: metadata }
|
|
520
|
+
end
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# Load a cached tiered model. The file recorded in metadata.json is
|
|
524
|
+
# verified against the checksum recorded at download time (which,
|
|
525
|
+
# for registry downloads, is the registry's sha256).
|
|
526
|
+
#
|
|
527
|
+
# @param resource_id [String] Tiered resource identifier
|
|
528
|
+
# @return [Hash, nil] Loaded model info
|
|
529
|
+
def load_tiered_cached(resource_id)
|
|
530
|
+
metadata_path = metadata_path_for(resource_id)
|
|
531
|
+
return nil unless File.exist?(metadata_path)
|
|
532
|
+
|
|
533
|
+
metadata = read_metadata(metadata_path)
|
|
534
|
+
return nil unless metadata
|
|
535
|
+
|
|
536
|
+
model_file = File.join(resource_dir_for(resource_id), metadata["file"].to_s)
|
|
537
|
+
return nil unless File.exist?(model_file) && File.size(model_file).positive?
|
|
538
|
+
|
|
539
|
+
verify_cached_integrity!(resource_id, metadata, model_file)
|
|
540
|
+
|
|
541
|
+
result = { model_path: model_file, metadata: metadata }
|
|
542
|
+
vocab = File.join(resource_dir_for(resource_id), "fasttext.#{extract_language(resource_id)}.#{tier_from_resource_id(resource_id)}.vocab.json")
|
|
543
|
+
result[:vocab_path] = vocab if File.exist?(vocab)
|
|
544
|
+
result
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
# Get metadata file path for a resource.
|
|
548
|
+
#
|
|
549
|
+
# @param resource_id [String] The resource identifier
|
|
550
|
+
# @return [String] Metadata file path
|
|
551
|
+
def metadata_path_for(resource_id)
|
|
552
|
+
File.join(resource_dir_for(resource_id), "metadata.json")
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
# Get resource directory path. Tiered ids nest under the onnx dir
|
|
556
|
+
# (`{lang}/models/onnx/{tier}`); everything else keeps today's
|
|
557
|
+
# layout (`{lang}/models/{type}`).
|
|
558
|
+
#
|
|
559
|
+
# @param resource_id [String] The resource identifier
|
|
560
|
+
# @return [String] Resource directory path
|
|
561
|
+
def resource_dir_for(resource_id)
|
|
562
|
+
if tiered_resource_id?(resource_id)
|
|
563
|
+
language, _type, tier = resource_id.split(":")
|
|
564
|
+
return File.join(@cache_path, language, "models", "onnx", tier)
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
language = extract_language(resource_id)
|
|
568
|
+
type = extract_type(resource_id)
|
|
569
|
+
File.join(@cache_path, language, "models", type)
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
# Check if all resource files exist.
|
|
573
|
+
#
|
|
574
|
+
# @param resource_id [String] The resource identifier
|
|
575
|
+
# @return [Boolean] True if all files exist
|
|
576
|
+
def resource_files_exist?(resource_id)
|
|
577
|
+
if tiered_resource_id?(resource_id)
|
|
578
|
+
metadata = read_metadata(metadata_path_for(resource_id))
|
|
579
|
+
return false unless metadata
|
|
580
|
+
|
|
581
|
+
model_file = File.join(resource_dir_for(resource_id), metadata["file"].to_s)
|
|
582
|
+
return File.exist?(model_file) && File.size(model_file).positive?
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
language = extract_language(resource_id)
|
|
586
|
+
type = extract_type(resource_id)
|
|
587
|
+
return false unless language && type
|
|
588
|
+
|
|
589
|
+
model_info = AVAILABLE_MODELS[type.to_sym][language.to_sym]
|
|
590
|
+
return false unless model_info
|
|
591
|
+
|
|
592
|
+
# For .gz files, check the decompressed version
|
|
593
|
+
filename = model_info[:file].sub('.gz', '')
|
|
594
|
+
model_file = File.join(resource_dir_for(resource_id), filename)
|
|
595
|
+
File.exist?(model_file) && File.size(model_file).positive?
|
|
596
|
+
end
|
|
597
|
+
|
|
598
|
+
private
|
|
599
|
+
|
|
600
|
+
# Parse resource identifier into components. Tiered model ids
|
|
601
|
+
# carry a third segment ("en:onnx:mini"); all others stay
|
|
602
|
+
# two-part. Overridden from BaseCache so {#extract_language}
|
|
603
|
+
# works for tiered ids.
|
|
604
|
+
#
|
|
605
|
+
# @param resource_id [String] The resource identifier
|
|
606
|
+
# @return [Array<String>, nil] Array of parts or nil if invalid
|
|
607
|
+
def parse_resource_id(resource_id)
|
|
608
|
+
parts = resource_id.to_s.split(":")
|
|
609
|
+
return nil unless [2, 3].include?(parts.size)
|
|
610
|
+
|
|
611
|
+
parts
|
|
612
|
+
end
|
|
613
|
+
|
|
614
|
+
# ---- Registry storage ----
|
|
615
|
+
#
|
|
616
|
+
# registry.json is downloaded ONCE per cache instance (and reused
|
|
617
|
+
# across setups while fresh). Its bytes are stored under the cache
|
|
618
|
+
# dir with a metadata.json recording their own sha256, so a
|
|
619
|
+
# corrupted or tampered registry is detected and re-fetched rather
|
|
620
|
+
# than trusted. Offline mode (KOTOSHU_OFFLINE=1) never fetches:
|
|
621
|
+
# a cached copy — even a stale one — is used, and a missing one
|
|
622
|
+
# raises. Resolve never reaches this code at all (cache-only).
|
|
623
|
+
|
|
624
|
+
# The parsed registry, fetched from cache or network at most once
|
|
625
|
+
# per cache instance.
|
|
626
|
+
#
|
|
627
|
+
# @param force [Boolean] re-fetch even if a fresh copy is cached
|
|
628
|
+
# @return [ModelRegistry]
|
|
629
|
+
# @raise [Kotoshu::Error] offline with no cached registry
|
|
630
|
+
def registry(force: false)
|
|
631
|
+
return @registry = load_registry(force: true) if force
|
|
632
|
+
|
|
633
|
+
@registry ||= load_registry(force: false)
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
def load_registry(force:)
|
|
637
|
+
cached = force ? nil : read_cached_registry
|
|
638
|
+
return cached if cached && !expired?(registry_metadata_path)
|
|
639
|
+
|
|
640
|
+
if offline?
|
|
641
|
+
return cached if cached
|
|
642
|
+
|
|
643
|
+
raise Kotoshu::Error,
|
|
644
|
+
"offline mode (KOTOSHU_OFFLINE=1) and no cached registry at " \
|
|
645
|
+
"#{registry_path}; run once online or copy registry.json there"
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
fetch_registry
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
# Read and sha-verify the cached registry. Returns nil when the
|
|
652
|
+
# bytes are absent, unparsable, or fail their recorded checksum.
|
|
653
|
+
def read_cached_registry
|
|
654
|
+
return nil unless File.exist?(registry_path) && File.exist?(registry_metadata_path)
|
|
655
|
+
|
|
656
|
+
bytes = File.binread(registry_path)
|
|
657
|
+
metadata = read_metadata(registry_metadata_path)
|
|
658
|
+
expected = metadata && metadata["sha256"]
|
|
659
|
+
return nil unless expected
|
|
660
|
+
return nil unless Digest::SHA256.hexdigest(bytes) == expected
|
|
661
|
+
|
|
662
|
+
ModelRegistry.from_json(bytes)
|
|
663
|
+
rescue StandardError
|
|
664
|
+
nil
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
def fetch_registry
|
|
668
|
+
url = @source_registry.url_for(:model_registry)
|
|
669
|
+
bytes = Kotoshu::Integrity::NetHTTP.get(url)
|
|
670
|
+
raise Kotoshu::Error, "registry not found at #{url}" if bytes.nil?
|
|
671
|
+
|
|
672
|
+
begin
|
|
673
|
+
model = ModelRegistry.from_json(bytes)
|
|
674
|
+
rescue StandardError => e
|
|
675
|
+
raise Kotoshu::IntegrityError.new(
|
|
676
|
+
"registry.json",
|
|
677
|
+
expected: "<valid registry JSON>",
|
|
678
|
+
actual: "<parse error: #{e.message}>",
|
|
679
|
+
url: url
|
|
680
|
+
)
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
FileUtils.mkdir_p(File.dirname(registry_path))
|
|
684
|
+
File.binwrite(registry_path, bytes)
|
|
685
|
+
write_metadata(registry_metadata_path,
|
|
686
|
+
"url" => url,
|
|
687
|
+
"sha256" => Digest::SHA256.hexdigest(bytes),
|
|
688
|
+
"cached_at" => Time.now.utc.iso8601)
|
|
689
|
+
model
|
|
690
|
+
end
|
|
691
|
+
|
|
692
|
+
def registry_path
|
|
693
|
+
File.join(@cache_path, "registry", "registry.json")
|
|
694
|
+
end
|
|
695
|
+
|
|
696
|
+
def registry_metadata_path
|
|
697
|
+
File.join(@cache_path, "registry", "metadata.json")
|
|
698
|
+
end
|
|
699
|
+
|
|
700
|
+
def offline?
|
|
701
|
+
Kotoshu.configuration.offline
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
# ---- Tiered download helpers ----
|
|
705
|
+
|
|
706
|
+
# Download from urls.primary, falling back to urls.mirror. Partial
|
|
707
|
+
# bytes from a failed attempt are removed first. Returns the URL
|
|
708
|
+
# that succeeded; re-raises the last error when both fail.
|
|
709
|
+
def download_primary_or_mirror!(entry, model_file)
|
|
710
|
+
urls = [entry.urls.primary, entry.urls.mirror].compact.uniq
|
|
711
|
+
last_error = nil
|
|
712
|
+
urls.each do |url|
|
|
713
|
+
File.delete(model_file) if File.exist?(model_file)
|
|
714
|
+
download_file(url, model_file)
|
|
715
|
+
return url
|
|
716
|
+
rescue StandardError => e
|
|
717
|
+
last_error = e
|
|
718
|
+
warn " download failed (#{url}): #{e.message}" if $VERBOSE
|
|
719
|
+
end
|
|
720
|
+
|
|
721
|
+
File.delete(model_file) if File.exist?(model_file)
|
|
722
|
+
raise last_error
|
|
723
|
+
end
|
|
724
|
+
|
|
725
|
+
# Whether a file is a Git LFS pointer stub rather than real content.
|
|
726
|
+
# Reads only the first line's prefix, so it stays cheap on 120MB models.
|
|
727
|
+
def lfs_pointer?(path)
|
|
728
|
+
File.open(path, "rb") do |f|
|
|
729
|
+
f.read(LFS_POINTER_PREFIX.bytesize) == LFS_POINTER_PREFIX
|
|
730
|
+
end
|
|
731
|
+
rescue StandardError
|
|
732
|
+
false
|
|
733
|
+
end
|
|
734
|
+
|
|
735
|
+
# Verify downloaded model bytes against the registry's sha256.
|
|
736
|
+
# Corrupt bytes are removed from disk before raising so the next
|
|
737
|
+
# attempt re-downloads (same contract as BaseCache#verify_and_audit).
|
|
738
|
+
# +remediation+ overrides the default re-download hint (the LID
|
|
739
|
+
# pair is fetched through Kotoshu.setup_lid, not setup LANG --model).
|
|
740
|
+
def verify_registry_sha256!(entry, model_file, resource_id, url, remediation: nil)
|
|
741
|
+
actual = Digest::SHA256.file(model_file).hexdigest
|
|
742
|
+
if actual == entry.sha256
|
|
743
|
+
@audit_log.record(
|
|
744
|
+
url: url, status: "verified", size: File.size(model_file),
|
|
745
|
+
sha256: actual, manifest_sha256: entry.sha256, resource_id: resource_id
|
|
746
|
+
)
|
|
747
|
+
return
|
|
748
|
+
end
|
|
749
|
+
|
|
750
|
+
@audit_log.record(
|
|
751
|
+
url: url, status: "mismatch", size: File.size(model_file),
|
|
752
|
+
sha256: actual, manifest_sha256: entry.sha256, resource_id: resource_id
|
|
753
|
+
)
|
|
754
|
+
File.delete(model_file)
|
|
755
|
+
raise Kotoshu::IntegrityError.new(
|
|
756
|
+
resource_id,
|
|
757
|
+
expected: entry.sha256,
|
|
758
|
+
actual: actual,
|
|
759
|
+
url: url,
|
|
760
|
+
remediation: remediation ||
|
|
761
|
+
"Run `kotoshu setup #{extract_language(resource_id)} --model` to re-download."
|
|
762
|
+
)
|
|
763
|
+
end
|
|
764
|
+
|
|
765
|
+
# Pull the vocab sibling for a tiered model. Missing vocab is a
|
|
766
|
+
# warning, not a failure (mirrors the legacy full-tier behavior).
|
|
767
|
+
def download_tiered_vocab(entry, dest_path)
|
|
768
|
+
return nil if entry.vocab_url.to_s.empty?
|
|
769
|
+
|
|
770
|
+
vocab_file = File.join(dest_path, filename_from_url(entry.vocab_url))
|
|
771
|
+
begin
|
|
772
|
+
download_file(entry.vocab_url, vocab_file)
|
|
773
|
+
vocab_file
|
|
774
|
+
rescue StandardError => e
|
|
775
|
+
warn " vocab.json unavailable: #{e.message}" if $VERBOSE
|
|
776
|
+
nil
|
|
777
|
+
end
|
|
778
|
+
end
|
|
779
|
+
|
|
780
|
+
# Basename of a URL's path, e.g.
|
|
781
|
+
# "https://h/fasttext.en.mini.onnx" => "fasttext.en.mini.onnx".
|
|
782
|
+
def filename_from_url(url)
|
|
783
|
+
File.basename(URI.parse(url.to_s).path)
|
|
784
|
+
end
|
|
785
|
+
|
|
786
|
+
# Verify a cached model file's SHA-256 against the checksum
|
|
787
|
+
# recorded in its metadata. Raises {Kotoshu::IntegrityError} on
|
|
788
|
+
# mismatch so the caller surfaces a clear, actionable error.
|
|
789
|
+
# Caches written without a checksum field are accepted silently
|
|
790
|
+
# to preserve backward compatibility with pre-verification caches.
|
|
791
|
+
#
|
|
792
|
+
# Verify the cached LID pair against the checksum recorded at
|
|
793
|
+
# download time (which, for registry downloads, is the registry's
|
|
794
|
+
# sha256). A Git LFS pointer stub is refused by content first —
|
|
795
|
+
# same contract as {#verify_cached_integrity!}, against the
|
|
796
|
+
# fixed LID paths instead of a resource-id-derived layout.
|
|
797
|
+
#
|
|
798
|
+
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
799
|
+
# @param onnx_file [String] Path to the cached lid.176.onnx
|
|
800
|
+
# @param vocab_file [String] Path to the cached vocab sidecar
|
|
801
|
+
# @return [void]
|
|
802
|
+
# @raise [Kotoshu::IntegrityError, Kotoshu::Error] on mismatch;
|
|
803
|
+
# corrupt bytes are removed so the next setup re-downloads
|
|
804
|
+
def verify_lid_integrity!(metadata, onnx_file, vocab_file)
|
|
805
|
+
dir = File.join(@cache_path, LID_DIRECTORY)
|
|
806
|
+
if lfs_pointer?(onnx_file) || lfs_pointer?(vocab_file)
|
|
807
|
+
File.delete(onnx_file)
|
|
808
|
+
File.delete(vocab_file)
|
|
809
|
+
FileUtils.rm_f(File.join(dir, "metadata.json"))
|
|
810
|
+
raise Kotoshu::Error,
|
|
811
|
+
"Integrity verification failed for #{LID_REGISTRY_ID}: cached LID " \
|
|
812
|
+
"artifact is a Git LFS pointer stub, not model content. Run " \
|
|
813
|
+
"Kotoshu.setup_lid(force: true) to re-download."
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
expected = metadata["checksum"]
|
|
817
|
+
return unless expected
|
|
818
|
+
|
|
819
|
+
actual = Digest::SHA256.file(onnx_file).hexdigest
|
|
820
|
+
return if actual == expected
|
|
821
|
+
|
|
822
|
+
raise Kotoshu::IntegrityError.new(
|
|
823
|
+
LID_REGISTRY_ID,
|
|
824
|
+
expected: expected,
|
|
825
|
+
actual: actual,
|
|
826
|
+
url: metadata["url"],
|
|
827
|
+
remediation: "Run Kotoshu.setup_lid(force: true) to re-download."
|
|
828
|
+
)
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
# @param resource_id [String] The resource identifier (e.g. "en:onnx")
|
|
832
|
+
# @param metadata [Hash] Parsed metadata.json (string keys)
|
|
833
|
+
# @param model_file [String] Path to the cached model file
|
|
834
|
+
# @return [void]
|
|
835
|
+
# @raise [Kotoshu::IntegrityError] if checksums do not match
|
|
836
|
+
#
|
|
837
|
+
def verify_cached_integrity!(resource_id, metadata, model_file)
|
|
838
|
+
if lfs_pointer?(model_file)
|
|
839
|
+
# Content, not checksum, betrays the stub: the legacy download
|
|
840
|
+
# path recorded the pointer's own sha256. Purge both files so
|
|
841
|
+
# the next get re-downloads (same contract as the tiered path).
|
|
842
|
+
File.delete(model_file)
|
|
843
|
+
metadata_file = metadata_path_for(resource_id)
|
|
844
|
+
File.delete(metadata_file) if File.exist?(metadata_file)
|
|
845
|
+
raise Kotoshu::Error,
|
|
846
|
+
"Integrity verification failed for #{resource_id}: cached model " \
|
|
847
|
+
"#{File.basename(model_file)} is a Git LFS pointer stub, not " \
|
|
848
|
+
"model content (raw.githubusercontent.com serves pointer stubs " \
|
|
849
|
+
"for LFS-tracked .onnx files). Run " \
|
|
850
|
+
"`kotoshu cache download :#{extract_language(resource_id)} --model` to re-download."
|
|
851
|
+
end
|
|
852
|
+
|
|
853
|
+
expected = metadata["checksum"]
|
|
854
|
+
return unless expected
|
|
855
|
+
|
|
856
|
+
actual = Digest::SHA256.file(model_file).hexdigest
|
|
857
|
+
return if actual == expected
|
|
858
|
+
|
|
859
|
+
remediation = "Run `kotoshu cache download :#{extract_language(resource_id)} --model` to re-download."
|
|
860
|
+
raise Kotoshu::IntegrityError.new(
|
|
861
|
+
resource_id,
|
|
862
|
+
expected: expected,
|
|
863
|
+
actual: actual,
|
|
864
|
+
url: metadata["url"],
|
|
865
|
+
remediation: remediation
|
|
866
|
+
)
|
|
867
|
+
end
|
|
868
|
+
|
|
869
|
+
# Build metadata hash for a model.
|
|
870
|
+
#
|
|
871
|
+
# @param language [String] Language code
|
|
872
|
+
# @param type [String] Model type
|
|
873
|
+
# @param filename [String] Model filename
|
|
874
|
+
# @param url [String] Download URL
|
|
875
|
+
# @param model_file [String] Path to downloaded model file
|
|
876
|
+
# @return [Hash] Metadata hash
|
|
877
|
+
def build_model_metadata(language, type, filename, url, model_file)
|
|
878
|
+
{
|
|
879
|
+
version: Time.now.utc.iso8601,
|
|
880
|
+
url: url,
|
|
881
|
+
language: language,
|
|
882
|
+
type: type,
|
|
883
|
+
file: filename,
|
|
884
|
+
checksum: Digest::SHA256.file(model_file).hexdigest,
|
|
885
|
+
cached_at: Time.now.utc.iso8601
|
|
886
|
+
}
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
# Get URL for a model file.
|
|
890
|
+
#
|
|
891
|
+
# @param language [String] Language code
|
|
892
|
+
# @param type [String] Model type
|
|
893
|
+
# @param filename [String] Model filename
|
|
894
|
+
# @return [String, nil] Download URL
|
|
895
|
+
def model_url(language, type, filename)
|
|
896
|
+
case type
|
|
897
|
+
when "fasttext"
|
|
898
|
+
# Download from FastText CDN (Facebook Research)
|
|
899
|
+
# https://fasttext.cc/docs/en/english-vectors.html
|
|
900
|
+
"https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/#{filename}"
|
|
901
|
+
when "onnx"
|
|
902
|
+
# Download from models-fasttext-onnx GitHub repository.
|
|
903
|
+
# SourceRegistry owns the per-repo pin so we never accidentally
|
|
904
|
+
# fall back to the dictionaries pin.
|
|
905
|
+
@source_registry.url_for(:model, lang: language)
|
|
906
|
+
else
|
|
907
|
+
"#{@url_base}/dictionaries/main/#{language}/models/#{type}/#{filename}"
|
|
908
|
+
end
|
|
909
|
+
end
|
|
910
|
+
|
|
911
|
+
# URL for the vocab.json sibling file. The conversion script ships
|
|
912
|
+
# vocabularies alongside the .onnx so OnnxModel.from_file can resolve
|
|
913
|
+
# word→index without re-parsing the FastText .vec.
|
|
914
|
+
#
|
|
915
|
+
# @param language [String] Language code
|
|
916
|
+
# @return [String]
|
|
917
|
+
def vocab_url(language)
|
|
918
|
+
@source_registry.url_for(:model_vocab, lang: language)
|
|
919
|
+
end
|
|
920
|
+
|
|
921
|
+
# Download and decompress gzip file.
|
|
922
|
+
#
|
|
923
|
+
# @param url [String] URL to gzip file
|
|
924
|
+
# @param dest_path [String] Destination path (without .gz)
|
|
925
|
+
def download_and_decompress(url, dest_path)
|
|
926
|
+
# Download to temporary file first
|
|
927
|
+
temp_gz = "#{dest_path}.gz"
|
|
928
|
+
|
|
929
|
+
puts " Downloading from #{url.split('/').last}..." if $VERBOSE
|
|
930
|
+
|
|
931
|
+
downloaded_bytes = 0
|
|
932
|
+
URI.open(url, open_timeout: 30, read_timeout: 300) do |uri|
|
|
933
|
+
File.open(temp_gz, 'wb') do |f|
|
|
934
|
+
downloaded_bytes = f.write(uri.read)
|
|
935
|
+
end
|
|
936
|
+
end
|
|
937
|
+
|
|
938
|
+
puts " Downloaded: #{(downloaded_bytes.to_f / 1024 / 1024).round(2)} MB" if $VERBOSE
|
|
939
|
+
|
|
940
|
+
# Verify the download succeeded
|
|
941
|
+
unless File.exist?(temp_gz) && File.size(temp_gz).positive?
|
|
942
|
+
raise "Download failed: #{temp_gz} is empty or missing"
|
|
943
|
+
end
|
|
944
|
+
|
|
945
|
+
puts " Decompressing..." if $VERBOSE
|
|
946
|
+
|
|
947
|
+
# Remove existing file if present (handles partial downloads)
|
|
948
|
+
File.delete(dest_path) if File.exist?(dest_path)
|
|
949
|
+
|
|
950
|
+
# Decompress gzip with streaming
|
|
951
|
+
File.open(temp_gz, 'rb') do |gz_file|
|
|
952
|
+
Zlib::GzipReader.wrap(gz_file) do |gzip|
|
|
953
|
+
# Stream in chunks to avoid memory issues with large files
|
|
954
|
+
File.open(dest_path, 'wb') do |out_file|
|
|
955
|
+
chunk_size = 65_536 # 64KB chunks
|
|
956
|
+
while (chunk = gzip.read(chunk_size))
|
|
957
|
+
out_file.write(chunk)
|
|
958
|
+
# Print progress every 10MB
|
|
959
|
+
if $VERBOSE && out_file.pos % (10 * 1024 * 1024) < chunk_size
|
|
960
|
+
puts " Decompressed: #{(out_file.pos.to_f / 1024 / 1024).round(1)} MB..."
|
|
961
|
+
end
|
|
962
|
+
end
|
|
963
|
+
end
|
|
964
|
+
end
|
|
965
|
+
end
|
|
966
|
+
|
|
967
|
+
# Verify the decompression succeeded
|
|
968
|
+
unless File.exist?(dest_path) && File.size(dest_path).positive?
|
|
969
|
+
raise "Decompression failed: #{dest_path} is empty or missing"
|
|
970
|
+
end
|
|
971
|
+
|
|
972
|
+
# Clean up gz file
|
|
973
|
+
File.delete(temp_gz)
|
|
974
|
+
|
|
975
|
+
puts " ✓ Downloaded and decompressed" if $VERBOSE
|
|
976
|
+
end
|
|
977
|
+
|
|
978
|
+
# Convert FastText .vec file to ONNX format.
|
|
979
|
+
#
|
|
980
|
+
# @param language [String] Language code
|
|
981
|
+
# @param dest_path [String] Destination directory
|
|
982
|
+
# @param onnx_filename [String] Output ONNX filename
|
|
983
|
+
# @return [Hash] Converted model info
|
|
984
|
+
def convert_to_onnx(language, dest_path, onnx_filename)
|
|
985
|
+
puts "Converting FastText to ONNX for #{language}..." if $VERBOSE
|
|
986
|
+
|
|
987
|
+
# First, ensure we have the FastText .vec file
|
|
988
|
+
fasttext_resource_id = "#{language}:fasttext"
|
|
989
|
+
fasttext_result = get(fasttext_resource_id, force_download: false)
|
|
990
|
+
|
|
991
|
+
unless fasttext_result
|
|
992
|
+
raise "Failed to get FastText model for #{language} needed for ONNX conversion"
|
|
993
|
+
end
|
|
994
|
+
|
|
995
|
+
vec_file = fasttext_result[:model_path]
|
|
996
|
+
|
|
997
|
+
# Verify the .vec file exists
|
|
998
|
+
unless File.exist?(vec_file)
|
|
999
|
+
raise "FastText .vec file not found: #{vec_file}"
|
|
1000
|
+
end
|
|
1001
|
+
|
|
1002
|
+
# Output ONNX file path
|
|
1003
|
+
onnx_file = File.join(dest_path, onnx_filename)
|
|
1004
|
+
|
|
1005
|
+
# Get the conversion script path
|
|
1006
|
+
script_path = File.expand_path('../scripts/fasttext_to_onnx.py', __dir__)
|
|
1007
|
+
|
|
1008
|
+
unless File.exist?(script_path)
|
|
1009
|
+
raise "ONNX conversion script not found: #{script_path}"
|
|
1010
|
+
end
|
|
1011
|
+
|
|
1012
|
+
# Build conversion command
|
|
1013
|
+
# Use --vocab-size to limit vocabulary size and reduce conversion time
|
|
1014
|
+
vocab_size = fasttext_result.dig(:metadata, "vocab_size")&.to_i || 100_000
|
|
1015
|
+
|
|
1016
|
+
cmd = [
|
|
1017
|
+
'python3',
|
|
1018
|
+
script_path,
|
|
1019
|
+
vec_file,
|
|
1020
|
+
onnx_file,
|
|
1021
|
+
'--vocab-size', vocab_size.to_s
|
|
1022
|
+
]
|
|
1023
|
+
|
|
1024
|
+
puts " Running conversion: #{shell_join(cmd)}" if $VERBOSE
|
|
1025
|
+
|
|
1026
|
+
# Run conversion
|
|
1027
|
+
require 'open3'
|
|
1028
|
+
stdout, stderr, status = Open3.capture3(*cmd)
|
|
1029
|
+
|
|
1030
|
+
unless status.success?
|
|
1031
|
+
raise "ONNX conversion failed:\n#{stdout}\n#{stderr}"
|
|
1032
|
+
end
|
|
1033
|
+
|
|
1034
|
+
puts stdout if $VERBOSE
|
|
1035
|
+
|
|
1036
|
+
# Build metadata for the ONNX file
|
|
1037
|
+
metadata = {
|
|
1038
|
+
version: Time.now.utc.iso8601,
|
|
1039
|
+
url: "converted:#{vec_file}",
|
|
1040
|
+
language: language,
|
|
1041
|
+
type: "onnx",
|
|
1042
|
+
file: onnx_filename,
|
|
1043
|
+
checksum: Digest::SHA256.file(onnx_file).hexdigest,
|
|
1044
|
+
cached_at: Time.now.utc.iso8601,
|
|
1045
|
+
source_model: File.basename(vec_file),
|
|
1046
|
+
conversion_method: "fasttext_to_onnx.py"
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
# Save metadata
|
|
1050
|
+
write_metadata(File.join(dest_path, "metadata.json"), metadata)
|
|
1051
|
+
|
|
1052
|
+
puts " ✓ ONNX conversion complete" if $VERBOSE
|
|
1053
|
+
|
|
1054
|
+
{ model_path: onnx_file, metadata: metadata }
|
|
1055
|
+
end
|
|
1056
|
+
|
|
1057
|
+
# Try to download ONNX from GitHub, fall back to conversion if download fails.
|
|
1058
|
+
#
|
|
1059
|
+
# @param language [String] Language code
|
|
1060
|
+
# @param dest_path [String] Destination directory
|
|
1061
|
+
# @param onnx_filename [String] ONNX filename
|
|
1062
|
+
# @return [Hash] Downloaded or converted model info
|
|
1063
|
+
def download_or_convert_onnx(language, dest_path, onnx_filename)
|
|
1064
|
+
url = model_url(language, "onnx", onnx_filename)
|
|
1065
|
+
onnx_file = File.join(dest_path, onnx_filename)
|
|
1066
|
+
|
|
1067
|
+
puts " Attempting download from GitHub..." if $VERBOSE
|
|
1068
|
+
|
|
1069
|
+
# Try downloading from GitHub first
|
|
1070
|
+
begin
|
|
1071
|
+
download_file(url, onnx_file)
|
|
1072
|
+
|
|
1073
|
+
# Verify the downloaded file
|
|
1074
|
+
unless File.exist?(onnx_file) && File.size(onnx_file).positive?
|
|
1075
|
+
raise "Download failed: empty file"
|
|
1076
|
+
end
|
|
1077
|
+
|
|
1078
|
+
# raw.githubusercontent.com serves LFS pointer stubs for
|
|
1079
|
+
# LFS-tracked .onnx files; refuse one so the fallback path runs
|
|
1080
|
+
# instead of poisoning the cache with a stub.
|
|
1081
|
+
if lfs_pointer?(onnx_file)
|
|
1082
|
+
raise "downloaded a Git LFS pointer stub instead of model bytes from #{url}"
|
|
1083
|
+
end
|
|
1084
|
+
|
|
1085
|
+
# Pull the matching vocab.json so OnnxModel.from_file can resolve
|
|
1086
|
+
# word→index without re-parsing the source FastText .vec.
|
|
1087
|
+
begin
|
|
1088
|
+
download_file(vocab_url(language),
|
|
1089
|
+
File.join(dest_path, "fasttext.#{language}.vocab.json"))
|
|
1090
|
+
rescue StandardError => e
|
|
1091
|
+
warn " vocab.json unavailable for #{language}: #{e.message}" if $VERBOSE
|
|
1092
|
+
end
|
|
1093
|
+
|
|
1094
|
+
puts " ✓ Downloaded from GitHub" if $VERBOSE
|
|
1095
|
+
|
|
1096
|
+
# Build metadata for downloaded file
|
|
1097
|
+
metadata = {
|
|
1098
|
+
version: Time.now.utc.iso8601,
|
|
1099
|
+
url: url,
|
|
1100
|
+
language: language,
|
|
1101
|
+
type: "onnx",
|
|
1102
|
+
file: onnx_filename,
|
|
1103
|
+
checksum: Digest::SHA256.file(onnx_file).hexdigest,
|
|
1104
|
+
cached_at: Time.now.utc.iso8601,
|
|
1105
|
+
source: "github"
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
# Save metadata
|
|
1109
|
+
write_metadata(File.join(dest_path, "metadata.json"), metadata)
|
|
1110
|
+
|
|
1111
|
+
{ model_path: onnx_file, metadata: metadata }
|
|
1112
|
+
rescue StandardError => e
|
|
1113
|
+
puts " GitHub download failed: #{e.message}" if $VERBOSE
|
|
1114
|
+
puts " Falling back to local conversion..." if $VERBOSE
|
|
1115
|
+
|
|
1116
|
+
# Remove partial download if any
|
|
1117
|
+
File.delete(onnx_file) if File.exist?(onnx_file)
|
|
1118
|
+
|
|
1119
|
+
# Fall back to local conversion
|
|
1120
|
+
convert_to_onnx(language, dest_path, onnx_filename)
|
|
1121
|
+
end
|
|
1122
|
+
end
|
|
1123
|
+
|
|
1124
|
+
# Join shell command arguments safely (for display purposes).
|
|
1125
|
+
#
|
|
1126
|
+
# @param args [Array<String>] Command arguments
|
|
1127
|
+
# @return [String] Joined command string
|
|
1128
|
+
def shell_join(args)
|
|
1129
|
+
args.map { |a| /\s/.match?(a) ? "'#{a}'" : a }.join(' ')
|
|
1130
|
+
end
|
|
1131
|
+
|
|
1132
|
+
# Default cache path: $XDG_CACHE_HOME/kotoshu/models
|
|
1133
|
+
#
|
|
1134
|
+
# @return [String] Default cache path
|
|
1135
|
+
def default_cache_path
|
|
1136
|
+
File.join(Kotoshu::Paths.cache_path, "models")
|
|
1137
|
+
end
|
|
1138
|
+
|
|
1139
|
+
# Default cache TTL (30 days for models).
|
|
1140
|
+
#
|
|
1141
|
+
# @return [Integer] Default TTL in seconds
|
|
1142
|
+
def default_cache_ttl
|
|
1143
|
+
2_592_000 # 30 days
|
|
1144
|
+
end
|
|
1145
|
+
end
|
|
1146
|
+
end
|
|
1147
|
+
end
|