kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "json"
|
|
6
|
+
require "digest"
|
|
7
|
+
require "uri"
|
|
8
|
+
require "time"
|
|
9
|
+
|
|
10
|
+
module Kotoshu
|
|
11
|
+
module Cache
|
|
12
|
+
# Abstract base class for all cache implementations.
|
|
13
|
+
#
|
|
14
|
+
# Provides common functionality for:
|
|
15
|
+
# - HTTP downloads with metadata
|
|
16
|
+
# - Cache validation (exists, expired)
|
|
17
|
+
# - Statistics tracking (hits, misses, hit rate)
|
|
18
|
+
# - TTL management
|
|
19
|
+
#
|
|
20
|
+
# Subclasses implement specific download and loading logic.
|
|
21
|
+
#
|
|
22
|
+
# @abstract Subclass must implement {#download_resource}, {#load_cached}
|
|
23
|
+
class BaseCache
|
|
24
|
+
# @return [String] Path to the cache directory
|
|
25
|
+
attr_reader :cache_path
|
|
26
|
+
|
|
27
|
+
# @return [String] Base URL for downloading resources
|
|
28
|
+
attr_reader :url_base
|
|
29
|
+
|
|
30
|
+
# @return [Integer] Cache TTL in seconds
|
|
31
|
+
attr_reader :cache_ttl
|
|
32
|
+
|
|
33
|
+
# @return [Integer] Maximum total cache size in bytes. Enforced
|
|
34
|
+
# by {#evict}; the value is pulled from
|
|
35
|
+
# +Kotoshu::Configuration.instance.max_cache_size+ by default
|
|
36
|
+
# (env: +KOTOSHU_MAX_CACHE_SIZE+).
|
|
37
|
+
attr_reader :max_cache_size
|
|
38
|
+
|
|
39
|
+
# @return [String] GitHub repository URL
|
|
40
|
+
attr_reader :github_url
|
|
41
|
+
|
|
42
|
+
# @return [Kotoshu::SourceRegistry] Single source of truth for
|
|
43
|
+
# per-repo URLs and pins. Subclasses MUST build URLs through
|
|
44
|
+
# this registry rather than constructing URL strings inline.
|
|
45
|
+
attr_reader :source_registry
|
|
46
|
+
|
|
47
|
+
# Create a new cache.
|
|
48
|
+
#
|
|
49
|
+
# @param cache_path [String] Path to cache directory
|
|
50
|
+
# @param url_base [String] Base URL for downloads (deprecated; pass source_registry instead).
|
|
51
|
+
# When given, it still controls where downloads come from: the
|
|
52
|
+
# registry is derived from it so the deprecated parameter keeps
|
|
53
|
+
# its documented meaning.
|
|
54
|
+
# @param cache_ttl [Integer] Cache TTL in seconds
|
|
55
|
+
# @param github_url [String] GitHub repository URL
|
|
56
|
+
# @param resource_pin [String] Branch/tag/commit for URL templates (deprecated; use source_registry)
|
|
57
|
+
# @param manifest_url [String, nil] Override manifest.json URL
|
|
58
|
+
# @param audit_log [Integrity::AuditLog, nil] Override audit log
|
|
59
|
+
# @param source_registry [Kotoshu::SourceRegistry, nil] Single source of truth for URLs/pins
|
|
60
|
+
# @param max_cache_size [Integer, nil] Maximum cache size in bytes (default 1 GB)
|
|
61
|
+
def initialize(cache_path: nil, url_base: nil, cache_ttl: nil, github_url: nil,
|
|
62
|
+
resource_pin: nil, manifest_url: nil, audit_log: nil,
|
|
63
|
+
source_registry: nil, max_cache_size: nil)
|
|
64
|
+
@cache_path = cache_path || default_cache_path
|
|
65
|
+
@source_registry = source_registry || source_registry_for(url_base)
|
|
66
|
+
@url_base = url_base || @source_registry.base_url
|
|
67
|
+
@cache_ttl = cache_ttl || default_cache_ttl
|
|
68
|
+
@github_url = github_url || default_github_url
|
|
69
|
+
@resource_pin = resource_pin || @source_registry.pin_for_source(:spelling)
|
|
70
|
+
@manifest_url = manifest_url
|
|
71
|
+
@audit_log = audit_log || default_audit_log
|
|
72
|
+
@max_cache_size = max_cache_size || default_max_cache_size
|
|
73
|
+
@manifest = nil
|
|
74
|
+
@manifest_loaded = false
|
|
75
|
+
@hits = 0
|
|
76
|
+
@misses = 0
|
|
77
|
+
|
|
78
|
+
# Ensure cache directory exists
|
|
79
|
+
FileUtils.mkdir_p(@cache_path)
|
|
80
|
+
FileUtils.mkdir_p(File.join(@cache_path, "tmp"))
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Check if a resource is available in cache.
|
|
84
|
+
#
|
|
85
|
+
# @param resource_id [String] The resource identifier (e.g., language code)
|
|
86
|
+
# @return [Boolean] True if resource is cached and valid
|
|
87
|
+
def available?(resource_id)
|
|
88
|
+
return false unless supports_resource?(resource_id)
|
|
89
|
+
|
|
90
|
+
metadata_path = metadata_path_for(resource_id)
|
|
91
|
+
return false unless File.exist?(metadata_path)
|
|
92
|
+
return false if expired?(metadata_path)
|
|
93
|
+
|
|
94
|
+
resource_files_exist?(resource_id)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Whether readable cached data exists for +resource_id+, TTL
|
|
98
|
+
# ignored (plan 117). Expiry is a refresh signal for the setup /
|
|
99
|
+
# download paths, never a reason for readers to behave as if the
|
|
100
|
+
# data were absent: the bytes are checksummed at write time, so
|
|
101
|
+
# present-but-expired data is exactly the dataset it was.
|
|
102
|
+
#
|
|
103
|
+
# @param resource_id [String] The resource identifier
|
|
104
|
+
# @return [Boolean] True if metadata and data files exist
|
|
105
|
+
def cached_data?(resource_id)
|
|
106
|
+
return false unless supports_resource?(resource_id)
|
|
107
|
+
|
|
108
|
+
File.exist?(metadata_path_for(resource_id)) && resource_files_exist?(resource_id)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Get a resource from cache or download it.
|
|
112
|
+
#
|
|
113
|
+
# @param resource_id [String] The resource identifier
|
|
114
|
+
# @param force_download [Boolean] Force re-download even if cached
|
|
115
|
+
# @return [Object, nil] The cached resource or nil if not available
|
|
116
|
+
def get(resource_id, force_download: false)
|
|
117
|
+
return nil unless supports_resource?(resource_id)
|
|
118
|
+
|
|
119
|
+
metadata_path = metadata_path_for(resource_id)
|
|
120
|
+
|
|
121
|
+
if !force_download && cached?(metadata_path) && !expired?(metadata_path)
|
|
122
|
+
@hits += 1
|
|
123
|
+
return load_cached(resource_id)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
@misses += 1
|
|
127
|
+
download(resource_id)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Clear a specific resource from cache.
|
|
131
|
+
#
|
|
132
|
+
# @param resource_id [String] The resource identifier
|
|
133
|
+
# @return [Boolean] True if cache was cleared
|
|
134
|
+
def clear(resource_id)
|
|
135
|
+
return false unless supports_resource?(resource_id)
|
|
136
|
+
|
|
137
|
+
resource_dir = resource_dir_for(resource_id)
|
|
138
|
+
if File.exist?(resource_dir)
|
|
139
|
+
FileUtils.rm_rf(resource_dir)
|
|
140
|
+
return true
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
false
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Clear all cached resources.
|
|
147
|
+
#
|
|
148
|
+
# @return [void]
|
|
149
|
+
def clear_all
|
|
150
|
+
@hits = 0
|
|
151
|
+
@misses = 0
|
|
152
|
+
FileUtils.rm_rf(@cache_path)
|
|
153
|
+
FileUtils.mkdir_p(@cache_path)
|
|
154
|
+
FileUtils.mkdir_p(File.join(@cache_path, "tmp"))
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Get cache statistics.
|
|
158
|
+
#
|
|
159
|
+
# @return [Hash] Statistics including :hits, :misses, :hit_rate, :size
|
|
160
|
+
def stats
|
|
161
|
+
total = @hits + @misses
|
|
162
|
+
hit_rate = total.positive? ? (@hits.to_f / total) : 0.0
|
|
163
|
+
|
|
164
|
+
{
|
|
165
|
+
hits: @hits,
|
|
166
|
+
misses: @misses,
|
|
167
|
+
total: total,
|
|
168
|
+
hit_rate: hit_rate,
|
|
169
|
+
cached_resources: cached_resources,
|
|
170
|
+
size_bytes: cache_size,
|
|
171
|
+
oldest_entry: oldest_entry
|
|
172
|
+
}
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Reset statistics counters.
|
|
176
|
+
#
|
|
177
|
+
# @return [self] Self for chaining
|
|
178
|
+
def reset_stats
|
|
179
|
+
@hits = 0
|
|
180
|
+
@misses = 0
|
|
181
|
+
self
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Clean expired cache entries.
|
|
185
|
+
#
|
|
186
|
+
# @return [Hash] Cleanup statistics
|
|
187
|
+
def clean
|
|
188
|
+
expired_count = clean_expired
|
|
189
|
+
size_reclaimed = clean_by_size
|
|
190
|
+
|
|
191
|
+
{
|
|
192
|
+
expired_entries_removed: expired_count,
|
|
193
|
+
bytes_reclaimed: size_reclaimed
|
|
194
|
+
}
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Enforce the cache size cap by evicting the oldest entries
|
|
198
|
+
# (LRU by +cached_at+). When +dry_run:+ true, returns the plan
|
|
199
|
+
# without modifying the disk; otherwise removes the evicted
|
|
200
|
+
# entries via +FileUtils.rm_rf+ and returns the same plan.
|
|
201
|
+
#
|
|
202
|
+
# The plan is computed by {Cache::EvictionPolicy} — a pure value
|
|
203
|
+
# object that performs no IO. This method is the only place that
|
|
204
|
+
# collects on-disk entries and the only place that mutates the
|
|
205
|
+
# cache directory based on the plan, keeping the policy testable
|
|
206
|
+
# in isolation.
|
|
207
|
+
#
|
|
208
|
+
# @param dry_run [Boolean] return the plan without writing
|
|
209
|
+
# @return [Hash] { evict: Array<Hash>, keep: Array<Hash>,
|
|
210
|
+
# bytes_reclaimed: Integer } where each entry hash carries
|
|
211
|
+
# :path, :size, :cached_at
|
|
212
|
+
def evict(dry_run: false)
|
|
213
|
+
policy = EvictionPolicy.new(max_size: @max_cache_size)
|
|
214
|
+
plan = policy.plan(collect_eviction_entries)
|
|
215
|
+
return plan if dry_run
|
|
216
|
+
|
|
217
|
+
plan[:evict].each { |e| FileUtils.rm_rf(e[:path]) }
|
|
218
|
+
plan
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# List all cached resources.
|
|
222
|
+
#
|
|
223
|
+
# @return [Array<String>] List of cached resource identifiers
|
|
224
|
+
def cached_resources
|
|
225
|
+
raise NotImplementedError, "Subclass must implement"
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Check if a resource type is supported.
|
|
229
|
+
#
|
|
230
|
+
# @param resource_id [String] The resource identifier
|
|
231
|
+
# @return [Boolean] True if supported
|
|
232
|
+
def supports_resource?(resource_id)
|
|
233
|
+
raise NotImplementedError, "Subclass must implement"
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Download a resource from GitHub.
|
|
237
|
+
#
|
|
238
|
+
# @param resource_id [String] The resource identifier
|
|
239
|
+
# @return [Object, nil] Downloaded resource or nil on error
|
|
240
|
+
# @raise [Kotoshu::DictionaryNotFoundError] when the fetch fails
|
|
241
|
+
# (unreachable source, HTTP error); the documented cache API
|
|
242
|
+
# boundary raises the public error class rather than leaking
|
|
243
|
+
# transport errors
|
|
244
|
+
# @raise [Kotoshu::ResourceNotCachedError] in offline mode —
|
|
245
|
+
# only explicit paths (Kotoshu.setup, kotoshu cache download)
|
|
246
|
+
# reach here, so fail loudly instead of silently skipping
|
|
247
|
+
def download(resource_id)
|
|
248
|
+
return nil unless supports_resource?(resource_id)
|
|
249
|
+
|
|
250
|
+
language, explicit_type = resource_id.to_s.split(":", 2)
|
|
251
|
+
raise Kotoshu::ResourceNotCachedError.new(language, explicit_type || cache_kind) if offline?
|
|
252
|
+
|
|
253
|
+
resource_dir = resource_dir_for(resource_id)
|
|
254
|
+
FileUtils.mkdir_p(resource_dir)
|
|
255
|
+
|
|
256
|
+
begin
|
|
257
|
+
download_resource(resource_id, resource_dir)
|
|
258
|
+
rescue Kotoshu::DictionaryNotFoundError
|
|
259
|
+
raise
|
|
260
|
+
rescue StandardError => e
|
|
261
|
+
warn "Error downloading #{resource_id}: #{e.message}" if $VERBOSE
|
|
262
|
+
nil
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# Abstract: Download a specific resource.
|
|
267
|
+
#
|
|
268
|
+
# @param resource_id [String] The resource identifier
|
|
269
|
+
# @param dest_path [String] Destination directory
|
|
270
|
+
# @return [Object] Downloaded resource
|
|
271
|
+
# @abstract Subclass must implement
|
|
272
|
+
def download_resource(resource_id, dest_path)
|
|
273
|
+
raise NotImplementedError, "Subclass must implement"
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# True when downloads are forbidden (config.offline /
|
|
277
|
+
# KOTOSHU_OFFLINE=1). Checked in #download so every cache
|
|
278
|
+
# subclass inherits the enforcement.
|
|
279
|
+
#
|
|
280
|
+
# @return [Boolean] Whether offline mode is on
|
|
281
|
+
def offline?
|
|
282
|
+
Kotoshu.configuration.offline
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Short label for error messages ("frequency", "language",
|
|
286
|
+
# "model") derived from the nearest named class — anonymous
|
|
287
|
+
# subclasses (test recorders) fall back to their parent's name.
|
|
288
|
+
#
|
|
289
|
+
# @return [String] The cache kind
|
|
290
|
+
def cache_kind
|
|
291
|
+
klass = self.class
|
|
292
|
+
klass = klass.superclass while klass && klass.name.nil?
|
|
293
|
+
klass&.name.to_s[/(\w+)Cache\z/, 1]&.downcase || "resource"
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Abstract: Load cached resource data.
|
|
297
|
+
#
|
|
298
|
+
# @param resource_id [String] The resource identifier
|
|
299
|
+
# @return [Object, nil] Loaded resource or nil
|
|
300
|
+
# @abstract Subclass must implement
|
|
301
|
+
def load_cached(resource_id)
|
|
302
|
+
raise NotImplementedError, "Subclass must implement"
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Abstract: Get metadata file path for a resource.
|
|
306
|
+
#
|
|
307
|
+
# @param resource_id [String] The resource identifier
|
|
308
|
+
# @return [String] Metadata file path
|
|
309
|
+
# @abstract Subclass must implement
|
|
310
|
+
def metadata_path_for(resource_id)
|
|
311
|
+
raise NotImplementedError, "Subclass must implement"
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# Abstract: Get resource directory path.
|
|
315
|
+
#
|
|
316
|
+
# @param resource_id [String] The resource identifier
|
|
317
|
+
# @return [String] Resource directory path
|
|
318
|
+
# @abstract Subclass must implement
|
|
319
|
+
def resource_dir_for(resource_id)
|
|
320
|
+
raise NotImplementedError, "Subclass must implement"
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Abstract: Check if all resource files exist.
|
|
324
|
+
#
|
|
325
|
+
# @param resource_id [String] The resource identifier
|
|
326
|
+
# @return [Boolean] True if all files exist
|
|
327
|
+
# @abstract Subclass must implement
|
|
328
|
+
def resource_files_exist?(resource_id)
|
|
329
|
+
raise NotImplementedError, "Subclass must implement"
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
protected
|
|
333
|
+
|
|
334
|
+
# Download content from a URL.
|
|
335
|
+
#
|
|
336
|
+
# Fetch failures — unreachable host, transport errors, HTTP error
|
|
337
|
+
# statuses — are mapped to {Kotoshu::DictionaryNotFoundError} so
|
|
338
|
+
# callers see the documented public error at the cache boundary.
|
|
339
|
+
#
|
|
340
|
+
# @param url [String] URL to download
|
|
341
|
+
# @return [String] Downloaded content
|
|
342
|
+
# @raise [Kotoshu::DictionaryNotFoundError] when the fetch fails
|
|
343
|
+
def download_url(url)
|
|
344
|
+
# Transport-level backstop: no bytes move in offline mode even
|
|
345
|
+
# for callers that bypass #download (e.g. get_* fallbacks).
|
|
346
|
+
raise "Refusing to download #{url}: offline mode is enabled (KOTOSHU_OFFLINE=1)" if offline?
|
|
347
|
+
|
|
348
|
+
uri = URI.parse(url)
|
|
349
|
+
|
|
350
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
351
|
+
http.use_ssl = (uri.scheme == "https")
|
|
352
|
+
http.open_timeout = 10
|
|
353
|
+
http.read_timeout = 30
|
|
354
|
+
|
|
355
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
|
356
|
+
|
|
357
|
+
response = http.request(request)
|
|
358
|
+
|
|
359
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
360
|
+
raise download_failure(url, "#{response.code} #{response.message}")
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
response.body
|
|
364
|
+
rescue SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
|
|
365
|
+
Errno::ENETUNREACH, Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout,
|
|
366
|
+
Net::ProtocolError, EOFError, OpenSSL::SSL::SSLError => e
|
|
367
|
+
raise download_failure(url, "#{e.class}: #{e.message}")
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# Build the public error for a failed fetch. The URL is stored as
|
|
371
|
+
# the error's #path so callers can report which source failed.
|
|
372
|
+
#
|
|
373
|
+
# @param url [String] URL that could not be fetched
|
|
374
|
+
# @param reason [String] Human-readable failure reason
|
|
375
|
+
# @return [Kotoshu::DictionaryNotFoundError]
|
|
376
|
+
def download_failure(url, reason)
|
|
377
|
+
Kotoshu::DictionaryNotFoundError.new(url, "Failed to download #{url}: #{reason}")
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Download a file to disk, streaming in chunks.
|
|
381
|
+
#
|
|
382
|
+
# @param url [String] Source URL
|
|
383
|
+
# @param dest_path [String] Destination file path
|
|
384
|
+
# @param reporter [#start,#update,#maybe_report_periodic,#finish,nil]
|
|
385
|
+
# Optional progress reporter. Defaults to
|
|
386
|
+
# Kotoshu.configuration.download_reporter (typically nil for
|
|
387
|
+
# programmatic use, set by the CLI during setup).
|
|
388
|
+
def download_file(url, dest_path, reporter: nil)
|
|
389
|
+
# Transport-level backstop, same as download_url.
|
|
390
|
+
raise "Refusing to download #{url}: offline mode is enabled (KOTOSHU_OFFLINE=1)" if offline?
|
|
391
|
+
|
|
392
|
+
reporter ||= Kotoshu.configuration.download_reporter
|
|
393
|
+
uri = URI.parse(url)
|
|
394
|
+
|
|
395
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
396
|
+
http.use_ssl = (uri.scheme == "https")
|
|
397
|
+
http.open_timeout = 30
|
|
398
|
+
http.read_timeout = 300
|
|
399
|
+
|
|
400
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
|
401
|
+
|
|
402
|
+
http.request(request) do |response|
|
|
403
|
+
case response
|
|
404
|
+
when Net::HTTPSuccess
|
|
405
|
+
content_length = content_length_from(response)
|
|
406
|
+
FileUtils.mkdir_p(File.dirname(dest_path))
|
|
407
|
+
received = 0
|
|
408
|
+
reporter&.start(content_length)
|
|
409
|
+
File.open(dest_path, "wb") do |file|
|
|
410
|
+
response.read_body do |chunk|
|
|
411
|
+
file.write(chunk)
|
|
412
|
+
received += chunk.bytesize
|
|
413
|
+
reporter&.update(received)
|
|
414
|
+
reporter&.maybe_report_periodic
|
|
415
|
+
end
|
|
416
|
+
end
|
|
417
|
+
reporter&.finish
|
|
418
|
+
when Net::HTTPRedirection
|
|
419
|
+
download_file(response["location"], dest_path, reporter: reporter)
|
|
420
|
+
else
|
|
421
|
+
raise "Failed to download #{url}: #{response.code} #{response.message}"
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
# Extract Content-Length safely. Some servers omit it (chunked
|
|
427
|
+
# transfer encoding); caller treats nil as "size unknown".
|
|
428
|
+
# @param response [Net::HTTPResponse]
|
|
429
|
+
# @return [Integer, nil]
|
|
430
|
+
def content_length_from(response)
|
|
431
|
+
raw = response["Content-Length"]
|
|
432
|
+
return nil if raw.nil? || raw.strip.empty?
|
|
433
|
+
|
|
434
|
+
Integer(raw)
|
|
435
|
+
rescue ArgumentError
|
|
436
|
+
nil
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# Write metadata to file.
|
|
440
|
+
#
|
|
441
|
+
# @param path [String] Metadata file path
|
|
442
|
+
# @param metadata [Hash] Metadata to write
|
|
443
|
+
def write_metadata(path, metadata)
|
|
444
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
445
|
+
File.write(path, JSON.pretty_generate(metadata))
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
# Check if cached file exists.
|
|
449
|
+
#
|
|
450
|
+
# @param metadata_path [String] Path to metadata file
|
|
451
|
+
# @return [Boolean] True if cached
|
|
452
|
+
def cached?(metadata_path)
|
|
453
|
+
File.exist?(metadata_path)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# Check if cached file is expired.
|
|
457
|
+
#
|
|
458
|
+
# @param metadata_path [String] Path to metadata file
|
|
459
|
+
# @return [Boolean] True if expired
|
|
460
|
+
def expired?(metadata_path)
|
|
461
|
+
return true unless File.exist?(metadata_path)
|
|
462
|
+
|
|
463
|
+
metadata = read_metadata(metadata_path)
|
|
464
|
+
return true unless metadata
|
|
465
|
+
|
|
466
|
+
cached_time_str = metadata["cached_at"] || metadata["version"]
|
|
467
|
+
return true unless cached_time_str
|
|
468
|
+
|
|
469
|
+
begin
|
|
470
|
+
cached_time = Time.iso8601(cached_time_str)
|
|
471
|
+
Time.now.utc - cached_time > @cache_ttl
|
|
472
|
+
rescue StandardError
|
|
473
|
+
true
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
# Calculate checksum of content.
|
|
478
|
+
#
|
|
479
|
+
# @param content [String] Content to checksum
|
|
480
|
+
# @return [String] SHA256 checksum
|
|
481
|
+
def checksum(content)
|
|
482
|
+
Digest::SHA256.hexdigest(content)
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
# Verify downloaded content against the manifest and log to audit.
|
|
486
|
+
#
|
|
487
|
+
# If a manifest is published for this cache's content repo, the content's
|
|
488
|
+
# SHA-256 is checked against the manifest entry for `relative_path`. A
|
|
489
|
+
# mismatch raises {Kotoshu::IntegrityError} — callers MUST remove the
|
|
490
|
+
# corrupt bytes from disk so the next call re-downloads. When no manifest
|
|
491
|
+
# entry exists (kotoshu/dictionaries hasn't shipped one yet), the content
|
|
492
|
+
# is logged as `"unverified"` and accepted — graceful degradation.
|
|
493
|
+
#
|
|
494
|
+
# @param url [String] Source URL (for audit log)
|
|
495
|
+
# @param relative_path [String] Manifest lookup key (e.g., "en/spelling/index.dic")
|
|
496
|
+
# @param content [String] Downloaded bytes
|
|
497
|
+
# @param resource_id [String, nil] Caller-supplied resource identifier
|
|
498
|
+
# @return [void]
|
|
499
|
+
def verify_and_audit(url:, relative_path:, content:, resource_id: nil)
|
|
500
|
+
sha = Digest::SHA256.hexdigest(content)
|
|
501
|
+
entry = manifest_entry_for(relative_path)
|
|
502
|
+
|
|
503
|
+
if entry.nil?
|
|
504
|
+
@audit_log.record(
|
|
505
|
+
url: url, status: "unverified", size: content.bytesize,
|
|
506
|
+
sha256: sha, manifest_sha256: nil, resource_id: resource_id
|
|
507
|
+
)
|
|
508
|
+
return
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
if sha == entry.sha256
|
|
512
|
+
@audit_log.record(
|
|
513
|
+
url: url, status: "verified", size: content.bytesize,
|
|
514
|
+
sha256: sha, manifest_sha256: entry.sha256, resource_id: resource_id
|
|
515
|
+
)
|
|
516
|
+
else
|
|
517
|
+
@audit_log.record(
|
|
518
|
+
url: url, status: "mismatch", size: content.bytesize,
|
|
519
|
+
sha256: sha, manifest_sha256: entry.sha256, resource_id: resource_id
|
|
520
|
+
)
|
|
521
|
+
raise Kotoshu::IntegrityError.new(
|
|
522
|
+
relative_path, expected: entry.sha256, actual: sha, url: url
|
|
523
|
+
)
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
# Pin used in URL templates (default "main"; override via constructor
|
|
528
|
+
# or KOTOSHU_RESOURCE_PIN env var through Configuration).
|
|
529
|
+
#
|
|
530
|
+
# @return [String]
|
|
531
|
+
attr_reader :resource_pin
|
|
532
|
+
|
|
533
|
+
private
|
|
534
|
+
|
|
535
|
+
# Look up a manifest entry by relative path. Loads the manifest
|
|
536
|
+
# lazily on first call; treats HTTP 404/410 as "no manifest" (returns
|
|
537
|
+
# nil) so verification is gracefully skipped.
|
|
538
|
+
def manifest_entry_for(relative_path)
|
|
539
|
+
load_manifest! unless @manifest_loaded
|
|
540
|
+
@manifest&.fetch(relative_path)
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
# Fetch the manifest once per cache instance. Sets @manifest_loaded
|
|
544
|
+
# regardless of outcome so we don't retry on every download.
|
|
545
|
+
def load_manifest!
|
|
546
|
+
@manifest_loaded = true
|
|
547
|
+
url = manifest_url
|
|
548
|
+
return unless url
|
|
549
|
+
|
|
550
|
+
begin
|
|
551
|
+
@manifest = Kotoshu::Integrity::Manifest.load(url)
|
|
552
|
+
rescue StandardError => e
|
|
553
|
+
warn "Manifest fetch failed for #{url}: #{e.message}" if $VERBOSE
|
|
554
|
+
@manifest = nil
|
|
555
|
+
end
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
# Default manifest URL — subclasses override to point at their repo's
|
|
559
|
+
# manifest.json. Returns nil to opt out of manifest verification.
|
|
560
|
+
def manifest_url
|
|
561
|
+
@manifest_url
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
# Get cache size in bytes.
|
|
565
|
+
#
|
|
566
|
+
# @return [Integer] Total size in bytes
|
|
567
|
+
def cache_size
|
|
568
|
+
total = 0
|
|
569
|
+
Dir.glob(File.join(@cache_path, "**", "*")).each do |path|
|
|
570
|
+
total += File.size(path) if File.file?(path)
|
|
571
|
+
end
|
|
572
|
+
total
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
# Get oldest cached entry timestamp.
|
|
576
|
+
#
|
|
577
|
+
# @return [String, nil] ISO8601 timestamp or nil
|
|
578
|
+
def oldest_entry
|
|
579
|
+
oldest = nil
|
|
580
|
+
|
|
581
|
+
Dir.glob(File.join(@cache_path, "**", "metadata.json")).each do |metadata_path|
|
|
582
|
+
metadata = read_metadata(metadata_path)
|
|
583
|
+
next unless metadata
|
|
584
|
+
|
|
585
|
+
timestamp = metadata["cached_at"] || metadata["version"]
|
|
586
|
+
next unless timestamp
|
|
587
|
+
|
|
588
|
+
oldest = timestamp if oldest.nil? || timestamp < oldest
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
oldest
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# Clean expired cache entries.
|
|
595
|
+
#
|
|
596
|
+
# @return [Integer] Number of entries removed
|
|
597
|
+
def clean_expired
|
|
598
|
+
count = 0
|
|
599
|
+
|
|
600
|
+
Dir.glob(File.join(@cache_path, "**", "metadata.json")).each do |metadata_path|
|
|
601
|
+
next unless expired?(metadata_path)
|
|
602
|
+
|
|
603
|
+
dir_path = File.dirname(metadata_path)
|
|
604
|
+
FileUtils.rm_rf(dir_path)
|
|
605
|
+
count += 1
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
count
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
# Clean cache entries by size.
|
|
612
|
+
#
|
|
613
|
+
# @return [Integer] Bytes reclaimed
|
|
614
|
+
def clean_by_size
|
|
615
|
+
0 # Override in subclass if needed
|
|
616
|
+
end
|
|
617
|
+
|
|
618
|
+
# Walk the cache directory and build one eviction-entry record
|
|
619
|
+
# per discovered metadata.json. Each record carries the resource
|
|
620
|
+
# directory path (what to remove), total size of files under it,
|
|
621
|
+
# and the +cached_at+ timestamp from metadata. Entries whose
|
|
622
|
+
# metadata cannot be parsed still appear (with nil +cached_at+)
|
|
623
|
+
# so they are not stranded forever.
|
|
624
|
+
#
|
|
625
|
+
# @return [Array<Hash>] { path:, size:, cached_at: }
|
|
626
|
+
def collect_eviction_entries
|
|
627
|
+
Dir.glob(File.join(@cache_path, "**", "metadata.json")).map do |metadata_path|
|
|
628
|
+
dir = File.dirname(metadata_path)
|
|
629
|
+
metadata = read_metadata(metadata_path) || {}
|
|
630
|
+
{
|
|
631
|
+
path: dir,
|
|
632
|
+
size: dir_size(dir),
|
|
633
|
+
cached_at: metadata["cached_at"] || metadata["version"]
|
|
634
|
+
}
|
|
635
|
+
end
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
# Sum the size of every regular file under +path+, recursively.
|
|
639
|
+
#
|
|
640
|
+
# @param path [String] directory to measure
|
|
641
|
+
# @return [Integer] total bytes
|
|
642
|
+
def dir_size(path)
|
|
643
|
+
total = 0
|
|
644
|
+
Dir.glob(File.join(path, "**", "*")).each do |p|
|
|
645
|
+
total += File.size(p) if File.file?(p)
|
|
646
|
+
end
|
|
647
|
+
total
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
# Parse resource identifier into components.
|
|
651
|
+
#
|
|
652
|
+
# @param resource_id [String] The resource identifier (e.g., "en:spelling" or "en:fasttext")
|
|
653
|
+
# @return [Array<String>, nil] Array of parts or nil if invalid
|
|
654
|
+
def parse_resource_id(resource_id)
|
|
655
|
+
parts = resource_id.split(":")
|
|
656
|
+
return nil unless parts.size == 2
|
|
657
|
+
|
|
658
|
+
parts
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
# Extract language code from resource identifier.
|
|
662
|
+
#
|
|
663
|
+
# @param resource_id [String] The resource identifier
|
|
664
|
+
# @return [String, nil] Language code or nil if invalid
|
|
665
|
+
def extract_language(resource_id)
|
|
666
|
+
parts = parse_resource_id(resource_id)
|
|
667
|
+
return nil unless parts
|
|
668
|
+
|
|
669
|
+
parts[0]
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
# Extract resource type from resource identifier.
|
|
673
|
+
#
|
|
674
|
+
# @param resource_id [String] The resource identifier
|
|
675
|
+
# @return [String, nil] Resource type or nil if invalid
|
|
676
|
+
def extract_type(resource_id)
|
|
677
|
+
parts = parse_resource_id(resource_id)
|
|
678
|
+
return nil unless parts
|
|
679
|
+
|
|
680
|
+
parts[1]
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
# Default cache path: $XDG_CACHE_HOME/kotoshu
|
|
684
|
+
#
|
|
685
|
+
# @return [String] Default cache path
|
|
686
|
+
def default_cache_path
|
|
687
|
+
Kotoshu::Paths.cache_path
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
# Default URL base.
|
|
691
|
+
#
|
|
692
|
+
# @return [String] Default URL base
|
|
693
|
+
def default_url_base
|
|
694
|
+
Kotoshu::SourceRegistry::DEFAULT_BASE_URL
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
# Default source registry — pulls from global Configuration so
|
|
698
|
+
# ENV (KOTOSHU_REPOS_BASE_URL, KOTOSHU_DICTIONARIES_PIN, etc.)
|
|
699
|
+
# and programmatic config reach the cache layer automatically.
|
|
700
|
+
#
|
|
701
|
+
# @return [Kotoshu::SourceRegistry]
|
|
702
|
+
def default_source_registry
|
|
703
|
+
Kotoshu::Configuration.instance.source_registry
|
|
704
|
+
end
|
|
705
|
+
|
|
706
|
+
# Registry for an explicitly-passed url_base. The deprecated
|
|
707
|
+
# `url_base:` constructor parameter predates SourceRegistry; when
|
|
708
|
+
# it is given it must still drive downloads, so derive a registry
|
|
709
|
+
# rooted at that base instead of silently ignoring it.
|
|
710
|
+
#
|
|
711
|
+
# @param url_base [String, nil] Explicit base URL from the caller
|
|
712
|
+
# @return [Kotoshu::SourceRegistry]
|
|
713
|
+
def source_registry_for(url_base)
|
|
714
|
+
return default_source_registry unless url_base
|
|
715
|
+
|
|
716
|
+
Kotoshu::SourceRegistry.new(base_url: url_base)
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
# Default GitHub URL.
|
|
720
|
+
#
|
|
721
|
+
# @return [String] Default GitHub URL
|
|
722
|
+
def default_github_url
|
|
723
|
+
"https://github.com/kotoshu"
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
# Default cache TTL (7 days).
|
|
727
|
+
#
|
|
728
|
+
# @return [Integer] Default TTL in seconds
|
|
729
|
+
def default_cache_ttl
|
|
730
|
+
604_800
|
|
731
|
+
end
|
|
732
|
+
|
|
733
|
+
# Default maximum cache size — pulled from
|
|
734
|
+
# +Kotoshu::Configuration.instance.max_cache_size+ so ENV
|
|
735
|
+
# overrides (+KOTOSHU_MAX_CACHE_SIZE+) and programmatic settings
|
|
736
|
+
# flow through to {#evict} naturally.
|
|
737
|
+
#
|
|
738
|
+
# @return [Integer] Default max cache size in bytes
|
|
739
|
+
def default_max_cache_size
|
|
740
|
+
Kotoshu::Configuration.instance.max_cache_size
|
|
741
|
+
end
|
|
742
|
+
|
|
743
|
+
# Build a default audit log bound to the configured rotation
|
|
744
|
+
# policy. Pulls +audit_max_bytes+ and +audit_rotations+ from
|
|
745
|
+
# +Kotoshu::Configuration.instance+ so ENV overrides
|
|
746
|
+
# (+KOTOSHU_AUDIT_MAX_BYTES+, +KOTOSHU_AUDIT_ROTATIONS+) and
|
|
747
|
+
# programmatic settings flow through naturally.
|
|
748
|
+
#
|
|
749
|
+
# @return [Kotoshu::Integrity::AuditLog]
|
|
750
|
+
def default_audit_log
|
|
751
|
+
config = Kotoshu::Configuration.instance
|
|
752
|
+
policy = Kotoshu::Integrity::RotationPolicy.new(
|
|
753
|
+
max_bytes: config.audit_max_bytes,
|
|
754
|
+
rotations: config.audit_rotations
|
|
755
|
+
)
|
|
756
|
+
Kotoshu::Integrity::AuditLog.new(rotation_policy: policy)
|
|
757
|
+
end
|
|
758
|
+
|
|
759
|
+
public
|
|
760
|
+
|
|
761
|
+
# Read metadata from file.
|
|
762
|
+
#
|
|
763
|
+
# Public because the CLI cache subcommand and other inspection
|
|
764
|
+
# paths consume it to display cache state without going through
|
|
765
|
+
# the full download-and-load pipeline.
|
|
766
|
+
#
|
|
767
|
+
# @param path [String] Metadata file path
|
|
768
|
+
# @return [Hash, nil] Metadata or nil
|
|
769
|
+
def read_metadata(path)
|
|
770
|
+
return nil unless File.exist?(path)
|
|
771
|
+
|
|
772
|
+
JSON.parse(File.read(path))
|
|
773
|
+
rescue JSON::ParserError
|
|
774
|
+
nil
|
|
775
|
+
end
|
|
776
|
+
end
|
|
777
|
+
end
|
|
778
|
+
end
|