kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Cache
|
|
5
|
+
# Base cache interface.
|
|
6
|
+
#
|
|
7
|
+
# All cache implementations should follow this interface.
|
|
8
|
+
#
|
|
9
|
+
# @abstract Subclass must implement {#fetch}, {#write}, {#read}, {#delete}, {#clear}
|
|
10
|
+
module Cache
|
|
11
|
+
# Retrieve a value from cache, or compute it.
|
|
12
|
+
#
|
|
13
|
+
# @param key [Object] The cache key
|
|
14
|
+
# @yield Block to compute value on cache miss
|
|
15
|
+
# @return [Object] The cached or computed value
|
|
16
|
+
# @abstract Subclass must implement
|
|
17
|
+
def fetch(key, &block)
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Write a value to cache.
|
|
22
|
+
#
|
|
23
|
+
# @param key [Object] The cache key
|
|
24
|
+
# @param value [Object] The value to store
|
|
25
|
+
# @return [Object] The stored value
|
|
26
|
+
# @abstract Subclass must implement
|
|
27
|
+
def write(key, value)
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Read a value from cache.
|
|
32
|
+
#
|
|
33
|
+
# @param key [Object] The cache key
|
|
34
|
+
# @return [Object, nil] The cached value or nil
|
|
35
|
+
# @abstract Subclass must implement
|
|
36
|
+
def read(key)
|
|
37
|
+
raise NotImplementedError
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Delete a value from cache.
|
|
41
|
+
#
|
|
42
|
+
# @param key [Object] The cache key
|
|
43
|
+
# @return [Object, nil] The deleted value or nil
|
|
44
|
+
# @abstract Subclass must implement
|
|
45
|
+
def delete(key)
|
|
46
|
+
raise NotImplementedError
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Clear all entries from cache.
|
|
50
|
+
#
|
|
51
|
+
# @return [self] Self for chaining
|
|
52
|
+
# @abstract Subclass must implement
|
|
53
|
+
def clear
|
|
54
|
+
raise NotImplementedError
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Check if key exists in cache.
|
|
58
|
+
#
|
|
59
|
+
# @param key [Object] The cache key
|
|
60
|
+
# @return [Boolean] True if key exists
|
|
61
|
+
# @abstract Subclass must implement
|
|
62
|
+
def key?(key)
|
|
63
|
+
raise NotImplementedError
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Get number of entries in cache.
|
|
67
|
+
#
|
|
68
|
+
# @return [Integer] Number of entries
|
|
69
|
+
# @abstract Subclass must implement
|
|
70
|
+
def size
|
|
71
|
+
raise NotImplementedError
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Get cache statistics.
|
|
75
|
+
#
|
|
76
|
+
# @return [Hash] Statistics including :hits, :misses, :size, :hit_rate
|
|
77
|
+
# @abstract Subclass must implement
|
|
78
|
+
def stats
|
|
79
|
+
raise NotImplementedError
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Reset statistics counters.
|
|
83
|
+
#
|
|
84
|
+
# @return [self] Self for chaining
|
|
85
|
+
# @abstract Subclass must implement
|
|
86
|
+
def reset_stats
|
|
87
|
+
raise NotImplementedError
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Cache
|
|
5
|
+
# Pure value object: given a max total size and a list of cache
|
|
6
|
+
# entries, decide which entries to evict to fit under the cap.
|
|
7
|
+
#
|
|
8
|
+
# The policy is LRU by +cached_at+ timestamp (oldest evicted first).
|
|
9
|
+
# It performs no IO and has no concept of what an "entry" actually
|
|
10
|
+
# is on disk — it just sees a flat list of records each carrying
|
|
11
|
+
# +:path+, +:size+, +:cached_at+, and returns a plan. BaseCache#evict
|
|
12
|
+
# is responsible for collecting those records and executing the plan.
|
|
13
|
+
#
|
|
14
|
+
# ISO8601 timestamps sort lexicographically because they are
|
|
15
|
+
# zero-padded, so the policy can compare them as plain strings
|
|
16
|
+
# without parsing. Entries with a nil +cached_at+ sort oldest
|
|
17
|
+
# (treated as the epoch) so they are evicted first — a corrupt
|
|
18
|
+
# metadata.json should not strand an entry forever.
|
|
19
|
+
class EvictionPolicy
|
|
20
|
+
# @return [Integer] maximum total size in bytes
|
|
21
|
+
attr_reader :max_size
|
|
22
|
+
|
|
23
|
+
# @param max_size [Integer, #to_i] maximum total size in bytes
|
|
24
|
+
# @raise [ArgumentError] if max_size is negative
|
|
25
|
+
def initialize(max_size:)
|
|
26
|
+
@max_size = max_size.to_i
|
|
27
|
+
return unless @max_size.negative?
|
|
28
|
+
|
|
29
|
+
raise ArgumentError, "max_size must be >= 0"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Decide which entries to evict.
|
|
33
|
+
#
|
|
34
|
+
# @param entries [Array<Hash>] each hash has :path (String),
|
|
35
|
+
# :size (Integer), :cached_at (String, ISO8601, or nil)
|
|
36
|
+
# @return [Hash] { evict: Array<Hash>, keep: Array<Hash>,
|
|
37
|
+
# bytes_reclaimed: Integer }
|
|
38
|
+
def plan(entries)
|
|
39
|
+
sorted = entries.sort_by { |e| sort_key(e[:cached_at]) }
|
|
40
|
+
total = sorted.sum { |e| e[:size].to_i }
|
|
41
|
+
|
|
42
|
+
return { evict: [], keep: sorted, bytes_reclaimed: 0 } if total <= max_size
|
|
43
|
+
|
|
44
|
+
evict = []
|
|
45
|
+
reclaimed = 0
|
|
46
|
+
sorted.each do |e|
|
|
47
|
+
break if total - reclaimed <= max_size
|
|
48
|
+
|
|
49
|
+
evict << e
|
|
50
|
+
reclaimed += e[:size].to_i
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
{ evict: evict, keep: sorted - evict, bytes_reclaimed: reclaimed }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
# ISO8601 strings sort correctly as strings; nil sorts first so
|
|
59
|
+
# entries with missing timestamps are evicted ahead of any valid
|
|
60
|
+
# entry.
|
|
61
|
+
def sort_key(cached_at)
|
|
62
|
+
cached_at || ""
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# base_cache autoloaded via Kotoshu::Cache
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cache
|
|
7
|
+
# Frequency cache for Kelly Project frequency lists.
|
|
8
|
+
#
|
|
9
|
+
# Manages Kelly frequency list downloads from the kotoshu/frequency-list-kelly
|
|
10
|
+
# repository. Resources are cached locally in `$XDG_CACHE_HOME/kotoshu/frequency-lists/{code}/`
|
|
11
|
+
# with metadata for versioning and expiration.
|
|
12
|
+
#
|
|
13
|
+
# Extends BaseCache for common download, metadata, and validation logic.
|
|
14
|
+
#
|
|
15
|
+
# @example Getting cached frequency data
|
|
16
|
+
# cache = FrequencyCache.new
|
|
17
|
+
# result = cache.get('en')
|
|
18
|
+
# # => { frequency_path: "~/.cache/kotoshu/frequency-lists/en/frequency.json",
|
|
19
|
+
# # tiers: { top_50: Set<...>, top_200: Set<...>, top_1000: Set<...> },
|
|
20
|
+
# # metadata: { ... } }
|
|
21
|
+
#
|
|
22
|
+
# @example Checking if frequency data is available
|
|
23
|
+
# cache = FrequencyCache.new
|
|
24
|
+
# available = cache.available?('en')
|
|
25
|
+
# # => true
|
|
26
|
+
class FrequencyCache < BaseCache
|
|
27
|
+
# Kelly Project languages available
|
|
28
|
+
KELLY_LANGUAGES = %w[ar zh en el it no ru sv].freeze
|
|
29
|
+
|
|
30
|
+
# GitHub repository for Kelly frequency lists
|
|
31
|
+
GITHUB_REPO = "kotoshu/frequency-list-kelly"
|
|
32
|
+
GITHUB_BRANCH = "main"
|
|
33
|
+
|
|
34
|
+
# Get list of available languages.
|
|
35
|
+
#
|
|
36
|
+
# @return [Array<String>] List of available language codes
|
|
37
|
+
def available_languages
|
|
38
|
+
KELLY_LANGUAGES.dup
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Get frequency data for a language (alias for get).
|
|
42
|
+
#
|
|
43
|
+
# @param language_code [String] ISO 639-1 language code
|
|
44
|
+
# @param force_download [Boolean] Force re-download even if cached
|
|
45
|
+
# @return [Hash, nil] Frequency data with :frequency_path, :tiers, :metadata keys
|
|
46
|
+
def get_frequency(language_code, force_download: false)
|
|
47
|
+
get(language_code, force_download: force_download)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Check if a resource type is supported.
|
|
51
|
+
#
|
|
52
|
+
# @param resource_id [String] The resource identifier (language code)
|
|
53
|
+
# @return [Boolean] True if supported
|
|
54
|
+
def supports_resource?(resource_id)
|
|
55
|
+
KELLY_LANGUAGES.include?(resource_id)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The frequency cache always roots under .../frequency-lists,
|
|
59
|
+
# including when a base cache path is injected (ResourceManager
|
|
60
|
+
# passes the shared cfg.cache_path, the CLI passes its root):
|
|
61
|
+
# without the suffix, writers and readers split across two
|
|
62
|
+
# layouts and the metadata goes permanently stale (plan 117).
|
|
63
|
+
#
|
|
64
|
+
# @param cache_path [String, nil] base cache directory
|
|
65
|
+
def initialize(cache_path: nil, **kwargs)
|
|
66
|
+
base = cache_path || default_cache_path
|
|
67
|
+
unless base.end_with?("frequency-lists")
|
|
68
|
+
base = File.join(base, "frequency-lists")
|
|
69
|
+
end
|
|
70
|
+
super(cache_path: base, **kwargs)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Install a local frequency file into the cache without going
|
|
74
|
+
# through the network. Mirrors {LanguageCache#install_local} for
|
|
75
|
+
# the frequency resource type — symlinks the user's file into the
|
|
76
|
+
# cache layout and writes a `local-source` metadata record so
|
|
77
|
+
# subsequent {#available?} / {#get} calls find it.
|
|
78
|
+
#
|
|
79
|
+
# @param language_code [String] ISO 639-1 language code
|
|
80
|
+
# @param path [String] Path to the local frequency.json file
|
|
81
|
+
# @param force [Boolean] Overwrite an existing install
|
|
82
|
+
# @return [Hash] { frequency_path:, metadata_path:, source: :local }
|
|
83
|
+
def install_local(language_code, path:, force: false)
|
|
84
|
+
lang_path = language_dir(language_code)
|
|
85
|
+
FileUtils.mkdir_p(lang_path)
|
|
86
|
+
|
|
87
|
+
target_frequency = File.join(lang_path, "frequency.json")
|
|
88
|
+
target_metadata = metadata_path_for(language_code)
|
|
89
|
+
|
|
90
|
+
if File.exist?(target_frequency) || File.symlink?(target_frequency)
|
|
91
|
+
raise ArgumentError, "#{target_frequency} already exists (use force: true to overwrite)" unless force
|
|
92
|
+
|
|
93
|
+
File.unlink(target_frequency)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
File.symlink(File.expand_path(path), target_frequency)
|
|
97
|
+
|
|
98
|
+
write_metadata(target_metadata,
|
|
99
|
+
"version" => Time.now.utc.iso8601,
|
|
100
|
+
"url" => "local:#{File.expand_path(path)}",
|
|
101
|
+
"language" => language_code,
|
|
102
|
+
"type" => "kelly_frequency",
|
|
103
|
+
"source" => "local",
|
|
104
|
+
"checksum" => checksum(File.binread(path)),
|
|
105
|
+
"cached_at" => Time.now.utc.iso8601)
|
|
106
|
+
|
|
107
|
+
{ frequency_path: target_frequency, metadata_path: target_metadata, source: :local }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# List all cached resources.
|
|
111
|
+
#
|
|
112
|
+
# @return [Array<String>] List of cached language codes
|
|
113
|
+
def cached_resources
|
|
114
|
+
directories = Dir.glob(File.join(@cache_path, "*")).select do |path|
|
|
115
|
+
basename = File.basename(path)
|
|
116
|
+
# Skip dotfiles AND the working-directory scratch slot that
|
|
117
|
+
# BaseCache#initialize creates at <cache_path>/tmp/.
|
|
118
|
+
File.directory?(path) && !basename.start_with?(".") && basename != "tmp"
|
|
119
|
+
end
|
|
120
|
+
directories.map { |path| File.basename(path) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Load cached resource data (implements abstract method).
|
|
124
|
+
#
|
|
125
|
+
# Public — this is the cache-only reader BaseCache declares
|
|
126
|
+
# publicly; resolve paths and FrequencyProvider use it to read
|
|
127
|
+
# without any download.
|
|
128
|
+
#
|
|
129
|
+
# @param language_code [String] The language code
|
|
130
|
+
# @return [Hash, nil] Loaded frequency data
|
|
131
|
+
def load_cached(language_code)
|
|
132
|
+
frequency_file = frequency_file_path(language_code)
|
|
133
|
+
metadata_path = metadata_path(language_code)
|
|
134
|
+
|
|
135
|
+
return nil unless File.exist?(frequency_file) && File.exist?(metadata_path)
|
|
136
|
+
|
|
137
|
+
metadata = read_metadata(metadata_path)
|
|
138
|
+
return nil unless metadata
|
|
139
|
+
|
|
140
|
+
# Load frequency file
|
|
141
|
+
# CommonWordsLoader autoloaded via Kotoshu::Data
|
|
142
|
+
data = Data::CommonWordsLoader.load_from_frequency_file(frequency_file)
|
|
143
|
+
|
|
144
|
+
{
|
|
145
|
+
frequency_path: frequency_file,
|
|
146
|
+
tiers: data[:tiers],
|
|
147
|
+
metadata: metadata
|
|
148
|
+
}
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
protected
|
|
152
|
+
|
|
153
|
+
# Download a specific resource (implements abstract method).
|
|
154
|
+
#
|
|
155
|
+
# @param language_code [String] The language code
|
|
156
|
+
# @param dest_path [String] Destination directory
|
|
157
|
+
# @return [Hash] Downloaded frequency data
|
|
158
|
+
def download_resource(language_code, dest_path)
|
|
159
|
+
FileUtils.mkdir_p(dest_path)
|
|
160
|
+
|
|
161
|
+
frequency_file = File.join(dest_path, "frequency.json")
|
|
162
|
+
metadata_path = File.join(dest_path, "metadata.json")
|
|
163
|
+
|
|
164
|
+
# Download from GitHub
|
|
165
|
+
url = frequency_url(language_code)
|
|
166
|
+
|
|
167
|
+
warn "Downloading Kelly frequency data for #{language_code} from #{url}..." if $VERBOSE
|
|
168
|
+
|
|
169
|
+
response = download_url(url)
|
|
170
|
+
verify_and_audit(url: url,
|
|
171
|
+
relative_path: "data/#{language_code}.json",
|
|
172
|
+
content: response,
|
|
173
|
+
resource_id: language_code)
|
|
174
|
+
|
|
175
|
+
# Validate it's valid JSON
|
|
176
|
+
JSON.parse(response)
|
|
177
|
+
|
|
178
|
+
# Save frequency file
|
|
179
|
+
File.binwrite(frequency_file, response)
|
|
180
|
+
|
|
181
|
+
# Save metadata
|
|
182
|
+
metadata = {
|
|
183
|
+
version: Time.now.utc.iso8601,
|
|
184
|
+
url: url,
|
|
185
|
+
language: language_code,
|
|
186
|
+
type: "kelly_frequency",
|
|
187
|
+
checksum: checksum(response),
|
|
188
|
+
cached_at: Time.now.utc.iso8601
|
|
189
|
+
}
|
|
190
|
+
write_metadata(metadata_path, metadata)
|
|
191
|
+
|
|
192
|
+
# Load and return the data
|
|
193
|
+
load_cached(language_code)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Get metadata file path for a resource.
|
|
197
|
+
#
|
|
198
|
+
# @param language_code [String] The language code
|
|
199
|
+
# @return [String] Metadata file path
|
|
200
|
+
def metadata_path_for(language_code)
|
|
201
|
+
File.join(language_dir(language_code), "metadata.json")
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Get resource directory path.
|
|
205
|
+
#
|
|
206
|
+
# @param language_code [String] The language code
|
|
207
|
+
# @return [String] Resource directory path
|
|
208
|
+
def resource_dir_for(language_code)
|
|
209
|
+
language_dir(language_code)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Check if all resource files exist.
|
|
213
|
+
#
|
|
214
|
+
# @param language_code [String] The language code
|
|
215
|
+
# @return [Boolean] True if all files exist
|
|
216
|
+
def resource_files_exist?(language_code)
|
|
217
|
+
File.exist?(frequency_file_path(language_code))
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
private
|
|
221
|
+
|
|
222
|
+
# Get the directory path for a language.
|
|
223
|
+
#
|
|
224
|
+
# @param language_code [String] ISO 639-1 language code
|
|
225
|
+
# @return [String] Directory path
|
|
226
|
+
def language_dir(language_code)
|
|
227
|
+
File.join(@cache_path, language_code)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Get the path to the frequency JSON file.
|
|
231
|
+
#
|
|
232
|
+
# @param language_code [String] ISO 639-1 language code
|
|
233
|
+
# @return [String] File path
|
|
234
|
+
def frequency_file_path(language_code)
|
|
235
|
+
File.join(language_dir(language_code), "frequency.json")
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Get the path to the metadata file.
|
|
239
|
+
#
|
|
240
|
+
# @param language_code [String] ISO 639-1 language code
|
|
241
|
+
# @return [String] File path
|
|
242
|
+
def metadata_path(language_code)
|
|
243
|
+
metadata_path_for(language_code)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Get the GitHub URL for a language's frequency file.
|
|
247
|
+
#
|
|
248
|
+
# @param language_code [String] ISO 639-1 language code
|
|
249
|
+
# @return [String] Download URL
|
|
250
|
+
def frequency_url(language_code)
|
|
251
|
+
@source_registry.url_for(:frequency, lang: language_code)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# Kelly repo manifest URL (used for integrity verification).
|
|
255
|
+
#
|
|
256
|
+
# @return [String]
|
|
257
|
+
def manifest_url
|
|
258
|
+
@manifest_url || @source_registry.url_for(:freq_manifest)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Default cache path: $XDG_CACHE_HOME/kotoshu/frequency-lists/
|
|
262
|
+
#
|
|
263
|
+
# @return [String] Default cache directory path
|
|
264
|
+
def default_cache_path
|
|
265
|
+
File.join(Kotoshu::Paths.cache_path, "frequency-lists")
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Default URL base for Kelly frequency lists.
|
|
269
|
+
#
|
|
270
|
+
# @return [String] Default URL base
|
|
271
|
+
def default_url_base
|
|
272
|
+
"https://raw.githubusercontent.com"
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Default cache TTL (7 days).
|
|
276
|
+
#
|
|
277
|
+
# @return [Integer] Default TTL in seconds
|
|
278
|
+
def default_cache_ttl
|
|
279
|
+
604_800
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
end
|