kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,511 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# base_cache autoloaded via Kotoshu::Cache
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cache
|
|
7
|
+
# Language cache for dynamic dictionary and grammar rule downloads.
|
|
8
|
+
#
|
|
9
|
+
# Manages per-language dictionary and grammar rule downloads from a remote
|
|
10
|
+
# repository. Resources are cached locally in `$XDG_CACHE_HOME/kotoshu/languages/{code}/`
|
|
11
|
+
# with metadata for versioning and expiration.
|
|
12
|
+
#
|
|
13
|
+
# Extends BaseCache for common download, metadata, and validation logic.
|
|
14
|
+
#
|
|
15
|
+
# @example Getting a cached spelling dictionary
|
|
16
|
+
# cache = LanguageCache.new
|
|
17
|
+
# result = cache.get('en')
|
|
18
|
+
# # => { aff_path: "~/.cache/kotoshu/languages/en/spelling/index.aff",
|
|
19
|
+
# # dic_path: "~/.cache/kotoshu/languages/en/spelling/index.dic",
|
|
20
|
+
# # metadata: { ... } }
|
|
21
|
+
#
|
|
22
|
+
# @example Checking cache statistics
|
|
23
|
+
# stats = cache.stats
|
|
24
|
+
# # => { hits: 5, misses: 1, hit_rate: 0.83, ... }
|
|
25
|
+
class LanguageCache < BaseCache
|
|
26
|
+
# Supported resource types
|
|
27
|
+
RESOURCE_TYPES = %w[spelling grammar frequency].freeze
|
|
28
|
+
|
|
29
|
+
# Available languages (plan 107): every language staged in the
|
|
30
|
+
# dictionaries manifest, vendored offline-safe by
|
|
31
|
+
# Cache::StagedLanguages and refreshed by
|
|
32
|
+
# `rake kotoshu:staged_languages:sync`. Two tiers:
|
|
33
|
+
# - full feature — a language module under Kotoshu::Languages
|
|
34
|
+
# (registered keyboard + verified specimens); see
|
|
35
|
+
# {full_feature_languages};
|
|
36
|
+
# - basic — manifest-present but module-less: the script
|
|
37
|
+
# fallback tokenizer (Language::Script) and the generic
|
|
38
|
+
# keyboard default still serve setup/check/suggest.
|
|
39
|
+
# Local files come via setup --aff/--dic instead.
|
|
40
|
+
AVAILABLE_LANGUAGES = StagedLanguages::CODES
|
|
41
|
+
|
|
42
|
+
# Get or download spelling dictionary for a language.
|
|
43
|
+
#
|
|
44
|
+
# @param language [String] Language code (e.g., 'en', 'de')
|
|
45
|
+
# @param force_download [Boolean] Force re-download even if cached
|
|
46
|
+
# @return [Hash] Dictionary paths and metadata
|
|
47
|
+
def get_spelling(language, force_download: false)
|
|
48
|
+
resource_id = "#{language}:spelling"
|
|
49
|
+
result = get(resource_id, force_download: force_download)
|
|
50
|
+
result || download_spelling(language)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Install a spelling dictionary from local files (no download).
|
|
54
|
+
# Used by ResourceManager.setup_from_local when the user already
|
|
55
|
+
# has .aff/.dic files on disk. Symlinks the source files into the
|
|
56
|
+
# cache directory so subsequent cache lookups find them. Existing
|
|
57
|
+
# symlinks are replaced when force: true; existing real files
|
|
58
|
+
# raise ArgumentError unless force: true.
|
|
59
|
+
#
|
|
60
|
+
# @param language [String] Language code
|
|
61
|
+
# @param aff [String] Path to .aff file
|
|
62
|
+
# @param dic [String] Path to .dic file
|
|
63
|
+
# @param force [Boolean] Overwrite existing install
|
|
64
|
+
# @return [Hash] Installed paths
|
|
65
|
+
def install_local(language, aff:, dic:, force: false)
|
|
66
|
+
require "fileutils"
|
|
67
|
+
|
|
68
|
+
resource_id = "#{language}:spelling"
|
|
69
|
+
lang_path = resource_dir_for(resource_id)
|
|
70
|
+
FileUtils.mkdir_p(lang_path)
|
|
71
|
+
|
|
72
|
+
target_aff = File.join(lang_path, "index.aff")
|
|
73
|
+
target_dic = File.join(lang_path, "index.dic")
|
|
74
|
+
|
|
75
|
+
if File.exist?(target_aff) || File.symlink?(target_aff)
|
|
76
|
+
raise ArgumentError, "#{target_aff} already exists (use force: true to overwrite)" unless force
|
|
77
|
+
|
|
78
|
+
File.unlink(target_aff)
|
|
79
|
+
end
|
|
80
|
+
if File.exist?(target_dic) || File.symlink?(target_dic)
|
|
81
|
+
raise ArgumentError, "#{target_dic} already exists (use force: true to overwrite)" unless force
|
|
82
|
+
|
|
83
|
+
File.unlink(target_dic)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
File.symlink(File.expand_path(aff), target_aff)
|
|
87
|
+
File.symlink(File.expand_path(dic), target_dic)
|
|
88
|
+
|
|
89
|
+
write_metadata(metadata_path_for(resource_id),
|
|
90
|
+
build_metadata(language, "spelling", "local-source"))
|
|
91
|
+
|
|
92
|
+
{ aff_path: target_aff, dic_path: target_dic, source: :local }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Alias for get_spelling for backward compatibility.
|
|
96
|
+
#
|
|
97
|
+
# @param language [String] Language code
|
|
98
|
+
# @param force_download [Boolean] Force re-download
|
|
99
|
+
# @return [Hash] Dictionary paths and metadata
|
|
100
|
+
def get_dictionary(language, force_download: false)
|
|
101
|
+
get_spelling(language, force_download)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Get or download grammar rules for a language.
|
|
105
|
+
#
|
|
106
|
+
# @param language [String] Language code
|
|
107
|
+
# @param force_download [Boolean] Force re-download
|
|
108
|
+
# @return [Hash] Rules path and metadata
|
|
109
|
+
def get_grammar(language, force_download: false)
|
|
110
|
+
resource_id = "#{language}:grammar"
|
|
111
|
+
result = get(resource_id, force_download: force_download)
|
|
112
|
+
result || download_grammar(language)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Check if frequency data is available for a language.
|
|
116
|
+
#
|
|
117
|
+
# @param language_code [String] ISO 639-1 language code
|
|
118
|
+
# @return [Boolean] True if frequency data exists
|
|
119
|
+
def frequency_available?(language_code)
|
|
120
|
+
resource_id = "#{language_code}:frequency"
|
|
121
|
+
available?(resource_id)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Get list of available languages.
|
|
125
|
+
#
|
|
126
|
+
# @return [Array<String>] List of supported language codes
|
|
127
|
+
def available_languages
|
|
128
|
+
AVAILABLE_LANGUAGES.dup
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Languages on the full-feature tier: staged in the manifest
|
|
132
|
+
# AND wired as a gem module (Kotoshu::Languages) — registered
|
|
133
|
+
# keyboard layout plus engine-verified specimens. Everything
|
|
134
|
+
# else in {available_languages} is the basic tier (plan 107).
|
|
135
|
+
#
|
|
136
|
+
# @return [Array<String>] Sorted full-feature language codes
|
|
137
|
+
def self.full_feature_languages
|
|
138
|
+
Language::Registry.supported_codes
|
|
139
|
+
.map { |code| code.split("-").first }
|
|
140
|
+
.uniq
|
|
141
|
+
.select { |base| AVAILABLE_LANGUAGES.include?(base) }
|
|
142
|
+
.sort
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Absolute on-disk path for a (language, resource_type) pair.
|
|
146
|
+
#
|
|
147
|
+
# Composes `cache_path/languages/{lang}/{type}`. Use this rather
|
|
148
|
+
# than reaching into `cache_path` from outside the cache — it
|
|
149
|
+
# keeps the on-disk layout encapsulated and lets the layout
|
|
150
|
+
# evolve without breaking callers.
|
|
151
|
+
#
|
|
152
|
+
# @param language [String] Language code (e.g., 'en', 'de')
|
|
153
|
+
# @param type [String] Resource type ('spelling', 'grammar', 'frequency')
|
|
154
|
+
# @return [String] Absolute directory path for the resource
|
|
155
|
+
def language_path(language, type)
|
|
156
|
+
File.join(@cache_path, "languages", language, type)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Get language metadata (word count, license, source).
|
|
160
|
+
#
|
|
161
|
+
# @param language_code [String] The language code
|
|
162
|
+
# @return [Hash] Language info
|
|
163
|
+
def language_info(language_code)
|
|
164
|
+
{
|
|
165
|
+
"de" => { name: "German", word_count: 75_873, license: "GPL", source: "igerman98" },
|
|
166
|
+
"en" => { name: "English", word_count: 49_568, license: "LGPL/MPL/GPL", source: "SCOWL" },
|
|
167
|
+
"es" => { name: "Spanish", word_count: 57_344, license: "GPL", source: "LibreOffice" },
|
|
168
|
+
"fr" => { name: "French", word_count: 84_310, license: "MPL 2.0", source: "Grammalecte" },
|
|
169
|
+
"pt" => { name: "Portuguese", word_count: 312_368, license: "LGPLv3 + MPL", source: "VERO" },
|
|
170
|
+
"ru" => { name: "Russian", word_count: 146_269, license: "BSD-style", source: "Alexander Lebedev" }
|
|
171
|
+
}[language_code] || { name: language_code.upcase, word_count: 0, license: "Unknown", source: "Unknown" }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Get cache size in bytes (override for language-specific tracking).
|
|
175
|
+
#
|
|
176
|
+
# @return [Integer] Total size in bytes
|
|
177
|
+
def cache_size
|
|
178
|
+
total = 0
|
|
179
|
+
Dir.glob(File.join(@cache_path, "languages", "**", "*.dic")).each do |path|
|
|
180
|
+
total += File.size(path) if File.file?(path)
|
|
181
|
+
end
|
|
182
|
+
total
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# List all cached resources.
|
|
186
|
+
#
|
|
187
|
+
# @return [Array<String>] List of cached resource identifiers
|
|
188
|
+
def cached_resources
|
|
189
|
+
Dir.glob(File.join(@cache_path, "languages", "**", "metadata.json")).map do |path|
|
|
190
|
+
relative = Pathname.new(path).relative_path_from(Pathname.new(@cache_path))
|
|
191
|
+
parts = relative.to_s.split("/")
|
|
192
|
+
"#{parts[1]}:#{parts[2]}"
|
|
193
|
+
end.uniq
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Check if a resource type is supported.
|
|
197
|
+
#
|
|
198
|
+
# @param resource_id [String] The resource identifier (e.g., "en:spelling")
|
|
199
|
+
# @return [Boolean] True if supported
|
|
200
|
+
def supports_resource?(resource_id)
|
|
201
|
+
parts = resource_id.split(":")
|
|
202
|
+
return false unless parts.size == 2
|
|
203
|
+
|
|
204
|
+
language, type = parts
|
|
205
|
+
AVAILABLE_LANGUAGES.include?(language) && RESOURCE_TYPES.include?(type)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
protected
|
|
209
|
+
|
|
210
|
+
# Download a spelling dictionary.
|
|
211
|
+
#
|
|
212
|
+
# @param language [String] Language code
|
|
213
|
+
# @return [Hash] Dictionary paths and metadata
|
|
214
|
+
# Fetch one spelling file, preferring the spelling/ sublayout and
|
|
215
|
+
# falling back to the flat {lang}/index.{ext} layout. Returns the
|
|
216
|
+
# URL that actually served the bytes alongside the content.
|
|
217
|
+
def fetch_spelling_file(language, ext)
|
|
218
|
+
primary = @source_registry.url_for(:spelling, lang: language, ext: ext)
|
|
219
|
+
begin
|
|
220
|
+
content = download_url(primary)
|
|
221
|
+
return [primary, content] if content
|
|
222
|
+
rescue Kotoshu::DictionaryNotFoundError
|
|
223
|
+
nil # fall through to the flat layout
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
flat = @source_registry.url_for(:spelling_flat, lang: language, ext: ext)
|
|
227
|
+
[flat, download_url(flat)]
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def download_spelling(language)
|
|
231
|
+
raise Kotoshu::ResourceNotCachedError.new(language, "spelling") if offline?
|
|
232
|
+
|
|
233
|
+
lang_path = resource_dir_for("#{language}:spelling")
|
|
234
|
+
resource_id = "#{language}:spelling"
|
|
235
|
+
|
|
236
|
+
# Download index.aff / index.dic. Only `en` ships under the
|
|
237
|
+
# spelling/ sublayout upstream; staged languages sit flat at
|
|
238
|
+
# {lang}/index.* — try the sublayout first, fall back to flat.
|
|
239
|
+
aff_url, aff_content = fetch_spelling_file(language, "aff")
|
|
240
|
+
verify_and_audit(url: aff_url,
|
|
241
|
+
relative_path: "#{language}/spelling/index.aff",
|
|
242
|
+
content: aff_content,
|
|
243
|
+
resource_id: resource_id)
|
|
244
|
+
File.binwrite(File.join(lang_path, "index.aff"), aff_content)
|
|
245
|
+
|
|
246
|
+
dic_url, dic_content = fetch_spelling_file(language, "dic")
|
|
247
|
+
verify_and_audit(url: dic_url,
|
|
248
|
+
relative_path: "#{language}/spelling/index.dic",
|
|
249
|
+
content: dic_content,
|
|
250
|
+
resource_id: resource_id)
|
|
251
|
+
File.binwrite(File.join(lang_path, "index.dic"), dic_content)
|
|
252
|
+
|
|
253
|
+
# Save metadata
|
|
254
|
+
metadata = build_metadata(language, "spelling", checksum(dic_content))
|
|
255
|
+
write_metadata(metadata_path_for(resource_id), metadata)
|
|
256
|
+
|
|
257
|
+
{
|
|
258
|
+
aff_path: File.join(lang_path, "index.aff"),
|
|
259
|
+
dic_path: File.join(lang_path, "index.dic"),
|
|
260
|
+
cached: false,
|
|
261
|
+
metadata: metadata
|
|
262
|
+
}
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Download grammar rules.
|
|
266
|
+
#
|
|
267
|
+
# @param language [String] Language code
|
|
268
|
+
# @return [Hash] Rules path and metadata
|
|
269
|
+
def download_grammar(language)
|
|
270
|
+
raise Kotoshu::ResourceNotCachedError.new(language, "grammar") if offline?
|
|
271
|
+
|
|
272
|
+
lang_path = resource_dir_for("#{language}:grammar")
|
|
273
|
+
resource_id = "#{language}:grammar"
|
|
274
|
+
|
|
275
|
+
# Download rules.yaml
|
|
276
|
+
rules_url = @source_registry.url_for(:grammar, lang: language)
|
|
277
|
+
rules_content = download_url(rules_url)
|
|
278
|
+
verify_and_audit(url: rules_url,
|
|
279
|
+
relative_path: "#{language}/grammar/rules.yaml",
|
|
280
|
+
content: rules_content,
|
|
281
|
+
resource_id: resource_id)
|
|
282
|
+
File.binwrite(File.join(lang_path, "rules.yaml"), rules_content)
|
|
283
|
+
|
|
284
|
+
# Save metadata
|
|
285
|
+
metadata = build_metadata(language, "grammar", checksum(rules_content))
|
|
286
|
+
write_metadata(metadata_path_for(resource_id), metadata)
|
|
287
|
+
|
|
288
|
+
{
|
|
289
|
+
rules_path: lang_path,
|
|
290
|
+
cached: false,
|
|
291
|
+
metadata: metadata
|
|
292
|
+
}
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# Download frequency data.
|
|
296
|
+
#
|
|
297
|
+
# @param language [String] Language code
|
|
298
|
+
# @return [Hash] Frequency data path and metadata
|
|
299
|
+
def download_frequency(language)
|
|
300
|
+
lang_path = resource_dir_for("#{language}:frequency")
|
|
301
|
+
resource_id = "#{language}:frequency"
|
|
302
|
+
|
|
303
|
+
# Download frequency.json from Kelly repository
|
|
304
|
+
freq_url = @source_registry.url_for(:frequency, lang: language)
|
|
305
|
+
freq_content = download_url(freq_url)
|
|
306
|
+
verify_and_audit(url: freq_url,
|
|
307
|
+
relative_path: "data/#{language}.json",
|
|
308
|
+
content: freq_content,
|
|
309
|
+
resource_id: resource_id)
|
|
310
|
+
|
|
311
|
+
# Validate JSON
|
|
312
|
+
JSON.parse(freq_content)
|
|
313
|
+
|
|
314
|
+
# Save frequency file
|
|
315
|
+
freq_file = File.join(lang_path, "frequency.json")
|
|
316
|
+
File.write(freq_file, freq_content)
|
|
317
|
+
|
|
318
|
+
# Save metadata (with custom URL for Kelly)
|
|
319
|
+
metadata = build_metadata(language, "kelly_frequency", checksum(freq_content))
|
|
320
|
+
metadata[:url] = freq_url # Override with specific Kelly URL
|
|
321
|
+
write_metadata(metadata_path_for(resource_id), metadata)
|
|
322
|
+
|
|
323
|
+
{
|
|
324
|
+
frequency_path: freq_file,
|
|
325
|
+
cached: false,
|
|
326
|
+
metadata: metadata
|
|
327
|
+
}
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
private
|
|
331
|
+
|
|
332
|
+
# LanguageCache serves from the kotoshu/dictionaries repo for spelling
|
|
333
|
+
# and grammar; frequency lives in a separate repo (kelly). Pin the
|
|
334
|
+
# manifest URL at the dictionaries repo since that's the primary
|
|
335
|
+
# surface users see. Kelly's manifest can be added when that repo
|
|
336
|
+
# ships one.
|
|
337
|
+
def manifest_url
|
|
338
|
+
@manifest_url || @source_registry.url_for(:dict_manifest)
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Download a specific resource (implements abstract method).
|
|
342
|
+
#
|
|
343
|
+
# @param resource_id [String] The resource identifier
|
|
344
|
+
# @param _dest_path [String] Destination directory (unused —
|
|
345
|
+
# spelling resources pick their own layout)
|
|
346
|
+
# @return [Object] Downloaded resource
|
|
347
|
+
def download_resource(resource_id, _dest_path)
|
|
348
|
+
language = extract_language(resource_id)
|
|
349
|
+
type = extract_type(resource_id)
|
|
350
|
+
return nil unless language && type
|
|
351
|
+
|
|
352
|
+
case type
|
|
353
|
+
when "spelling" then download_spelling(language)
|
|
354
|
+
when "grammar" then download_grammar(language)
|
|
355
|
+
when "frequency" then download_frequency(language)
|
|
356
|
+
else raise "Unknown resource type: #{type}"
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
# Load metadata for a resource.
|
|
361
|
+
#
|
|
362
|
+
# @param resource_id [String] The resource identifier
|
|
363
|
+
# @return [Hash, nil] Metadata or nil if not found
|
|
364
|
+
def load_metadata_for(resource_id)
|
|
365
|
+
metadata_path = metadata_path_for(resource_id)
|
|
366
|
+
return nil unless File.exist?(metadata_path)
|
|
367
|
+
|
|
368
|
+
read_metadata(metadata_path)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# Load cached resource by type.
|
|
372
|
+
#
|
|
373
|
+
# @param resource_id [String] The resource identifier
|
|
374
|
+
# @param type [String] The resource type
|
|
375
|
+
# @param metadata [Hash] The resource metadata
|
|
376
|
+
# @return [Hash, nil] Loaded resource or nil
|
|
377
|
+
def load_cached_resource_by_type(resource_id, type, metadata)
|
|
378
|
+
case type
|
|
379
|
+
when "spelling" then load_cached_spelling(resource_id, metadata)
|
|
380
|
+
when "grammar" then load_cached_grammar(resource_id, metadata)
|
|
381
|
+
when "frequency" then load_cached_frequency(resource_id, metadata)
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Load cached spelling resource.
|
|
386
|
+
#
|
|
387
|
+
# @param resource_id [String] The resource identifier
|
|
388
|
+
# @param metadata [Hash] The resource metadata
|
|
389
|
+
# @return [Hash] Spelling resource data
|
|
390
|
+
def load_cached_spelling(resource_id, metadata)
|
|
391
|
+
lang_path = resource_dir_for(resource_id)
|
|
392
|
+
{
|
|
393
|
+
aff_path: File.join(lang_path, "index.aff"),
|
|
394
|
+
dic_path: File.join(lang_path, "index.dic"),
|
|
395
|
+
cached: true,
|
|
396
|
+
metadata: metadata
|
|
397
|
+
}
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Load cached grammar resource.
|
|
401
|
+
#
|
|
402
|
+
# @param resource_id [String] The resource identifier
|
|
403
|
+
# @param metadata [Hash] The resource metadata
|
|
404
|
+
# @return [Hash] Grammar resource data
|
|
405
|
+
def load_cached_grammar(resource_id, metadata)
|
|
406
|
+
lang_path = resource_dir_for(resource_id)
|
|
407
|
+
{
|
|
408
|
+
rules_path: lang_path,
|
|
409
|
+
cached: true,
|
|
410
|
+
metadata: metadata
|
|
411
|
+
}
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# Load cached frequency resource.
|
|
415
|
+
#
|
|
416
|
+
# @param resource_id [String] The resource identifier
|
|
417
|
+
# @param metadata [Hash] The resource metadata
|
|
418
|
+
# @return [Hash, nil] Frequency resource data or nil
|
|
419
|
+
def load_cached_frequency(resource_id, metadata)
|
|
420
|
+
# CommonWordsLoader autoloaded via Kotoshu::Data
|
|
421
|
+
freq_file = File.join(resource_dir_for(resource_id), "frequency.json")
|
|
422
|
+
return nil unless File.exist?(freq_file)
|
|
423
|
+
|
|
424
|
+
data = Data::CommonWordsLoader.load_from_frequency_file(freq_file)
|
|
425
|
+
{
|
|
426
|
+
frequency_path: freq_file,
|
|
427
|
+
tiers: data[:tiers],
|
|
428
|
+
metadata: metadata
|
|
429
|
+
}
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
# Build metadata hash for a resource.
|
|
433
|
+
#
|
|
434
|
+
# @param language [String] Language code
|
|
435
|
+
# @param type [String] Resource type
|
|
436
|
+
# @param content_checksum [String] SHA256 checksum of content
|
|
437
|
+
# @return [Hash] Metadata hash
|
|
438
|
+
def build_metadata(language, type, content_checksum)
|
|
439
|
+
{
|
|
440
|
+
version: Time.now.utc.iso8601,
|
|
441
|
+
url: @url_base,
|
|
442
|
+
language: language,
|
|
443
|
+
type: type,
|
|
444
|
+
checksum: content_checksum,
|
|
445
|
+
cached_at: Time.now.utc.iso8601
|
|
446
|
+
}
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
public
|
|
450
|
+
|
|
451
|
+
# Load cached resource data (implements abstract method).
|
|
452
|
+
#
|
|
453
|
+
# Public — this is the cache-only reader BaseCache declares
|
|
454
|
+
# publicly; resolve paths use it to read without any download.
|
|
455
|
+
#
|
|
456
|
+
# @param resource_id [String] The resource identifier
|
|
457
|
+
# @return [Object, nil] Loaded resource or nil
|
|
458
|
+
def load_cached(resource_id)
|
|
459
|
+
parts = parse_resource_id(resource_id)
|
|
460
|
+
return nil unless parts
|
|
461
|
+
|
|
462
|
+
type = parts[1]
|
|
463
|
+
metadata = load_metadata_for(resource_id)
|
|
464
|
+
return nil unless metadata
|
|
465
|
+
|
|
466
|
+
load_cached_resource_by_type(resource_id, type, metadata)
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Get metadata file path for a resource.
|
|
470
|
+
#
|
|
471
|
+
# @param resource_id [String] The resource identifier
|
|
472
|
+
# @return [String] Metadata file path
|
|
473
|
+
def metadata_path_for(resource_id)
|
|
474
|
+
language = extract_language(resource_id)
|
|
475
|
+
type = extract_type(resource_id)
|
|
476
|
+
File.join(language_path(language, type), "metadata.json")
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
# Get resource directory path.
|
|
480
|
+
#
|
|
481
|
+
# @param resource_id [String] The resource identifier
|
|
482
|
+
# @return [String] Resource directory path
|
|
483
|
+
def resource_dir_for(resource_id)
|
|
484
|
+
language_path(extract_language(resource_id), extract_type(resource_id))
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
# Check if all resource files exist.
|
|
488
|
+
#
|
|
489
|
+
# @param resource_id [String] The resource identifier
|
|
490
|
+
# @return [Boolean] True if all files exist
|
|
491
|
+
def resource_files_exist?(resource_id)
|
|
492
|
+
type = extract_type(resource_id)
|
|
493
|
+
return false unless type
|
|
494
|
+
|
|
495
|
+
lang_path = resource_dir_for(resource_id)
|
|
496
|
+
|
|
497
|
+
case type
|
|
498
|
+
when "spelling"
|
|
499
|
+
File.exist?(File.join(lang_path, "index.aff")) &&
|
|
500
|
+
File.exist?(File.join(lang_path, "index.dic"))
|
|
501
|
+
when "grammar"
|
|
502
|
+
File.exist?(File.join(lang_path, "rules.yaml"))
|
|
503
|
+
when "frequency"
|
|
504
|
+
File.exist?(File.join(lang_path, "frequency.json"))
|
|
505
|
+
else
|
|
506
|
+
false
|
|
507
|
+
end
|
|
508
|
+
end
|
|
509
|
+
end
|
|
510
|
+
end
|
|
511
|
+
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Cache autoloaded via Kotoshu::Cache
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cache
|
|
7
|
+
# LRU (Least Recently Used) cache for fast lookups.
|
|
8
|
+
#
|
|
9
|
+
# This cache automatically evicts the least recently used entries
|
|
10
|
+
# when the maximum size is reached.
|
|
11
|
+
#
|
|
12
|
+
# @example Basic usage
|
|
13
|
+
# cache = LookupCache.new(max_size: 1000)
|
|
14
|
+
# cache.write("key", "value")
|
|
15
|
+
# cache.read("key") # => "value"
|
|
16
|
+
#
|
|
17
|
+
# @example Using fetch for lazy computation
|
|
18
|
+
# cache.fetch("expensive_key") { compute_expensive_value() }
|
|
19
|
+
class LookupCache
|
|
20
|
+
include Cache
|
|
21
|
+
|
|
22
|
+
# Default maximum cache size
|
|
23
|
+
DEFAULT_MAX_SIZE = 1000
|
|
24
|
+
|
|
25
|
+
# @return [Integer] Maximum number of entries
|
|
26
|
+
attr_reader :max_size
|
|
27
|
+
|
|
28
|
+
# Create a new LRU cache.
|
|
29
|
+
#
|
|
30
|
+
# @param max_size [Integer] Maximum number of entries (default: 1000)
|
|
31
|
+
def initialize(max_size: DEFAULT_MAX_SIZE)
|
|
32
|
+
@max_size = max_size
|
|
33
|
+
@data = {} # key => [value, access_order]
|
|
34
|
+
@access_order = 0
|
|
35
|
+
@stats = { hits: 0, misses: 0 }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Retrieve a value from cache, or compute it.
|
|
39
|
+
#
|
|
40
|
+
# @param key [Object] The cache key
|
|
41
|
+
# @param default [Object] Optional default value (if no block given)
|
|
42
|
+
# @yield Block to compute value on cache miss
|
|
43
|
+
# @return [Object] The cached or computed value
|
|
44
|
+
def fetch(key, default = nil)
|
|
45
|
+
if key?(key)
|
|
46
|
+
record_hit
|
|
47
|
+
@data[key][0] # Return value
|
|
48
|
+
else
|
|
49
|
+
record_miss
|
|
50
|
+
value = block_given? ? yield : default
|
|
51
|
+
write(key, value)
|
|
52
|
+
value
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Write a value to cache.
|
|
57
|
+
#
|
|
58
|
+
# @param key [Object] The cache key
|
|
59
|
+
# @param value [Object] The value to store
|
|
60
|
+
# @return [Object] The stored value
|
|
61
|
+
def write(key, value)
|
|
62
|
+
evict_if_needed
|
|
63
|
+
|
|
64
|
+
@access_order += 1
|
|
65
|
+
@data[key] = [value, @access_order]
|
|
66
|
+
|
|
67
|
+
value
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Read a value from cache.
|
|
71
|
+
#
|
|
72
|
+
# @param key [Object] The cache key
|
|
73
|
+
# @return [Object, nil] The cached value or nil
|
|
74
|
+
def read(key)
|
|
75
|
+
entry = @data[key]
|
|
76
|
+
|
|
77
|
+
if entry
|
|
78
|
+
record_hit
|
|
79
|
+
@access_order += 1
|
|
80
|
+
entry[1] = @access_order # Update access order
|
|
81
|
+
entry[0] # Return value
|
|
82
|
+
else
|
|
83
|
+
record_miss
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Delete a value from cache.
|
|
89
|
+
#
|
|
90
|
+
# @param key [Object] The cache key
|
|
91
|
+
# @return [Object, nil] The deleted value or nil
|
|
92
|
+
def delete(key)
|
|
93
|
+
entry = @data.delete(key)
|
|
94
|
+
entry&.first # Return value or nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Clear all entries from cache.
|
|
98
|
+
#
|
|
99
|
+
# @return [self] Self for chaining
|
|
100
|
+
def clear
|
|
101
|
+
@data.clear
|
|
102
|
+
@access_order = 0
|
|
103
|
+
self
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Check if key exists in cache.
|
|
107
|
+
#
|
|
108
|
+
# @param key [Object] The cache key
|
|
109
|
+
# @return [Boolean] True if key exists
|
|
110
|
+
def key?(key)
|
|
111
|
+
@data.key?(key)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Get number of entries in cache.
|
|
115
|
+
#
|
|
116
|
+
# @return [Integer] Number of entries
|
|
117
|
+
def size
|
|
118
|
+
@data.size
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Get cache statistics.
|
|
122
|
+
#
|
|
123
|
+
# @return [Hash] Statistics including :hits, :misses, :size, :hit_rate
|
|
124
|
+
def stats
|
|
125
|
+
total = @stats[:hits] + @stats[:misses]
|
|
126
|
+
hit_rate = total.positive? ? @stats[:hits].to_f / total : 0.0
|
|
127
|
+
|
|
128
|
+
{
|
|
129
|
+
hits: @stats[:hits],
|
|
130
|
+
misses: @stats[:misses],
|
|
131
|
+
size: size,
|
|
132
|
+
hit_rate: hit_rate.round(4)
|
|
133
|
+
}
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Reset statistics counters.
|
|
137
|
+
#
|
|
138
|
+
# @return [self] Self for chaining
|
|
139
|
+
def reset_stats
|
|
140
|
+
@stats = { hits: 0, misses: 0 }
|
|
141
|
+
self
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
private
|
|
145
|
+
|
|
146
|
+
# Record a cache hit.
|
|
147
|
+
def record_hit
|
|
148
|
+
@stats[:hits] += 1
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Record a cache miss.
|
|
152
|
+
def record_miss
|
|
153
|
+
@stats[:misses] += 1
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Evict least recently used entry if cache is full.
|
|
157
|
+
def evict_if_needed
|
|
158
|
+
return if @data.size < @max_size
|
|
159
|
+
|
|
160
|
+
# Find entry with lowest access order
|
|
161
|
+
lru_key = @data.min_by { |_, v| v[1] }&.first
|
|
162
|
+
@data.delete(lru_key) if lru_key
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|