kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# Base class for suggestion strategies.
|
|
7
|
+
#
|
|
8
|
+
# Subclasses must implement the {#generate} method.
|
|
9
|
+
#
|
|
10
|
+
# @example Implementing a custom strategy
|
|
11
|
+
# class MyStrategy < BaseStrategy
|
|
12
|
+
# def generate(context)
|
|
13
|
+
# # Return suggestions based on context.word
|
|
14
|
+
# SuggestionSet.from_words(%w[word1 word2], source: :my_strategy)
|
|
15
|
+
# end
|
|
16
|
+
# end
|
|
17
|
+
class BaseStrategy
|
|
18
|
+
# @return [Symbol] Strategy name
|
|
19
|
+
attr_reader :name
|
|
20
|
+
|
|
21
|
+
# @return [Hash] Strategy configuration
|
|
22
|
+
attr_reader :config
|
|
23
|
+
|
|
24
|
+
# Create a new base strategy.
|
|
25
|
+
#
|
|
26
|
+
# @param name [String, Symbol] Strategy name
|
|
27
|
+
# @param config [Hash] Configuration options
|
|
28
|
+
# @option config [Integer] max_results Maximum results to return
|
|
29
|
+
# @option config [Boolean] enabled Whether strategy is enabled
|
|
30
|
+
def initialize(name: :base, **config)
|
|
31
|
+
@name = name.to_sym
|
|
32
|
+
@config = config
|
|
33
|
+
@enabled = config.fetch(:enabled, true)
|
|
34
|
+
@max_results = config.fetch(:max_results, 10)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Generate suggestions for a word.
|
|
38
|
+
#
|
|
39
|
+
# @abstract Subclasses must implement this method.
|
|
40
|
+
# @param context [Context] The suggestion context
|
|
41
|
+
# @return [SuggestionSet] Generated suggestions
|
|
42
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
43
|
+
def generate(context)
|
|
44
|
+
raise NotImplementedError, "#{self.class} must implement #generate"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Check if this strategy is enabled.
|
|
48
|
+
#
|
|
49
|
+
# @return [Boolean] True if enabled
|
|
50
|
+
def enabled?
|
|
51
|
+
@enabled
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Get the max results configuration.
|
|
55
|
+
#
|
|
56
|
+
# @param default [Integer] Default value if not set
|
|
57
|
+
# @return [Integer] Max results
|
|
58
|
+
def max_results(default = 10)
|
|
59
|
+
@max_results || default
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Get a configuration value.
|
|
63
|
+
#
|
|
64
|
+
# @param key [Symbol] The config key
|
|
65
|
+
# @param default [Object] Default value if not set
|
|
66
|
+
# @return [Object] The config value
|
|
67
|
+
def get_config(key, default = nil)
|
|
68
|
+
@config.fetch(key, default)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Check if a config value is present.
|
|
72
|
+
#
|
|
73
|
+
# @param key [Symbol] The config key
|
|
74
|
+
# @return [Boolean] True if config has the key
|
|
75
|
+
def has_config?(key)
|
|
76
|
+
@config.key?(key)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Get the priority for this strategy.
|
|
80
|
+
#
|
|
81
|
+
# @return [Integer] Priority (lower = higher priority)
|
|
82
|
+
def priority
|
|
83
|
+
@config.fetch(:priority, 100)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Check if this strategy should handle the context.
|
|
87
|
+
#
|
|
88
|
+
# Default implementation checks if the word is not in the dictionary.
|
|
89
|
+
# Subclasses can override for more specific logic.
|
|
90
|
+
#
|
|
91
|
+
# @param context [Context] The suggestion context
|
|
92
|
+
# @return [Boolean] True if the strategy should handle this context
|
|
93
|
+
def handles?(context)
|
|
94
|
+
return false unless enabled?
|
|
95
|
+
|
|
96
|
+
!dictionary_lookup(context, context.word)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Whether the confidence cascade may skip this strategy when
|
|
100
|
+
# the traditional strategies are already confident about the
|
|
101
|
+
# word (see {Suggestions::SemanticCascade}). Only expensive,
|
|
102
|
+
# optional rerankers (e.g. {SemanticStrategy}) opt in.
|
|
103
|
+
#
|
|
104
|
+
# @return [Boolean]
|
|
105
|
+
def skip_when_confident?
|
|
106
|
+
false
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Create a suggestion from a word.
|
|
110
|
+
#
|
|
111
|
+
# @param word [String] The suggested word
|
|
112
|
+
# @param distance [Integer] Edit distance
|
|
113
|
+
# @param confidence [Float] Confidence score
|
|
114
|
+
# @param metadata [Hash] Additional metadata for ranking
|
|
115
|
+
# @return [Suggestion] New suggestion
|
|
116
|
+
def create_suggestion(word, distance: 0, confidence: 1.0, **metadata)
|
|
117
|
+
Suggestion.new(
|
|
118
|
+
word: word,
|
|
119
|
+
distance: distance,
|
|
120
|
+
confidence: confidence,
|
|
121
|
+
source: @name,
|
|
122
|
+
**metadata
|
|
123
|
+
)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Create a suggestion set from words.
|
|
127
|
+
#
|
|
128
|
+
# @param words [Array<String>] Array of words
|
|
129
|
+
# @param distances [Hash] Optional word => distance mapping
|
|
130
|
+
# @param original_word [String] The original misspelled word (for ranking)
|
|
131
|
+
# @return [SuggestionSet] New suggestion set
|
|
132
|
+
def create_suggestion_set(words, distances: {}, original_word: nil)
|
|
133
|
+
suggestions = words.map do |word|
|
|
134
|
+
# Try case-sensitive first, then case-insensitive for distance lookup
|
|
135
|
+
distance = if distances.key?(word)
|
|
136
|
+
distances[word]
|
|
137
|
+
else
|
|
138
|
+
distances.fetch(word.downcase, 1)
|
|
139
|
+
end
|
|
140
|
+
confidence = calculate_confidence(distance)
|
|
141
|
+
|
|
142
|
+
# Calculate n-gram similarity (like Hunspell) for better ranking
|
|
143
|
+
ngram_score = if original_word
|
|
144
|
+
calculate_ngram_similarity(original_word, word)
|
|
145
|
+
else
|
|
146
|
+
0
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
metadata = {
|
|
150
|
+
original_length: original_word&.length || word.length,
|
|
151
|
+
ngram_score: ngram_score
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
create_suggestion(word, distance: distance, confidence: confidence, **metadata)
|
|
155
|
+
end
|
|
156
|
+
SuggestionSet.new(suggestions, max_size: max_results)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Calculate typo correction similarity between two words.
|
|
160
|
+
#
|
|
161
|
+
# This is a custom similarity metric designed specifically for spelling
|
|
162
|
+
# correction, combining:
|
|
163
|
+
# - Character overlap (how many characters are shared)
|
|
164
|
+
# - Prefix weight (common prefix is very important for typos)
|
|
165
|
+
# - Suffix weight (common ending is also important)
|
|
166
|
+
# - Length penalty (very different lengths are less similar)
|
|
167
|
+
#
|
|
168
|
+
# Returns a value from 0.0 (no similarity) to 1.0 (identical).
|
|
169
|
+
#
|
|
170
|
+
# @param word1 [String] First word
|
|
171
|
+
# @param word2 [String] Second word
|
|
172
|
+
# @return [Float] Typo correction similarity (0.0 to 1.0)
|
|
173
|
+
def calculate_ngram_similarity(word1, word2)
|
|
174
|
+
return 0 if word1.nil? || word2.nil? || word1.empty? || word2.empty?
|
|
175
|
+
|
|
176
|
+
w1 = word1.downcase
|
|
177
|
+
w2 = word2.downcase
|
|
178
|
+
|
|
179
|
+
# Identical strings have maximum similarity
|
|
180
|
+
return 1.0 if w1 == w2
|
|
181
|
+
|
|
182
|
+
len1 = w1.length
|
|
183
|
+
len2 = w2.length
|
|
184
|
+
max_len = [len1, len2].max
|
|
185
|
+
|
|
186
|
+
# Calculate common prefix length (up to 4 characters)
|
|
187
|
+
prefix_len = 0
|
|
188
|
+
(0...[len1, len2, 4].min).each do |i|
|
|
189
|
+
break if w1[i] != w2[i]
|
|
190
|
+
|
|
191
|
+
prefix_len += 1
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Calculate common suffix length
|
|
195
|
+
suffix_len = 0
|
|
196
|
+
(1..[len1, len2, 4].min).each do |i|
|
|
197
|
+
break if w1[-i] != w2[-i]
|
|
198
|
+
|
|
199
|
+
suffix_len += 1
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Calculate character overlap (how many characters from w1 are in w2)
|
|
203
|
+
w2_chars = w2.chars
|
|
204
|
+
overlap = w1.chars.count { |c| w2_chars.include?(c) }
|
|
205
|
+
|
|
206
|
+
# Calculate similarity score
|
|
207
|
+
# 1. Base score from character overlap
|
|
208
|
+
similarity = overlap.to_f / max_len
|
|
209
|
+
|
|
210
|
+
# 2. Prefix bonus (common start is very important for typos)
|
|
211
|
+
prefix_bonus = prefix_len * 0.15
|
|
212
|
+
|
|
213
|
+
# 3. Suffix bonus (common ending is also important)
|
|
214
|
+
suffix_bonus = suffix_len * 0.05
|
|
215
|
+
|
|
216
|
+
# 4. Length penalty (very different lengths are less similar)
|
|
217
|
+
length_diff = (len1 - len2).abs
|
|
218
|
+
length_penalty = length_diff * 0.1
|
|
219
|
+
|
|
220
|
+
# Combine all factors
|
|
221
|
+
similarity = similarity + prefix_bonus + suffix_bonus - length_penalty
|
|
222
|
+
|
|
223
|
+
# Cap at 1.0, floor at 0.0
|
|
224
|
+
[[similarity, 1.0].min, 0.0].max
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Generate n-grams for a word.
|
|
228
|
+
#
|
|
229
|
+
# @param word [String] The word
|
|
230
|
+
# @param n [Integer] N-gram size
|
|
231
|
+
# @return [Set<String>] Set of n-grams
|
|
232
|
+
def generate_ngrams(word, n)
|
|
233
|
+
ngrams = Set.new
|
|
234
|
+
(word.length - n + 1).times do |i|
|
|
235
|
+
ngrams.add(word[i, n])
|
|
236
|
+
end
|
|
237
|
+
ngrams
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Convert strategy to string.
|
|
241
|
+
#
|
|
242
|
+
# @return [String] String representation
|
|
243
|
+
def to_s
|
|
244
|
+
"#{self.class.name}(name: #{@name}, enabled: #{enabled?})"
|
|
245
|
+
end
|
|
246
|
+
alias inspect to_s
|
|
247
|
+
|
|
248
|
+
private
|
|
249
|
+
|
|
250
|
+
# Look up a word in the dictionary.
|
|
251
|
+
#
|
|
252
|
+
# Type-driven dispatch (no `respond_to?`): IndexedDictionary is
|
|
253
|
+
# checked first because it is the most common case in the
|
|
254
|
+
# hot path; Hash and Array are ad-hoc dictionaries for tests
|
|
255
|
+
# and one-off scripts; anything else must implement the
|
|
256
|
+
# documented `lookup(word)` interface. Passing an object that
|
|
257
|
+
# doesn't quack like a dictionary surfaces as a NoMethodError.
|
|
258
|
+
#
|
|
259
|
+
# @param context [Context] The suggestion context
|
|
260
|
+
# @param word [String] The word to look up
|
|
261
|
+
# @return [Boolean] True if word exists
|
|
262
|
+
def dictionary_lookup(context, word)
|
|
263
|
+
dictionary = context.dictionary
|
|
264
|
+
|
|
265
|
+
case dictionary
|
|
266
|
+
when Kotoshu::Core::IndexedDictionary
|
|
267
|
+
dictionary.has_word?(word)
|
|
268
|
+
when Hash
|
|
269
|
+
dictionary.key?(word)
|
|
270
|
+
when Array
|
|
271
|
+
dictionary.include?(word)
|
|
272
|
+
else
|
|
273
|
+
dictionary.lookup(word)
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Get all words from the dictionary.
|
|
278
|
+
#
|
|
279
|
+
# Type-driven dispatch (no `respond_to?`). Mirrors
|
|
280
|
+
# {#dictionary_lookup}'s ordering.
|
|
281
|
+
#
|
|
282
|
+
# @param context [Context] The suggestion context
|
|
283
|
+
# @return [Array<String>] All words
|
|
284
|
+
def dictionary_words(context)
|
|
285
|
+
dictionary = context.dictionary
|
|
286
|
+
|
|
287
|
+
case dictionary
|
|
288
|
+
when Kotoshu::Core::IndexedDictionary
|
|
289
|
+
dictionary.words
|
|
290
|
+
when Hash
|
|
291
|
+
dictionary.keys
|
|
292
|
+
when Array
|
|
293
|
+
dictionary
|
|
294
|
+
else
|
|
295
|
+
dictionary.words
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# Calculate confidence from distance.
|
|
300
|
+
#
|
|
301
|
+
# Higher distance = lower confidence.
|
|
302
|
+
#
|
|
303
|
+
# @param distance [Integer] Edit distance
|
|
304
|
+
# @return [Float] Confidence score (0.0 to 1.0)
|
|
305
|
+
def calculate_confidence(distance)
|
|
306
|
+
return 1.0 if distance.zero?
|
|
307
|
+
|
|
308
|
+
# Simple decay: confidence = 1 / (1 + distance)
|
|
309
|
+
# Can be overridden by subclasses for more sophisticated calculations
|
|
310
|
+
1.0 / (1.0 + distance)
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
end
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# Composite strategy that chains multiple suggestion strategies.
|
|
7
|
+
# Implements the Composite Pattern for extensible suggestion generation.
|
|
8
|
+
#
|
|
9
|
+
# This is MORE OOP than Spylls which has a procedural suggestion pipeline.
|
|
10
|
+
# Here, strategies are proper objects that can be added/removed/reordered.
|
|
11
|
+
#
|
|
12
|
+
# @example Using composite strategy
|
|
13
|
+
# pipeline = CompositeStrategy.new(name: :pipeline)
|
|
14
|
+
# pipeline.add(EditDistanceStrategy.new)
|
|
15
|
+
# pipeline.add(PhoneticStrategy.new)
|
|
16
|
+
# pipeline.add(NgramStrategy.new)
|
|
17
|
+
# suggestions = pipeline.generate(context)
|
|
18
|
+
class CompositeStrategy < BaseStrategy
|
|
19
|
+
attr_reader :strategies
|
|
20
|
+
|
|
21
|
+
# @param name [String, Symbol] Name of the composite
|
|
22
|
+
# @param strategies [Array<BaseStrategy>] Initial strategies
|
|
23
|
+
# @param config [Hash] Configuration options
|
|
24
|
+
def initialize(name:, strategies: [], **config)
|
|
25
|
+
@strategies = strategies
|
|
26
|
+
super(name: name, **config)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Add a strategy to the pipeline.
|
|
30
|
+
#
|
|
31
|
+
# @param strategy [BaseStrategy] The strategy to add
|
|
32
|
+
# @return [CompositeStrategy] Self for chaining
|
|
33
|
+
def add(strategy)
|
|
34
|
+
@strategies << strategy
|
|
35
|
+
self
|
|
36
|
+
end
|
|
37
|
+
alias << add
|
|
38
|
+
|
|
39
|
+
# Remove a strategy from the pipeline.
|
|
40
|
+
#
|
|
41
|
+
# @param strategy [BaseStrategy] The strategy to remove
|
|
42
|
+
# @return [CompositeStrategy] Self for chaining
|
|
43
|
+
def remove(strategy)
|
|
44
|
+
@strategies.delete(strategy)
|
|
45
|
+
self
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Clear all strategies.
|
|
49
|
+
#
|
|
50
|
+
# @return [CompositeStrategy] Self for chaining
|
|
51
|
+
def clear
|
|
52
|
+
@strategies.clear
|
|
53
|
+
self
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Get strategies that can handle the given context.
|
|
57
|
+
#
|
|
58
|
+
# @param context [Context] The suggestion context
|
|
59
|
+
# @return [Array<BaseStrategy>] Applicable strategies
|
|
60
|
+
def applicable_strategies(context)
|
|
61
|
+
@strategies.select { |s| s.handles?(context) }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Generate suggestions by delegating to all applicable strategies.
|
|
65
|
+
#
|
|
66
|
+
# Candidates are collected from every strategy first, then handed
|
|
67
|
+
# to a single SuggestionSet so dedup and ranking run once over the
|
|
68
|
+
# full pool (not as a side effect of each merge). TODO 56 T5.1.
|
|
69
|
+
#
|
|
70
|
+
# Strategies that declare themselves skippable by the confidence
|
|
71
|
+
# cascade ({BaseStrategy#skip_when_confident?} — the semantic
|
|
72
|
+
# rerank) run after the traditional ones and are skipped
|
|
73
|
+
# entirely when the traditional pool already meets the
|
|
74
|
+
# configured threshold ({SemanticCascade}). At the default
|
|
75
|
+
# threshold (1.0, always rerank) the cascade is disarmed and
|
|
76
|
+
# every strategy runs exactly as before.
|
|
77
|
+
#
|
|
78
|
+
# @param context [Context] The suggestion context
|
|
79
|
+
# @return [SuggestionSet] Combined suggestions from all strategies
|
|
80
|
+
def generate(context)
|
|
81
|
+
cascade = semantic_cascade
|
|
82
|
+
strategies = applicable_strategies(context)
|
|
83
|
+
deferred = strategies.select(&:skip_when_confident?)
|
|
84
|
+
|
|
85
|
+
if deferred.empty? || !cascade.armed?
|
|
86
|
+
return generate_all(strategies, context)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
traditional = strategies.reject(&:skip_when_confident?)
|
|
90
|
+
pool = SuggestionSet.new(
|
|
91
|
+
traditional.flat_map { |strategy| strategy.generate(context).suggestions },
|
|
92
|
+
max_size: context.max_results
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
if cascade.skip?(pool.suggestions)
|
|
96
|
+
record_cascade_skip(context, pool.first, cascade)
|
|
97
|
+
return pool
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
pool.concat(deferred.flat_map { |strategy| strategy.generate(context).suggestions })
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Check if any strategy can handle the context.
|
|
104
|
+
#
|
|
105
|
+
# @param context [Context] The suggestion context
|
|
106
|
+
# @return [Boolean] True if any strategy handles the context
|
|
107
|
+
def handles?(context)
|
|
108
|
+
applicable_strategies(context).any?
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Get the number of strategies.
|
|
112
|
+
#
|
|
113
|
+
# @return [Integer] Number of strategies
|
|
114
|
+
def size
|
|
115
|
+
@strategies.size
|
|
116
|
+
end
|
|
117
|
+
alias count size
|
|
118
|
+
|
|
119
|
+
# Check if the composite has any strategies.
|
|
120
|
+
#
|
|
121
|
+
# @return [Boolean] True if there are strategies
|
|
122
|
+
def any?
|
|
123
|
+
@strategies.any?
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Iterate over strategies.
|
|
127
|
+
#
|
|
128
|
+
# @yield [strategy] Each strategy
|
|
129
|
+
# @return [Enumerator] Enumerator if no block given
|
|
130
|
+
def each_strategy(&)
|
|
131
|
+
return enum_for(:each_strategy) unless block_given?
|
|
132
|
+
|
|
133
|
+
@strategies.each(&)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Sort strategies by priority.
|
|
137
|
+
#
|
|
138
|
+
# @return [CompositeStrategy] Self for chaining
|
|
139
|
+
def sort_by_priority!
|
|
140
|
+
@strategies.sort_by!(&:priority)
|
|
141
|
+
self
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Convert to string.
|
|
145
|
+
#
|
|
146
|
+
# @return [String] String representation
|
|
147
|
+
def to_s
|
|
148
|
+
"#{self.class.name}(name: #{@name}, strategies: #{@strategies.map(&:name).join(', ')})"
|
|
149
|
+
end
|
|
150
|
+
alias inspect to_s
|
|
151
|
+
|
|
152
|
+
# Create a composite strategy with default algorithms.
|
|
153
|
+
#
|
|
154
|
+
# @param config [Hash] Configuration
|
|
155
|
+
# @return [CompositeStrategy] New composite with default strategies
|
|
156
|
+
def self.with_defaults(**config)
|
|
157
|
+
new(name: :default, **config)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
# Run every strategy and merge into a single set — the
|
|
163
|
+
# pre-cascade shape, byte-identical to the original pipeline.
|
|
164
|
+
def generate_all(strategies, context)
|
|
165
|
+
candidates = strategies.flat_map do |strategy|
|
|
166
|
+
strategy.generate(context).suggestions
|
|
167
|
+
end
|
|
168
|
+
SuggestionSet.new(candidates, max_size: context.max_results)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# The cascade decision object. An explicit
|
|
172
|
+
# +semantic_cascade_threshold+ in this composite's config wins;
|
|
173
|
+
# otherwise the process Configuration applies (SCHEMA default
|
|
174
|
+
# 1.0, KOTOSHU_SEMANTIC_CASCADE_THRESHOLD ENV).
|
|
175
|
+
def semantic_cascade
|
|
176
|
+
threshold = get_config(:semantic_cascade_threshold)
|
|
177
|
+
if threshold.nil?
|
|
178
|
+
SemanticCascade.from_configuration(Kotoshu.configuration)
|
|
179
|
+
else
|
|
180
|
+
SemanticCascade.new(threshold: threshold)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Count and (under debug/verbose) log a cascade skip.
|
|
185
|
+
def record_cascade_skip(context, top, cascade)
|
|
186
|
+
Kotoshu::Metrics.record_semantic_cascade_skip
|
|
187
|
+
return unless Kotoshu::Debug.enabled?
|
|
188
|
+
|
|
189
|
+
Kotoshu::Debug.logger&.verbose(
|
|
190
|
+
"semantic cascade: skipped rerank for \"#{context.word}\" " \
|
|
191
|
+
"(top confidence #{top.confidence.round(3)} >= #{cascade.threshold})"
|
|
192
|
+
)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|