kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# String similarity metrics for spell checking.
|
|
5
|
+
#
|
|
6
|
+
# Ported from Spylls (Python) string_metrics.py
|
|
7
|
+
#
|
|
8
|
+
# These metrics are used for:
|
|
9
|
+
# - Computing word similarity
|
|
10
|
+
# - Ranking suggestions
|
|
11
|
+
# - N-gram based scoring
|
|
12
|
+
module StringMetrics
|
|
13
|
+
# Number of occurrences of the exactly same characters in exactly same position.
|
|
14
|
+
#
|
|
15
|
+
# @param s1 [String] First string
|
|
16
|
+
# @param s2 [String] Second string
|
|
17
|
+
# @return [Integer] Count of matching characters at same positions
|
|
18
|
+
#
|
|
19
|
+
# @example
|
|
20
|
+
# Kotoshu::StringMetrics.commoncharacters("hello", "hallo") # => 4 ('h', 'l', 'l', 'o' match)
|
|
21
|
+
def self.commoncharacters(s1, s2)
|
|
22
|
+
return 0 if s1.nil? || s2.nil?
|
|
23
|
+
|
|
24
|
+
# Zip strings and count matching character pairs
|
|
25
|
+
[s1.length, s2.length].min.times.count do |i|
|
|
26
|
+
s1[i] == s2[i]
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Size of the common start of two strings.
|
|
31
|
+
#
|
|
32
|
+
# @param s1 [String] First string
|
|
33
|
+
# @param s2 [String] Second string
|
|
34
|
+
# @return [Integer] Length of common prefix
|
|
35
|
+
#
|
|
36
|
+
# @example
|
|
37
|
+
# Kotoshu::StringMetrics.leftcommonsubstring("foo", "bar") # => 0
|
|
38
|
+
# Kotoshu::StringMetrics.leftcommonsubstring("built", "build") # => 4
|
|
39
|
+
# Kotoshu::StringMetrics.leftcommonsubstring("cat", "cats") # => 3
|
|
40
|
+
def self.leftcommonsubstring(s1, s2)
|
|
41
|
+
return 0 if s1.nil? || s2.nil?
|
|
42
|
+
|
|
43
|
+
# Find first position where characters differ
|
|
44
|
+
s1.chars.zip(s2.chars).each_with_index do |(c1, c2), i|
|
|
45
|
+
return i if c1 != c2
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# All characters matched up to shorter string length
|
|
49
|
+
[s1.length, s2.length].min
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Calculate n-gram similarity between two strings.
|
|
53
|
+
#
|
|
54
|
+
# Calculates how many n-grams of s1 are contained in s2 (the more the number,
|
|
55
|
+
# the more words are similar).
|
|
56
|
+
#
|
|
57
|
+
# @param max_ngram_size [Integer] Maximum n-gram size to check
|
|
58
|
+
# @param s1 [String] String to compare
|
|
59
|
+
# @param s2 [String] String to compare
|
|
60
|
+
# @param weighted [Boolean] Subtract from result for ngrams NOT contained
|
|
61
|
+
# @param any_mismatch [Boolean] Add penalty for any string length difference
|
|
62
|
+
# @param longer_worse [Boolean] Add penalty when second string is longer
|
|
63
|
+
# @return [Integer] N-gram similarity score (higher is more similar)
|
|
64
|
+
#
|
|
65
|
+
# @example
|
|
66
|
+
# Kotoshu::StringMetrics.ngram(4, "hello", "help") # => 6
|
|
67
|
+
# Kotoshu::StringMetrics.ngram(4, "teachings", "teaching") # => higher score
|
|
68
|
+
def self.ngram(max_ngram_size, s1, s2, weighted: false, any_mismatch: false, longer_worse: false)
|
|
69
|
+
l2 = s2.length
|
|
70
|
+
return 0 if l2.zero?
|
|
71
|
+
|
|
72
|
+
l1 = s1.length
|
|
73
|
+
nscore = 0
|
|
74
|
+
|
|
75
|
+
# For all sizes of ngram up to desired...
|
|
76
|
+
(1..max_ngram_size).each do |ngram_size|
|
|
77
|
+
ns = 0
|
|
78
|
+
|
|
79
|
+
# Check every position in the first string
|
|
80
|
+
(0..(l1 - ngram_size)).each do |pos|
|
|
81
|
+
ngram = s1[pos, ngram_size]
|
|
82
|
+
|
|
83
|
+
# If the ngram is present in ANY place in second string, increase score
|
|
84
|
+
if s2.include?(ngram)
|
|
85
|
+
ns += 1
|
|
86
|
+
elsif weighted
|
|
87
|
+
# For "weighted" ngrams, decrease score if ngram is not found
|
|
88
|
+
ns -= 1
|
|
89
|
+
# Decrease once more if it was the beginning or end of first string
|
|
90
|
+
ns -= 1 if pos.zero? || pos + ngram_size == l1
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
nscore += ns
|
|
95
|
+
|
|
96
|
+
# There is no need to check for 4-gram if there were only one 3-gram
|
|
97
|
+
break if ns < 2 && !weighted
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Calculate penalty based on settings
|
|
101
|
+
penalty = if longer_worse
|
|
102
|
+
# Add penalty when second string is longer
|
|
103
|
+
(l2 - l1) - 2
|
|
104
|
+
elsif any_mismatch
|
|
105
|
+
# Add penalty for any string length difference
|
|
106
|
+
(l2 - l1).abs - 2
|
|
107
|
+
else
|
|
108
|
+
0
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Apply penalty if positive
|
|
112
|
+
penalty > 0 ? nscore - penalty : nscore
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Calculate LCS (Longest Common Subsequence) length.
|
|
116
|
+
#
|
|
117
|
+
# Classic dynamic programming algorithm. This is different from
|
|
118
|
+
# longest common substring - subsequence doesn't require contiguity.
|
|
119
|
+
#
|
|
120
|
+
# @param s1 [String] First string
|
|
121
|
+
# @param s2 [String] Second string
|
|
122
|
+
# @return [Integer] Length of longest common subsequence
|
|
123
|
+
#
|
|
124
|
+
# @example
|
|
125
|
+
# Kotoshu::StringMetrics.lcslen("AGGTAB", "GXTXAYB") # => 4 ("GTAB")
|
|
126
|
+
def self.lcslen(s1, s2)
|
|
127
|
+
return 0 if s1.nil? || s2.nil? || s1.empty? || s2.empty?
|
|
128
|
+
|
|
129
|
+
m = s1.length
|
|
130
|
+
n = s2.length
|
|
131
|
+
|
|
132
|
+
# Create DP table
|
|
133
|
+
# Using a 2D array for clarity, though we could optimize space
|
|
134
|
+
c = Array.new(m + 1) { Array.new(n + 1, 0) }
|
|
135
|
+
|
|
136
|
+
(0...m).each do |i|
|
|
137
|
+
(0...n).each do |j|
|
|
138
|
+
c[i + 1][j + 1] = if s1[i] == s2[j]
|
|
139
|
+
# Characters match - extend diagonal
|
|
140
|
+
c[i][j] + 1
|
|
141
|
+
elsif c[i][j + 1] >= c[i + 1][j]
|
|
142
|
+
# Take max from top or left
|
|
143
|
+
c[i][j + 1]
|
|
144
|
+
else
|
|
145
|
+
c[i + 1][j]
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
c[m][n]
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Context object passed to suggestion strategies.
|
|
6
|
+
# Encapsulates the state and parameters for suggestion generation.
|
|
7
|
+
class Context
|
|
8
|
+
attr_reader :word, :dictionary, :max_results, :options
|
|
9
|
+
|
|
10
|
+
def initialize(word:, dictionary:, max_results: 10, **options)
|
|
11
|
+
@word = word
|
|
12
|
+
@dictionary = dictionary
|
|
13
|
+
@max_results = max_results
|
|
14
|
+
@options = options
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Get an option value.
|
|
18
|
+
#
|
|
19
|
+
# @param key [Symbol] The option key
|
|
20
|
+
# @param default [Object] Default value if not found
|
|
21
|
+
# @return [Object] The option value
|
|
22
|
+
def option(key, default = nil)
|
|
23
|
+
@options.fetch(key, default)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Check if an option is present.
|
|
27
|
+
#
|
|
28
|
+
# @param key [Symbol] The option key
|
|
29
|
+
# @return [Boolean] True if option exists
|
|
30
|
+
def has_option?(key)
|
|
31
|
+
@options.key?(key)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Inspect the context.
|
|
35
|
+
#
|
|
36
|
+
# @return [String] Inspection string
|
|
37
|
+
def inspect
|
|
38
|
+
"Context(word: '#{@word}', max_results: #{@max_results})"
|
|
39
|
+
end
|
|
40
|
+
alias to_s inspect
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Loads and caches Kelly Project word-frequency tiers per language.
|
|
6
|
+
#
|
|
7
|
+
# Extracted from EditDistanceStrategy (TODO 56 T5.1 step 3, Phase A)
|
|
8
|
+
# so that strategy construction no longer performs disk IO and
|
|
9
|
+
# network access. The strategy holds a reference to a provider
|
|
10
|
+
# instance; the provider encapsulates the tiered fallback
|
|
11
|
+
# (FrequencyCache → frozen embedded tiers → local YAML → empty).
|
|
12
|
+
# The frozen tier layer (plan 119) carries the same en tables the
|
|
13
|
+
# Rust engine embeds, so a cache-cold machine ranks exactly like
|
|
14
|
+
# the frozen conformance vectors instead of the differently
|
|
15
|
+
# curated YAML.
|
|
16
|
+
#
|
|
17
|
+
# The provider memoizes per-language tiers, so repeated lookups
|
|
18
|
+
# for the same language are free after the first call.
|
|
19
|
+
class FrequencyProvider
|
|
20
|
+
EMPTY_TIERS = {
|
|
21
|
+
top_50: Set.new,
|
|
22
|
+
top_200: Set.new,
|
|
23
|
+
top_1000: Set.new
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
# @param frequency_cache [Cache::FrequencyCache, nil] Injectable
|
|
27
|
+
# cache instance (used by tests); defaults to a fresh
|
|
28
|
+
# FrequencyCache per load.
|
|
29
|
+
def initialize(frequency_cache: nil)
|
|
30
|
+
@frequency_cache = frequency_cache
|
|
31
|
+
@tiers_by_language = {}
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Return the frequency tiers for +language_code+.
|
|
35
|
+
#
|
|
36
|
+
# @param language_code [String] ISO 639-1 language code
|
|
37
|
+
# @return [Hash{Symbol => Set}] Hash with :top_50, :top_200, :top_1000
|
|
38
|
+
def tiers_for(language_code)
|
|
39
|
+
@tiers_by_language[language_code] ||= load(language_code)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def load(language_code)
|
|
45
|
+
cache_result = try_load_from_frequency_cache(language_code)
|
|
46
|
+
return cache_result[:tiers] if cache_result && cache_result[:tiers] && cache_result[:tiers][:top_1000].any?
|
|
47
|
+
|
|
48
|
+
frozen = FrozenTiers.tiers_for(language_code)
|
|
49
|
+
return frozen if frozen
|
|
50
|
+
|
|
51
|
+
yaml_data = Data::CommonWordsLoader.load(language_code)
|
|
52
|
+
return yaml_data[:tiers] if yaml_data[:tiers][:top_1000].any?
|
|
53
|
+
|
|
54
|
+
EMPTY_TIERS
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def try_load_from_frequency_cache(language_code)
|
|
58
|
+
cache = @frequency_cache || Cache::FrequencyCache.new
|
|
59
|
+
# Read cache-only: the suggestion hot path must never trigger a
|
|
60
|
+
# download (downloads happen only through explicit setup /
|
|
61
|
+
# `kotoshu cache download`). TTL expiry is a refresh signal,
|
|
62
|
+
# NOT a reason to switch datasets: expired-but-present bytes
|
|
63
|
+
# are still the checksummed Kelly tiers the conformance
|
|
64
|
+
# vectors freeze, while the YAML fallback below is a different,
|
|
65
|
+
# more coarsely curated set whose bonus differences re-scale
|
|
66
|
+
# every suggestion confidence (plan 117). So: any readable
|
|
67
|
+
# cached data wins; the YAML only serves machines with no
|
|
68
|
+
# frequency data at all.
|
|
69
|
+
return nil unless cache.cached_data?(language_code)
|
|
70
|
+
|
|
71
|
+
begin
|
|
72
|
+
cache.load_cached(language_code)
|
|
73
|
+
rescue StandardError => e
|
|
74
|
+
warn "Warning: Failed to load frequency cache for #{language_code}: #{e.message}" if $VERBOSE
|
|
75
|
+
nil
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Generator for spelling suggestions.
|
|
6
|
+
#
|
|
7
|
+
# This class orchestrates multiple suggestion algorithms to generate
|
|
8
|
+
# comprehensive spelling suggestions.
|
|
9
|
+
#
|
|
10
|
+
# @example Using default algorithms
|
|
11
|
+
# generator = Generator.new(dictionary)
|
|
12
|
+
# suggestions = generator.generate("helo")
|
|
13
|
+
#
|
|
14
|
+
# @example Using custom algorithms
|
|
15
|
+
# custom_strategy = MyStrategy.new
|
|
16
|
+
# generator = Generator.new(dictionary, algorithms: [custom_strategy])
|
|
17
|
+
class Generator
|
|
18
|
+
# Default suggestion algorithms.
|
|
19
|
+
DEFAULT_ALGORITHMS = [
|
|
20
|
+
Strategies::EditDistanceStrategy,
|
|
21
|
+
Strategies::PhoneticStrategy,
|
|
22
|
+
Strategies::KeyboardProximityStrategy,
|
|
23
|
+
Strategies::NgramStrategy
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
26
|
+
# @return [Object] The dictionary (any dictionary backend)
|
|
27
|
+
attr_reader :dictionary
|
|
28
|
+
|
|
29
|
+
# @return [Strategies::CompositeStrategy] The composite strategy
|
|
30
|
+
attr_reader :strategy
|
|
31
|
+
|
|
32
|
+
# Create a new suggestion generator.
|
|
33
|
+
#
|
|
34
|
+
# @param dictionary [Object] The dictionary instance
|
|
35
|
+
# @param algorithms [Array<Class, Strategies::BaseStrategy>, nil] Algorithm classes or instances
|
|
36
|
+
# @param max_suggestions [Integer] Maximum suggestions to return
|
|
37
|
+
# @param config [Hash] Configuration options
|
|
38
|
+
def initialize(dictionary, algorithms: nil, max_suggestions: 10, **config)
|
|
39
|
+
@dictionary = dictionary
|
|
40
|
+
@max_suggestions = max_suggestions
|
|
41
|
+
# Use default algorithms if none provided
|
|
42
|
+
algorithms_to_use = algorithms || DEFAULT_ALGORITHMS
|
|
43
|
+
@strategy = build_strategy(algorithms_to_use, config)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Generate suggestions for a word.
|
|
47
|
+
#
|
|
48
|
+
# @param word [String] The misspelled word
|
|
49
|
+
# @param max_suggestions [Integer] Maximum suggestions (optional)
|
|
50
|
+
# @return [SuggestionSet] Generated suggestions
|
|
51
|
+
#
|
|
52
|
+
# @example
|
|
53
|
+
# generator.generate("helo")
|
|
54
|
+
# # => #<Kotoshu::Suggestions::SuggestionSet ...>
|
|
55
|
+
def generate(word, max_suggestions: nil)
|
|
56
|
+
return SuggestionSet.empty if word.nil? || word.empty?
|
|
57
|
+
|
|
58
|
+
context = Context.new(
|
|
59
|
+
word: word,
|
|
60
|
+
dictionary: @dictionary,
|
|
61
|
+
max_results: max_suggestions || @max_suggestions
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@strategy.generate(context)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Alias for {#generate}, for API consistency (same parameters and
|
|
68
|
+
# return value).
|
|
69
|
+
#
|
|
70
|
+
# @example
|
|
71
|
+
# generator.suggest("helo")
|
|
72
|
+
# # => #<Kotoshu::Suggestions::SuggestionSet ...>
|
|
73
|
+
#
|
|
74
|
+
# @see #generate
|
|
75
|
+
alias suggest generate
|
|
76
|
+
|
|
77
|
+
# Check if a word is correct.
|
|
78
|
+
#
|
|
79
|
+
# @param word [String] The word to check
|
|
80
|
+
# @return [Boolean] True if the word is in the dictionary
|
|
81
|
+
#
|
|
82
|
+
# @example
|
|
83
|
+
# generator.correct?("hello") # => true
|
|
84
|
+
# generator.correct?("helo") # => false
|
|
85
|
+
def correct?(word)
|
|
86
|
+
return false if word.nil? || word.empty?
|
|
87
|
+
|
|
88
|
+
dictionary_lookup(word)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Check if a word is incorrect.
|
|
92
|
+
#
|
|
93
|
+
# @param word [String] The word to check
|
|
94
|
+
# @return [Boolean] True if the word is not in the dictionary
|
|
95
|
+
def incorrect?(word)
|
|
96
|
+
!correct?(word)
|
|
97
|
+
end
|
|
98
|
+
alias misspelled? incorrect?
|
|
99
|
+
|
|
100
|
+
# Get the default algorithms.
|
|
101
|
+
#
|
|
102
|
+
# @return [Array<Class>] Default algorithm classes
|
|
103
|
+
#
|
|
104
|
+
# @example
|
|
105
|
+
# Generator.default_algorithms
|
|
106
|
+
def self.default_algorithms
|
|
107
|
+
DEFAULT_ALGORITHMS.dup
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Set the default algorithms.
|
|
111
|
+
#
|
|
112
|
+
# @param algorithms [Array<Class>] Algorithm classes
|
|
113
|
+
#
|
|
114
|
+
# @example
|
|
115
|
+
# Generator.default_algorithms = [MyCustomStrategy]
|
|
116
|
+
class << self
|
|
117
|
+
attr_writer :default_algorithms
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
# Build the composite strategy from algorithm classes.
|
|
123
|
+
#
|
|
124
|
+
# @param algorithms [Array<Class, Strategies::BaseStrategy>] Algorithm classes or instances
|
|
125
|
+
# @param config [Hash] Configuration options
|
|
126
|
+
# @return [Strategies::CompositeStrategy] The composite strategy
|
|
127
|
+
def build_strategy(algorithms, config)
|
|
128
|
+
composite = Strategies::CompositeStrategy.new(name: :default, **config)
|
|
129
|
+
|
|
130
|
+
algorithms.each do |alg|
|
|
131
|
+
strategy = if alg.is_a?(Strategies::BaseStrategy)
|
|
132
|
+
alg
|
|
133
|
+
elsif alg.is_a?(Class) && alg < Strategies::BaseStrategy
|
|
134
|
+
alg.new(**config)
|
|
135
|
+
else
|
|
136
|
+
raise ArgumentError, "Invalid algorithm: #{alg.inspect}"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
composite.add(strategy)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
composite
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Look up a word in the dictionary.
|
|
146
|
+
#
|
|
147
|
+
# Dispatch is type-driven (no `respond_to?`): Hash and Array are
|
|
148
|
+
# supported as ad-hoc dictionaries for tests and one-off scripts;
|
|
149
|
+
# everything else is treated as a real dictionary backend that
|
|
150
|
+
# implements the documented `lookup(word)` interface. Passing an
|
|
151
|
+
# object that doesn't quack like a dictionary surfaces as a
|
|
152
|
+
# NoMethodError on `lookup` — that's intentional; the caller
|
|
153
|
+
# misconfigured the generator.
|
|
154
|
+
#
|
|
155
|
+
# @param word [String] The word
|
|
156
|
+
# @return [Boolean] True if found
|
|
157
|
+
def dictionary_lookup(word)
|
|
158
|
+
case @dictionary
|
|
159
|
+
when Hash
|
|
160
|
+
@dictionary.key?(word)
|
|
161
|
+
when Array
|
|
162
|
+
@dictionary.include?(word)
|
|
163
|
+
else
|
|
164
|
+
@dictionary.lookup(word)
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Pipeline for composable suggestion strategies.
|
|
6
|
+
#
|
|
7
|
+
# Allows chaining multiple suggestion strategies that execute in sequence,
|
|
8
|
+
# with optional early termination when a stage produces no results.
|
|
9
|
+
#
|
|
10
|
+
# @example Creating a pipeline
|
|
11
|
+
# pipeline = Pipeline.new do |p|
|
|
12
|
+
# p.add :sym_spell
|
|
13
|
+
# p.add :phonetic
|
|
14
|
+
# p.add :ngram
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# @example Executing a pipeline
|
|
18
|
+
# result = pipeline.execute(context, strategies)
|
|
19
|
+
class Pipeline
|
|
20
|
+
# @return [Array<Symbol>] Ordered stage names
|
|
21
|
+
attr_reader :stages
|
|
22
|
+
|
|
23
|
+
# Create a new pipeline.
|
|
24
|
+
#
|
|
25
|
+
# @yield [pipeline] Optional block to add stages
|
|
26
|
+
# @return [Pipeline] New pipeline
|
|
27
|
+
#
|
|
28
|
+
# @example With block
|
|
29
|
+
# pipeline = Pipeline.new do |p|
|
|
30
|
+
# p.add :sym_spell
|
|
31
|
+
# p.add :phonetic
|
|
32
|
+
# end
|
|
33
|
+
def initialize
|
|
34
|
+
@stages = []
|
|
35
|
+
yield self if block_given?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Add a stage to the pipeline.
|
|
39
|
+
#
|
|
40
|
+
# @param stage_name [Symbol] Name of the stage
|
|
41
|
+
# @return [Pipeline] Self for chaining
|
|
42
|
+
#
|
|
43
|
+
# @example
|
|
44
|
+
# pipeline.add(:sym_spell)
|
|
45
|
+
def add(stage_name)
|
|
46
|
+
@stages << stage_name
|
|
47
|
+
self
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Remove a stage from the pipeline.
|
|
51
|
+
#
|
|
52
|
+
# @param stage_name [Symbol] Name of the stage to remove
|
|
53
|
+
# @return [Pipeline] Self for chaining
|
|
54
|
+
#
|
|
55
|
+
# @example
|
|
56
|
+
# pipeline.remove(:phonetic)
|
|
57
|
+
def remove(stage_name)
|
|
58
|
+
@stages.delete(stage_name)
|
|
59
|
+
self
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Execute strategies through the pipeline.
|
|
63
|
+
#
|
|
64
|
+
# Strategies are executed in sequence. If a strategy returns
|
|
65
|
+
# an empty SuggestionSet, subsequent strategies are still executed
|
|
66
|
+
# unless early_termination is enabled.
|
|
67
|
+
#
|
|
68
|
+
# @param context [Context] The suggestion context
|
|
69
|
+
# @param strategies [Hash] Hash of stage_name => strategy_instance
|
|
70
|
+
# @param early_termination [Boolean] Whether to stop on empty result
|
|
71
|
+
# @return [SuggestionSet] Combined results from all stages
|
|
72
|
+
#
|
|
73
|
+
# @example
|
|
74
|
+
# strategies = { sym_spell: sym_spell_strategy, phonetic: phonetic_strategy }
|
|
75
|
+
# result = pipeline.execute(context, strategies)
|
|
76
|
+
def execute(context, strategies = nil, early_termination: false)
|
|
77
|
+
combined = SuggestionSet.empty
|
|
78
|
+
|
|
79
|
+
@stages.each do |stage_name|
|
|
80
|
+
strategy = if strategies.is_a?(Hash)
|
|
81
|
+
strategies[stage_name]
|
|
82
|
+
else
|
|
83
|
+
strategies
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
next unless strategy
|
|
87
|
+
|
|
88
|
+
result = strategy.generate(context)
|
|
89
|
+
|
|
90
|
+
# Combine results
|
|
91
|
+
combined = combine_results(combined, result)
|
|
92
|
+
|
|
93
|
+
# Early termination on empty result
|
|
94
|
+
break if early_termination && result.empty?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
combined
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Check if pipeline has a stage.
|
|
101
|
+
#
|
|
102
|
+
# @param stage_name [Symbol] Stage name
|
|
103
|
+
# @return [Boolean] True if stage exists
|
|
104
|
+
def has_stage?(stage_name)
|
|
105
|
+
@stages.include?(stage_name)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Clear all stages.
|
|
109
|
+
#
|
|
110
|
+
# @return [Pipeline] Self for chaining
|
|
111
|
+
def clear
|
|
112
|
+
@stages.clear
|
|
113
|
+
self
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Clone the pipeline.
|
|
117
|
+
#
|
|
118
|
+
# @return [Pipeline] New pipeline with same stages
|
|
119
|
+
def clone
|
|
120
|
+
self.class.new.tap { |p| @stages.each { |s| p.add(s) } }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
private
|
|
124
|
+
|
|
125
|
+
# Combine two suggestion sets.
|
|
126
|
+
#
|
|
127
|
+
# @param combined [SuggestionSet] Current combined results
|
|
128
|
+
# @param new_result [SuggestionSet] New results to add
|
|
129
|
+
# @return [SuggestionSet] Combined suggestion set
|
|
130
|
+
def combine_results(combined, new_result)
|
|
131
|
+
combined.concat(new_result.suggestions).unique
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Confidence cascade for the semantic rerank (TODO.impl/70).
|
|
6
|
+
#
|
|
7
|
+
# Formalizes the hybrid dictionary-first architecture: the
|
|
8
|
+
# expensive, ONNX-backed semantic rerank only runs when the
|
|
9
|
+
# traditional composite strategies are not already confident
|
|
10
|
+
# about the word (dictionary verdict first, neural rerank only
|
|
11
|
+
# for uncertain candidates).
|
|
12
|
+
#
|
|
13
|
+
# @example From the process configuration
|
|
14
|
+
# cascade = SemanticCascade.from_configuration(Kotoshu.configuration)
|
|
15
|
+
# cascade.skip?(suggestion_set.suggestions) # => true / false
|
|
16
|
+
#
|
|
17
|
+
# @example With an explicit threshold
|
|
18
|
+
# SemanticCascade.new(threshold: 0.9)
|
|
19
|
+
class SemanticCascade
|
|
20
|
+
# Sentinel threshold meaning "always rerank" (the default).
|
|
21
|
+
# NOT a plain >= comparison: composite confidence legitimately
|
|
22
|
+
# reaches exactly 1.0 (distance-0 matches, the default
|
|
23
|
+
# confidence of a fresh Suggestion), so `>= 1.0` would change
|
|
24
|
+
# the default behavior — threshold 1.0 must mean never-skip.
|
|
25
|
+
ALWAYS_RERANK = 1.0
|
|
26
|
+
|
|
27
|
+
# @return [Float] The configured skip threshold
|
|
28
|
+
attr_reader :threshold
|
|
29
|
+
|
|
30
|
+
# Create a cascade decision object.
|
|
31
|
+
#
|
|
32
|
+
# @param threshold [Float, nil] Skip threshold in [0.0, 1.0];
|
|
33
|
+
# nil falls back to the always-rerank default
|
|
34
|
+
def initialize(threshold: ALWAYS_RERANK)
|
|
35
|
+
@threshold = threshold.nil? ? ALWAYS_RERANK : threshold.to_f
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Build from a Configuration, so the SCHEMA default and the
|
|
39
|
+
# KOTOSHU_SEMANTIC_CASCADE_THRESHOLD ENV var apply automatically.
|
|
40
|
+
#
|
|
41
|
+
# @param configuration [Configuration] The configuration to read
|
|
42
|
+
# @return [SemanticCascade]
|
|
43
|
+
def self.from_configuration(configuration)
|
|
44
|
+
new(threshold: configuration.semantic_cascade_threshold)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Whether the cascade can ever skip. Only a threshold strictly
|
|
48
|
+
# below 1.0 arms it; 0.0 means never rerank.
|
|
49
|
+
#
|
|
50
|
+
# @return [Boolean]
|
|
51
|
+
def armed?
|
|
52
|
+
threshold < ALWAYS_RERANK
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Whether the semantic rerank should be skipped for this pool of
|
|
56
|
+
# traditional candidates. Skips only when the pool already has a
|
|
57
|
+
# top candidate at or above the threshold; an empty pool never
|
|
58
|
+
# skips (the dictionary path found nothing — the rerank may
|
|
59
|
+
# still help).
|
|
60
|
+
#
|
|
61
|
+
# @param suggestions [Array<Suggestion>] Sorted traditional candidates
|
|
62
|
+
# @return [Boolean]
|
|
63
|
+
def skip?(suggestions)
|
|
64
|
+
return false unless armed?
|
|
65
|
+
|
|
66
|
+
top = suggestions.first
|
|
67
|
+
return false unless top
|
|
68
|
+
|
|
69
|
+
top.confidence >= threshold
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|