kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Suggestions
|
|
7
|
+
module Strategies
|
|
8
|
+
# Edit distance suggestion strategy with enhanced ranking.
|
|
9
|
+
# Generates suggestions by finding words with small edit distance,
|
|
10
|
+
# ranked by word frequency, keyboard proximity, and common typo patterns.
|
|
11
|
+
#
|
|
12
|
+
# Multi-language support:
|
|
13
|
+
# - Automatically selects keyboard layout based on language_code
|
|
14
|
+
# - Loads frequency data from YAML files (Phase 1) or GitHub (Phase 2)
|
|
15
|
+
# - Supports language-specific typo patterns
|
|
16
|
+
#
|
|
17
|
+
# This is MORE OOP than Spylls which uses standalone functions
|
|
18
|
+
# for edit distance operations.
|
|
19
|
+
#
|
|
20
|
+
# Follows Open-Closed Principle: Extend by adding YAML files,
|
|
21
|
+
# NOT by modifying this class.
|
|
22
|
+
class EditDistanceStrategy < BaseStrategy
|
|
23
|
+
attr_reader :language_code, :keyboard_layout
|
|
24
|
+
|
|
25
|
+
# @param name [String, Symbol] Name of the strategy
|
|
26
|
+
# @param config [Hash] Configuration options
|
|
27
|
+
# @option config [String] :language_code Language code for keyboard layout (default: 'en')
|
|
28
|
+
# @option config [Keyboard::Layout] :keyboard_layout Custom keyboard layout (optional)
|
|
29
|
+
# @option config [Hash] :frequency_tiers Custom frequency tiers (optional)
|
|
30
|
+
# @option config [Integer] :max_distance Maximum edit distance (default: 2)
|
|
31
|
+
# @option config [Integer] :max_results Maximum results to return (default: 10)
|
|
32
|
+
def initialize(name: :edit_distance, language_code: 'en', keyboard_layout: nil,
|
|
33
|
+
frequency_tiers: nil, frequency_provider: nil, **config)
|
|
34
|
+
super(name: name, **config)
|
|
35
|
+
@language_code = language_code
|
|
36
|
+
|
|
37
|
+
# Use OOP registry for keyboard layout lookup
|
|
38
|
+
@keyboard_layout = resolve_keyboard_layout(keyboard_layout)
|
|
39
|
+
|
|
40
|
+
# Frequency data comes from a provider (extracted in TODO 56
|
|
41
|
+
# T5.1 step 3 Phase A) so the strategy constructor no longer
|
|
42
|
+
# performs disk IO or network access. Callers can inject a
|
|
43
|
+
# custom provider for testing.
|
|
44
|
+
@frequency_provider = frequency_provider || FrequencyProvider.new
|
|
45
|
+
|
|
46
|
+
# Use custom frequency tiers if provided, otherwise resolve
|
|
47
|
+
# lazily through the provider on first use.
|
|
48
|
+
if frequency_tiers
|
|
49
|
+
@frequency_tiers = frequency_tiers
|
|
50
|
+
@common_words = Set.new
|
|
51
|
+
else
|
|
52
|
+
@frequency_tiers = nil
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Public method to get current keyboard being used
|
|
57
|
+
#
|
|
58
|
+
# @return [Keyboard::Layout] The keyboard layout instance
|
|
59
|
+
def keyboard
|
|
60
|
+
@keyboard_layout
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Public method to get keyboard name
|
|
64
|
+
#
|
|
65
|
+
# @return [String] Keyboard layout name
|
|
66
|
+
def keyboard_name
|
|
67
|
+
@keyboard_layout.name
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Check if a substitution is a keyboard-adjacent typo
|
|
71
|
+
#
|
|
72
|
+
# @param char1 [String] First character
|
|
73
|
+
# @param char2 [String] Second character
|
|
74
|
+
# @return [Boolean] True if keys are adjacent
|
|
75
|
+
def adjacent_key_typo?(char1, char2)
|
|
76
|
+
@keyboard_layout.adjacent_keys(char1).include?(char2)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Get adjacent keys for a given key
|
|
80
|
+
#
|
|
81
|
+
# @param key [String] The key to find adjacent keys for
|
|
82
|
+
# @return [Array<String>] List of adjacent key characters
|
|
83
|
+
def adjacent_keys(key)
|
|
84
|
+
@keyboard_layout.adjacent_keys(key)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Get frequency bonus for a word
|
|
88
|
+
#
|
|
89
|
+
# @param word [String] The word to check
|
|
90
|
+
# @return [Integer] Frequency bonus (0-200)
|
|
91
|
+
def frequency_bonus(word)
|
|
92
|
+
tiers = frequency_tiers
|
|
93
|
+
return 0 unless tiers
|
|
94
|
+
|
|
95
|
+
word_downcase = word.downcase
|
|
96
|
+
|
|
97
|
+
# Top 50: 200 bonus
|
|
98
|
+
return 200 if tiers[:top_50]&.include?(word_downcase)
|
|
99
|
+
|
|
100
|
+
# Top 200: 100 bonus
|
|
101
|
+
return 100 if tiers[:top_200]&.include?(word_downcase)
|
|
102
|
+
|
|
103
|
+
# Top 1000: 50 bonus
|
|
104
|
+
return 50 if tiers[:top_1000]&.include?(word_downcase)
|
|
105
|
+
|
|
106
|
+
# Not in common words: no bonus
|
|
107
|
+
0
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Frequency tiers used for ranking. Lazy-loads from the
|
|
111
|
+
# provider on first access when not set at construction.
|
|
112
|
+
def frequency_tiers
|
|
113
|
+
@frequency_tiers ||= @frequency_provider.tiers_for(@language_code)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Generate suggestions based on enhanced edit distance scoring.
|
|
117
|
+
#
|
|
118
|
+
# Scoring factors:
|
|
119
|
+
# - Edit distance (primary factor)
|
|
120
|
+
# - Word frequency (common words rank higher)
|
|
121
|
+
# - Keyboard proximity (adjacent key typos rank higher)
|
|
122
|
+
# - Common typo patterns (missing double letters, etc.)
|
|
123
|
+
#
|
|
124
|
+
# @param context [Context] The suggestion context
|
|
125
|
+
# @return [SuggestionSet] Suggestions within max_distance
|
|
126
|
+
def generate(context)
|
|
127
|
+
word = context.word
|
|
128
|
+
max_dist = get_config(:max_distance, 2)
|
|
129
|
+
min_confidence = get_config(:min_confidence, 0.75) # Higher threshold for quality
|
|
130
|
+
min_similarity = get_config(:min_jaro_similarity, 0.70) # Minimum Jaro-Winkler similarity (0.0-1.0)
|
|
131
|
+
min_results = get_config(:min_results, 3) # Always return at least 3 suggestions if available
|
|
132
|
+
|
|
133
|
+
# When the dictionary is case-insensitive, normalize case before
|
|
134
|
+
# edit-distance comparison — otherwise "HELO" can never match
|
|
135
|
+
# "Hello" within distance 2 (case differences alone cost 4).
|
|
136
|
+
# The original dictionary casing is preserved on the returned
|
|
137
|
+
# suggestion (we only normalize for the comparison).
|
|
138
|
+
case_insensitive = dictionary_case_insensitive?(context)
|
|
139
|
+
compare_word = case_insensitive ? word.downcase : word
|
|
140
|
+
|
|
141
|
+
# Calculate enhanced scores for all candidates
|
|
142
|
+
candidates = []
|
|
143
|
+
# Length filter: edit distance cannot be less than the length
|
|
144
|
+
# difference. Skip dictionary words whose length differs from
|
|
145
|
+
# the input by more than max_dist — they can't possibly match.
|
|
146
|
+
# This is the single biggest performance win for this strategy:
|
|
147
|
+
# without it, we pay the full O(m*n) DP cost on every word in
|
|
148
|
+
# the dictionary. When the dictionary is an IndexedDictionary,
|
|
149
|
+
# the lookup is O(buckets_in_range) instead of O(n) scan.
|
|
150
|
+
target_length = compare_word.length
|
|
151
|
+
length_min = target_length - max_dist
|
|
152
|
+
length_max = target_length + max_dist
|
|
153
|
+
|
|
154
|
+
words_to_check = dictionary_words_in_length_range(
|
|
155
|
+
context, min: length_min, max: length_max
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
words_to_check.each do |dict_word|
|
|
159
|
+
next if dict_word == word
|
|
160
|
+
|
|
161
|
+
compare_dict = case_insensitive ? dict_word.downcase : dict_word
|
|
162
|
+
# distance_with_threshold bails out early when the row
|
|
163
|
+
# minimum exceeds max_dist — avoids computing the full
|
|
164
|
+
# DP for clearly-different pairs of similar-length words.
|
|
165
|
+
dist = edit_distance_with_threshold(compare_word, compare_dict, max_dist)
|
|
166
|
+
next unless dist && dist > 0
|
|
167
|
+
|
|
168
|
+
# Calculate enhanced score (lower is better)
|
|
169
|
+
score = calculate_enhanced_score(compare_word, compare_dict, dist)
|
|
170
|
+
candidates << [dict_word, dist, score]
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Single-edit sweep: enumerate the distance-1 edits of the
|
|
174
|
+
# misspelling itself (adjacent transposition, substitution,
|
|
175
|
+
# insertion, deletion) and keep the ones the dictionary
|
|
176
|
+
# accepts. The headword loop above can only ever surface raw
|
|
177
|
+
# dictionary stems — without this sweep, valid dictionary
|
|
178
|
+
# FORMS (affixed like "definately" -> "definitely", or
|
|
179
|
+
# capitalization variants like "Teh" -> "The") can never be
|
|
180
|
+
# suggested, no matter how close they are. Transposition is
|
|
181
|
+
# enumerated as a single operation (restricted Damerau), so
|
|
182
|
+
# a swapped pair costs 1, not 2.
|
|
183
|
+
edit_sweep_candidates(context, word, case_insensitive: case_insensitive,
|
|
184
|
+
max_dist: max_dist) do |candidate, dist|
|
|
185
|
+
compare_candidate = case_insensitive ? candidate.downcase : candidate
|
|
186
|
+
score = calculate_enhanced_score(compare_word, compare_candidate, dist)
|
|
187
|
+
candidates << [candidate, dist, score]
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Sort by enhanced score (lower is better)
|
|
191
|
+
sorted_candidates = candidates.sort_by { |_, _, score| score }
|
|
192
|
+
|
|
193
|
+
# Drop case variants of the same word, keeping the
|
|
194
|
+
# best-scoring form (first after the sort). The headword
|
|
195
|
+
# loop and the edit sweep can both propose a word — e.g.
|
|
196
|
+
# "the" (stem) and "The" (lookup-valid INITCAP form of the
|
|
197
|
+
# same stem) — and the final SuggestionSet would deduplicate
|
|
198
|
+
# them anyway; deduplicating here keeps the strategy output
|
|
199
|
+
# free of case duplicates and the score range used for
|
|
200
|
+
# confidence normalization honest.
|
|
201
|
+
seen_words = Set.new
|
|
202
|
+
sorted_candidates = sorted_candidates.select { |dict_word, _, _| seen_words.add?(dict_word.downcase) }
|
|
203
|
+
|
|
204
|
+
# Calculate confidence scores with threshold filtering
|
|
205
|
+
if sorted_candidates.empty?
|
|
206
|
+
return SuggestionSet.empty
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
max_score = sorted_candidates.map { |_, _, s| s.to_f }.max
|
|
210
|
+
min_score = sorted_candidates.map { |_, _, s| s.to_f }.min
|
|
211
|
+
score_range = (max_score - min_score).abs
|
|
212
|
+
|
|
213
|
+
# Create suggestions with confidence-based filtering
|
|
214
|
+
suggestions = []
|
|
215
|
+
sorted_candidates.each do |dict_word, dist, score|
|
|
216
|
+
# Normalize score to confidence (0.0 to 1.0)
|
|
217
|
+
# Lower score = higher confidence
|
|
218
|
+
if score_range > 0
|
|
219
|
+
normalized = (score.to_f - min_score) / score_range # 0 to 1
|
|
220
|
+
confidence = 1.0 - normalized # Invert: lower score = higher confidence
|
|
221
|
+
else
|
|
222
|
+
confidence = 1.0
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# Calculate Jaro-Winkler similarity for additional filtering.
|
|
226
|
+
# Use the same case normalization as the edit distance so the
|
|
227
|
+
# similarity score is consistent with the distance threshold.
|
|
228
|
+
compare_dict = case_insensitive ? dict_word.downcase : dict_word
|
|
229
|
+
jaro_similarity = calculate_ngram_similarity(compare_word, compare_dict)
|
|
230
|
+
|
|
231
|
+
# Skip low-confidence or low-similarity suggestions (unless we need more for min_results)
|
|
232
|
+
if (confidence < min_confidence || jaro_similarity < min_similarity) && (suggestions.size >= min_results)
|
|
233
|
+
next
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
suggestions << Suggestion.new(
|
|
237
|
+
word: dict_word,
|
|
238
|
+
distance: dist,
|
|
239
|
+
confidence: confidence,
|
|
240
|
+
source: @name,
|
|
241
|
+
original_length: word.length,
|
|
242
|
+
ngram_score: jaro_similarity, # Now stores Jaro-Winkler similarity (0.0-1.0)
|
|
243
|
+
enhanced_score: score
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
# Stop when we have enough high-quality suggestions
|
|
247
|
+
break if suggestions.size >= max_results
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
SuggestionSet.new(suggestions, max_size: max_results)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# Check if this strategy should handle the context.
|
|
254
|
+
#
|
|
255
|
+
# @param context [Context] The suggestion context
|
|
256
|
+
# @return [Boolean] True if the word needs correction
|
|
257
|
+
def handles?(context)
|
|
258
|
+
return false unless enabled?
|
|
259
|
+
|
|
260
|
+
# Only handle if the word is not in the dictionary
|
|
261
|
+
!dictionary_lookup(context, context.word)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
private
|
|
265
|
+
|
|
266
|
+
# Get all words from the dictionary.
|
|
267
|
+
#
|
|
268
|
+
# @param context [Context] The suggestion context
|
|
269
|
+
# @return [Array<String>] All dictionary words
|
|
270
|
+
def dictionary_words(context)
|
|
271
|
+
dictionary = context.dictionary
|
|
272
|
+
|
|
273
|
+
if indexed_dictionary?(dictionary)
|
|
274
|
+
dictionary.all_words
|
|
275
|
+
else
|
|
276
|
+
case dictionary
|
|
277
|
+
when Kotoshu::Dictionary::Base then dictionary.words
|
|
278
|
+
when Hash then dictionary.keys
|
|
279
|
+
when Set then dictionary.to_a
|
|
280
|
+
when Array then dictionary.dup
|
|
281
|
+
else Array(dictionary).flat_map(&:to_a)
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Get dictionary words whose length is in [min, max].
|
|
287
|
+
#
|
|
288
|
+
# Uses the dictionary's find_by_length_range when the backend
|
|
289
|
+
# is one that supports it (Dictionary::Base subclasses,
|
|
290
|
+
# IndexedDictionary) — both expose an index-backed
|
|
291
|
+
# implementation. Falls back to manual filtering for ad-hoc
|
|
292
|
+
# dictionary shapes (Hash, Array, Set) used in tests.
|
|
293
|
+
#
|
|
294
|
+
# @param context [Context] The suggestion context
|
|
295
|
+
# @param min [Integer] Minimum length (inclusive)
|
|
296
|
+
# @param max [Integer] Maximum length (inclusive)
|
|
297
|
+
# @return [Array<String>] Words in the length range
|
|
298
|
+
def dictionary_words_in_length_range(context, min:, max:)
|
|
299
|
+
dictionary = context.dictionary
|
|
300
|
+
|
|
301
|
+
case dictionary
|
|
302
|
+
when Kotoshu::Dictionary::Base, ::Kotoshu::Core::IndexedDictionary
|
|
303
|
+
dictionary.find_by_length_range(min_length: min, max_length: max)
|
|
304
|
+
else
|
|
305
|
+
dictionary_words(context).select { |w| w.length >= min && w.length <= max }
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def indexed_dictionary?(dictionary)
|
|
310
|
+
defined?(::Kotoshu::Core::IndexedDictionary) && dictionary.is_a?(::Kotoshu::Core::IndexedDictionary)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Check if a word exists in the dictionary.
|
|
314
|
+
#
|
|
315
|
+
# @param context [Context] The suggestion context
|
|
316
|
+
# @param word [String] The word to check
|
|
317
|
+
# @return [Boolean] True if word exists
|
|
318
|
+
def dictionary_lookup(context, word)
|
|
319
|
+
dictionary = context.dictionary
|
|
320
|
+
|
|
321
|
+
if defined?(::Kotoshu::Core::IndexedDictionary) && dictionary.is_a?(::Kotoshu::Core::IndexedDictionary)
|
|
322
|
+
dictionary.has_word?(word)
|
|
323
|
+
else
|
|
324
|
+
case dictionary
|
|
325
|
+
when Kotoshu::Dictionary::Base then dictionary.lookup(word)
|
|
326
|
+
when Hash then dictionary.key?(word)
|
|
327
|
+
when Set, Array then dictionary.include?(word)
|
|
328
|
+
else false
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# Determine whether the context's dictionary treats lookups
|
|
334
|
+
# case-insensitively. When true, the edit-distance and similarity
|
|
335
|
+
# calculations lowercase both sides so case differences don't
|
|
336
|
+
# masquerade as edit-distance penalties.
|
|
337
|
+
#
|
|
338
|
+
# @param context [Context] The suggestion context
|
|
339
|
+
# @return [Boolean]
|
|
340
|
+
def dictionary_case_insensitive?(context)
|
|
341
|
+
dictionary = context.dictionary
|
|
342
|
+
# PlainText and similar backends expose `case_sensitive` directly.
|
|
343
|
+
# Hunspell/IndexedDictionary route case through the casing layer,
|
|
344
|
+
# so we leave them case-sensitive here (their casing machinery
|
|
345
|
+
# already produces the right variants upstream).
|
|
346
|
+
return false unless dictionary.is_a?(::Kotoshu::Dictionary::PlainText)
|
|
347
|
+
|
|
348
|
+
!dictionary.case_sensitive
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Calculate Damerau-Levenshtein edit distance between two strings.
|
|
352
|
+
# Delegates to Algorithms::EditDistance.
|
|
353
|
+
#
|
|
354
|
+
# @param str1 [String] First string
|
|
355
|
+
# @param str2 [String] Second string
|
|
356
|
+
# @return [Integer] Edit distance
|
|
357
|
+
def edit_distance(str1, str2)
|
|
358
|
+
Algorithms::EditDistance.distance(str1, str2)
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Optimized edit distance with early termination.
|
|
362
|
+
# Returns early if distance exceeds threshold.
|
|
363
|
+
#
|
|
364
|
+
# @param str1 [String] First string
|
|
365
|
+
# @param str2 [String] Second string
|
|
366
|
+
# @param threshold [Integer] Maximum distance to calculate
|
|
367
|
+
# @return [Integer, nil] Distance or nil if exceeds threshold
|
|
368
|
+
def edit_distance_with_threshold(str1, str2, threshold)
|
|
369
|
+
Algorithms::EditDistance.distance_with_threshold(str1, str2, threshold)
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Enumerate the single-edit candidates of the misspelling and
|
|
373
|
+
# yield the ones the dictionary accepts, with their true
|
|
374
|
+
# (Damerau-Levenshtein) distance. Edit order is fixed and
|
|
375
|
+
# mirrors typo frequency: transposition, substitution,
|
|
376
|
+
# insertion, deletion — ties in the candidate sort resolve in
|
|
377
|
+
# this order, and the native engine reproduces it exactly.
|
|
378
|
+
#
|
|
379
|
+
# Substitutions and insertions use the dictionary's TRY
|
|
380
|
+
# string (Hunspell TRY directive); with no TRY available
|
|
381
|
+
# they are skipped, exactly as in Hunspell. Deletions and
|
|
382
|
+
# transpositions need no alphabet and always run.
|
|
383
|
+
#
|
|
384
|
+
# Candidates are validated with the full dictionary lookup,
|
|
385
|
+
# so affixed forms (stem + suffix/prefix, e.g. "definitely")
|
|
386
|
+
# and capitalization variants (e.g. "The") are accepted with
|
|
387
|
+
# their surface spelling preserved.
|
|
388
|
+
#
|
|
389
|
+
# @param context [Context] The suggestion context
|
|
390
|
+
# @param word [String] The misspelled word
|
|
391
|
+
# @param case_insensitive [Boolean] Comparison case handling
|
|
392
|
+
# @param max_dist [Integer] Maximum distance to accept
|
|
393
|
+
# @yield [String, Integer] Each accepted candidate and distance
|
|
394
|
+
def edit_sweep_candidates(context, word, case_insensitive:, max_dist:)
|
|
395
|
+
compare_word = case_insensitive ? word.downcase : word
|
|
396
|
+
try_string = try_string_for(context)
|
|
397
|
+
seen = Set.new
|
|
398
|
+
|
|
399
|
+
each_sweep_edit(word, try_string) do |candidate|
|
|
400
|
+
next if candidate == word
|
|
401
|
+
next unless seen.add?(candidate)
|
|
402
|
+
|
|
403
|
+
next unless dictionary_lookup(context, candidate)
|
|
404
|
+
|
|
405
|
+
compare_candidate = case_insensitive ? candidate.downcase : candidate
|
|
406
|
+
dist = edit_distance_with_threshold(compare_word, compare_candidate, max_dist)
|
|
407
|
+
next unless dist && dist > 0
|
|
408
|
+
|
|
409
|
+
yield candidate, dist
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
# Yield every single-edit variant of the word in canonical
|
|
414
|
+
# order: adjacent transpositions (Permutations.swapchar,
|
|
415
|
+
# restricted Damerau operation), substitutions over the TRY
|
|
416
|
+
# alphabet (Permutations.badchar), insertions over the TRY
|
|
417
|
+
# alphabet (Permutations.forgotchar), and deletions
|
|
418
|
+
# (Permutations.extrachar). Reuses the spylls-derived
|
|
419
|
+
# permutation primitives — the same generators the
|
|
420
|
+
# Hunspell-style suggester uses.
|
|
421
|
+
#
|
|
422
|
+
# @param word [String] The word to edit
|
|
423
|
+
# @param try_string [String, nil] Substitution/insertion alphabet
|
|
424
|
+
# @yield [String] Each edit variant
|
|
425
|
+
def each_sweep_edit(word, try_string, &block)
|
|
426
|
+
# Adjacent transposition (and, for 4/5-letter words,
|
|
427
|
+
# double adjacent swaps) — the most common typing error.
|
|
428
|
+
Algorithms::Permutations.swapchar(word, &block)
|
|
429
|
+
|
|
430
|
+
# Substitution of each character with each TRY character.
|
|
431
|
+
Algorithms::Permutations.badchar(word, try_string, &block)
|
|
432
|
+
|
|
433
|
+
# Insertion of each TRY character at each position.
|
|
434
|
+
Algorithms::Permutations.forgotchar(word, try_string, &block)
|
|
435
|
+
|
|
436
|
+
# Deletion of one character at each position.
|
|
437
|
+
Algorithms::Permutations.extrachar(word, &block)
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# The substitution/insertion alphabet for the edit sweep: an
|
|
441
|
+
# explicit +try_string+ config wins (used by callers without
|
|
442
|
+
# a TRY-capable dictionary); otherwise the dictionary's TRY
|
|
443
|
+
# string (nil for backends without one — the sweep then runs
|
|
444
|
+
# transpositions and deletions only).
|
|
445
|
+
#
|
|
446
|
+
# @param context [Context] The suggestion context
|
|
447
|
+
# @return [String, nil] TRY characters or nil
|
|
448
|
+
def try_string_for(context)
|
|
449
|
+
configured = get_config(:try_string)
|
|
450
|
+
return configured if configured
|
|
451
|
+
|
|
452
|
+
dictionary = context.dictionary
|
|
453
|
+
case dictionary
|
|
454
|
+
when Kotoshu::Dictionary::Base then dictionary.try_string
|
|
455
|
+
else nil
|
|
456
|
+
end
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
public
|
|
460
|
+
|
|
461
|
+
# Calculate enhanced score combining multiple factors.
|
|
462
|
+
#
|
|
463
|
+
# Lower score = better suggestion
|
|
464
|
+
#
|
|
465
|
+
# @param original [String] The original misspelled word
|
|
466
|
+
# @param suggestion [String] The suggested word
|
|
467
|
+
# @param distance [Integer] Edit distance
|
|
468
|
+
# @return [Float] Enhanced score (lower is better)
|
|
469
|
+
def calculate_enhanced_score(original, suggestion, distance)
|
|
470
|
+
score = distance * 1000.0 # Base score from edit distance
|
|
471
|
+
|
|
472
|
+
# Factor 1: Word frequency bonus (common words get lower score)
|
|
473
|
+
score -= frequency_bonus(suggestion)
|
|
474
|
+
|
|
475
|
+
# Factor 2: Keyboard proximity penalty (typo-like patterns get lower score)
|
|
476
|
+
score += keyboard_penalty(original, suggestion)
|
|
477
|
+
|
|
478
|
+
# Factor 3: Common typo pattern bonus
|
|
479
|
+
# Transposition (swap adjacent chars) is the MOST common typo
|
|
480
|
+
trans_bonus = transposition_bonus(original, suggestion)
|
|
481
|
+
score -= trans_bonus
|
|
482
|
+
|
|
483
|
+
# Factor 4: Missing double letter bonus (helo -> hello)
|
|
484
|
+
score -= typo_pattern_bonus(original, suggestion)
|
|
485
|
+
|
|
486
|
+
# Factor 5: Length similarity bonus (similar length is better)
|
|
487
|
+
length_diff = (original.length - suggestion.length).abs
|
|
488
|
+
score += length_diff * 50
|
|
489
|
+
|
|
490
|
+
score
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
# Calculate bonus for transposition (swap adjacent characters).
|
|
494
|
+
# This is the MOST common typing error, so it gets the highest bonus.
|
|
495
|
+
#
|
|
496
|
+
# @param original [String] The original word
|
|
497
|
+
# @param suggestion [String] The suggested word
|
|
498
|
+
# @return [Float] Transposition bonus (0 or 200)
|
|
499
|
+
def transposition_bonus(original, suggestion)
|
|
500
|
+
# Transposition only makes sense for same-length words
|
|
501
|
+
return 0 unless original.length == suggestion.length
|
|
502
|
+
|
|
503
|
+
o = original.downcase
|
|
504
|
+
s = suggestion.downcase
|
|
505
|
+
|
|
506
|
+
# Count transpositions needed
|
|
507
|
+
transpositions = 0
|
|
508
|
+
(0...o.length).each do |i|
|
|
509
|
+
next if o[i] == s[i]
|
|
510
|
+
|
|
511
|
+
# Find matching char in suggestion
|
|
512
|
+
match_idx = s.index(o[i], i + 1)
|
|
513
|
+
if match_idx && (match_idx == i + 1 || (match_idx > i + 1 && s[i] == o[match_idx]))
|
|
514
|
+
# This is a simple adjacent swap
|
|
515
|
+
transpositions += 1
|
|
516
|
+
end
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
# Only give bonus for single transposition
|
|
520
|
+
transpositions == 1 ? 200 : (transpositions * 100)
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# Calculate keyboard proximity penalty.
|
|
524
|
+
#
|
|
525
|
+
# Substitutions between adjacent keys get lower penalty.
|
|
526
|
+
# Uses OOP keyboard layout for language-aware distance calculations.
|
|
527
|
+
#
|
|
528
|
+
# @param original [String] The original word
|
|
529
|
+
# @param suggestion [String] The suggested word
|
|
530
|
+
# @return [Float] Keyboard penalty (0-200)
|
|
531
|
+
def keyboard_penalty(original, suggestion)
|
|
532
|
+
penalty = 0
|
|
533
|
+
|
|
534
|
+
# Find the edit script to see what changed
|
|
535
|
+
o_chars = original.chars
|
|
536
|
+
s_chars = suggestion.chars
|
|
537
|
+
|
|
538
|
+
# Simple comparison for equal-length words (substitutions)
|
|
539
|
+
if o_chars.length == s_chars.length
|
|
540
|
+
o_chars.each_with_index do |c1, i|
|
|
541
|
+
c2 = s_chars[i]
|
|
542
|
+
next if c1 == c2
|
|
543
|
+
|
|
544
|
+
# Use OOP keyboard layout for distance calculation
|
|
545
|
+
key_dist = @keyboard_layout.distance(c1, c2)
|
|
546
|
+
|
|
547
|
+
penalty += if key_dist == Float::INFINITY
|
|
548
|
+
# Symbol or unknown key - medium penalty
|
|
549
|
+
50
|
|
550
|
+
elsif key_dist == 1
|
|
551
|
+
10 # Very likely typo (adjacent keys)
|
|
552
|
+
elsif key_dist == 2
|
|
553
|
+
30 # Somewhat likely
|
|
554
|
+
else
|
|
555
|
+
100 # Unlikely to be typo (far keys)
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
penalty
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
# Calculate bonus for common typo patterns.
|
|
564
|
+
#
|
|
565
|
+
# @param original [String] The original word
|
|
566
|
+
# @param suggestion [String] The suggested word
|
|
567
|
+
# @return [Float] Pattern bonus (0-300)
|
|
568
|
+
def typo_pattern_bonus(original, suggestion)
|
|
569
|
+
bonus = 0
|
|
570
|
+
|
|
571
|
+
# Pattern 1: Missing double letter (helo -> hello)
|
|
572
|
+
# This is the MOST COMMON typo after transposition, give it highest bonus
|
|
573
|
+
if suggestion.length == original.length + 1
|
|
574
|
+
# Check if suggestion has a double letter that original is missing
|
|
575
|
+
suggestion.chars.each_cons(2).with_index do |pair, i|
|
|
576
|
+
if pair[0] == pair[1] # Found double letter at positions i and i+1
|
|
577
|
+
# Check if removing the second occurrence (at i+1) gives us the original word
|
|
578
|
+
# For "hello" with "ll" at position 2, remove position 3: "hel" + "o" = "helo"
|
|
579
|
+
expected = suggestion[0...i + 1] + suggestion[i + 2..-1]
|
|
580
|
+
if expected == original
|
|
581
|
+
bonus += 300 # Strong bonus for missing double letter (MORE than transposition!)
|
|
582
|
+
break
|
|
583
|
+
end
|
|
584
|
+
end
|
|
585
|
+
end
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# Pattern 2: Extra double letter (helllo -> hello)
|
|
589
|
+
if original.length == suggestion.length + 1
|
|
590
|
+
# Check if original has a double letter that suggestion doesn't
|
|
591
|
+
original.chars.each_cons(2).with_index do |pair, i|
|
|
592
|
+
if pair[0] == pair[1] # Found double letter in original
|
|
593
|
+
# Check if removing it gives the suggestion
|
|
594
|
+
reconstructed = original[0...i + 1] + original[i + 1..-1]
|
|
595
|
+
if reconstructed == suggestion
|
|
596
|
+
bonus += 100 # Bonus for extra double letter
|
|
597
|
+
break
|
|
598
|
+
end
|
|
599
|
+
end
|
|
600
|
+
end
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
# Pattern 3: Common prefixes/suffixes
|
|
604
|
+
if original.start_with?(suggestion[0...3]) && suggestion.length > original.length
|
|
605
|
+
bonus += 30 # Suggestion extends common prefix
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
bonus
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
private
|
|
612
|
+
|
|
613
|
+
# Resolve keyboard layout using OOP registry pattern
|
|
614
|
+
#
|
|
615
|
+
# @param keyboard_layout [Keyboard::Layout, String, nil] Layout override
|
|
616
|
+
# @return [Keyboard::Layout] Resolved layout
|
|
617
|
+
def resolve_keyboard_layout(keyboard_layout)
|
|
618
|
+
if keyboard_layout.is_a?(Keyboard::Layout)
|
|
619
|
+
keyboard_layout
|
|
620
|
+
elsif keyboard_layout.is_a?(String)
|
|
621
|
+
Keyboard::Registry.layout_by_name(keyboard_layout)
|
|
622
|
+
elsif @language_code
|
|
623
|
+
Keyboard::Registry.layout_for(@language_code)
|
|
624
|
+
else
|
|
625
|
+
Keyboard::Registry.layout_by_name('QWERTY')
|
|
626
|
+
end
|
|
627
|
+
end
|
|
628
|
+
end
|
|
629
|
+
end
|
|
630
|
+
end
|
|
631
|
+
end
|