kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# Keyboard proximity suggestion strategy.
|
|
7
|
+
#
|
|
8
|
+
# Generates suggestions by finding words that can be formed by
|
|
9
|
+
# substituting adjacent keys on a QWERTY keyboard.
|
|
10
|
+
#
|
|
11
|
+
# @example Creating a keyboard proximity strategy
|
|
12
|
+
# strategy = KeyboardProximityStrategy.new
|
|
13
|
+
# result = strategy.generate(context)
|
|
14
|
+
class KeyboardProximityStrategy < BaseStrategy
|
|
15
|
+
# QWERTY keyboard layout (US).
|
|
16
|
+
#
|
|
17
|
+
# Each key maps to its adjacent keys.
|
|
18
|
+
KEYBOARD_LAYOUT = {
|
|
19
|
+
"`" => %w[1 tab],
|
|
20
|
+
"1" => ["`", "2", "q"],
|
|
21
|
+
"2" => %w[1 3 w q],
|
|
22
|
+
"3" => %w[2 4 e w],
|
|
23
|
+
"4" => %w[3 5 r e],
|
|
24
|
+
"5" => %w[4 6 t r],
|
|
25
|
+
"6" => %w[5 7 y t],
|
|
26
|
+
"7" => %w[6 8 u y],
|
|
27
|
+
"8" => %w[7 9 i u],
|
|
28
|
+
"9" => %w[8 0 o i],
|
|
29
|
+
"0" => %w[9 p o],
|
|
30
|
+
"-" => ["0", "="],
|
|
31
|
+
"=" => ["-"],
|
|
32
|
+
"q" => %w[tab w a 1],
|
|
33
|
+
"w" => %w[q e a s 2],
|
|
34
|
+
"e" => %w[w r s d 3],
|
|
35
|
+
"r" => %w[e t d f 4],
|
|
36
|
+
"t" => %w[r y f g 5],
|
|
37
|
+
"y" => %w[t u g h 6],
|
|
38
|
+
"u" => %w[y i h j 7],
|
|
39
|
+
"i" => %w[u o j k 8],
|
|
40
|
+
"o" => %w[i p k l 9],
|
|
41
|
+
"p" => ["o", "l", ";", "0"],
|
|
42
|
+
"[" => ["p", "'"],
|
|
43
|
+
"]" => ["enter", "\\"],
|
|
44
|
+
"\\" => ["enter"], # Backslash neighbors
|
|
45
|
+
"a" => %w[caps s z q],
|
|
46
|
+
"s" => %w[a d z x w],
|
|
47
|
+
"d" => %w[s f x c e],
|
|
48
|
+
"f" => %w[d g c v r],
|
|
49
|
+
"g" => %w[f h v b t],
|
|
50
|
+
"h" => %w[g j b n y],
|
|
51
|
+
"j" => %w[h k n m u],
|
|
52
|
+
"k" => ["j", "l", "m", ",", "i"],
|
|
53
|
+
"l" => ["k", ";", ",", ".", "o"],
|
|
54
|
+
";" => ["l", "'", ".", "p"],
|
|
55
|
+
"'" => [";"],
|
|
56
|
+
"z" => %w[shift s x a],
|
|
57
|
+
"x" => %w[z c s d],
|
|
58
|
+
"c" => %w[x v d f],
|
|
59
|
+
"v" => %w[c b f g],
|
|
60
|
+
"b" => %w[v n g h],
|
|
61
|
+
"n" => %w[b m h j],
|
|
62
|
+
"m" => ["n", ",", "j", "k"],
|
|
63
|
+
"," => ["m", ".", "k", "l"],
|
|
64
|
+
"." => [",", "/", "l", ";"],
|
|
65
|
+
"/" => [".", "shift"],
|
|
66
|
+
" " => [] # Space has no neighbors
|
|
67
|
+
}.freeze
|
|
68
|
+
|
|
69
|
+
# Create a new keyboard proximity strategy.
|
|
70
|
+
#
|
|
71
|
+
# @param name [String, Symbol] Name of the strategy
|
|
72
|
+
# @param config [Hash] Configuration options
|
|
73
|
+
# @option config [Integer] max_distance Maximum keyboard distance
|
|
74
|
+
# @option config [Integer] max_results Maximum results to return
|
|
75
|
+
def initialize(name: :keyboard_proximity, **config)
|
|
76
|
+
super
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Generate suggestions based on keyboard proximity.
|
|
80
|
+
#
|
|
81
|
+
# @param context [Context] The suggestion context
|
|
82
|
+
# @return [SuggestionSet] Suggestions within keyboard distance
|
|
83
|
+
def generate(context)
|
|
84
|
+
word = context.word
|
|
85
|
+
max_dist = get_config(:max_distance, 2)
|
|
86
|
+
min_similarity = get_config(:min_similarity, 0.70) # Filter low-similarity suggestions
|
|
87
|
+
|
|
88
|
+
# One index per generate call: the variant set reaches tens
|
|
89
|
+
# of thousands of lookups, each of which used to scan the
|
|
90
|
+
# whole word list (with a downcase per candidate on the
|
|
91
|
+
# case-insensitive branch). WordIndex preserves find_word
|
|
92
|
+
# semantics exactly — exact (lowercase) match first, then
|
|
93
|
+
# first-in-list-order on the case-insensitive branch.
|
|
94
|
+
index = WordIndex.build(dictionary_words(context))
|
|
95
|
+
|
|
96
|
+
# Generate keyboard variants
|
|
97
|
+
variants = keyboard_variants(word, max_dist)
|
|
98
|
+
|
|
99
|
+
# Find matching dictionary words with their edit distances and similarity
|
|
100
|
+
results_with_distances = {}
|
|
101
|
+
variants.each do |variant|
|
|
102
|
+
dict_word = index.find(variant)
|
|
103
|
+
next unless dict_word && dict_word != word
|
|
104
|
+
|
|
105
|
+
# Calculate edit distance from original word
|
|
106
|
+
dist = edit_distance(word, dict_word)
|
|
107
|
+
next if dist > max_dist
|
|
108
|
+
|
|
109
|
+
# Calculate typo correction similarity
|
|
110
|
+
similarity = calculate_ngram_similarity(word, dict_word)
|
|
111
|
+
next if similarity < min_similarity # Filter by similarity threshold
|
|
112
|
+
|
|
113
|
+
# Keep the minimum distance for each word
|
|
114
|
+
results_with_distances[dict_word] ||= dist
|
|
115
|
+
results_with_distances[dict_word] = dist if dist < results_with_distances[dict_word]
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Sort by distance and create suggestions
|
|
119
|
+
sorted_words = results_with_distances.sort_by { |_, dist| dist }.map(&:first)
|
|
120
|
+
create_suggestion_set(sorted_words, distances: results_with_distances, original_word: word)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Check if this strategy should handle the context.
|
|
124
|
+
#
|
|
125
|
+
# @param context [Context] The suggestion context
|
|
126
|
+
# @return [Boolean] True if the word needs correction
|
|
127
|
+
def handles?(context)
|
|
128
|
+
return false unless enabled?
|
|
129
|
+
|
|
130
|
+
!dictionary_lookup(context, context.word)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
private
|
|
134
|
+
|
|
135
|
+
# Calculate edit distance between two strings.
|
|
136
|
+
# Uses Damerau-Levenshtein distance (substitution, insertion,
|
|
137
|
+
# deletion, adjacent transposition) via Algorithms::EditDistance,
|
|
138
|
+
# so a swapped pair ("teh" -> "the") costs 1, not 2.
|
|
139
|
+
#
|
|
140
|
+
# @param str1 [String] First string
|
|
141
|
+
# @param str2 [String] Second string
|
|
142
|
+
# @return [Integer] Edit distance
|
|
143
|
+
def edit_distance(str1, str2)
|
|
144
|
+
Algorithms::EditDistance.distance(str1, str2)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Get neighbors for a key.
|
|
148
|
+
#
|
|
149
|
+
# @param char [String] The character
|
|
150
|
+
# @return [Array<String>] Neighbor keys
|
|
151
|
+
def neighbors(char)
|
|
152
|
+
KEYBOARD_LAYOUT[char.downcase] || []
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Generate keyboard variants of a word.
|
|
156
|
+
#
|
|
157
|
+
# Each round applies one edit operation to every current
|
|
158
|
+
# variant: substitution with a keyboard-adjacent key,
|
|
159
|
+
# insertion of an adjacent key, deletion, and adjacent
|
|
160
|
+
# transposition (the restricted Damerau operation — the most
|
|
161
|
+
# common typing error, which is why it is enumerated here as a
|
|
162
|
+
# single edit like the others).
|
|
163
|
+
#
|
|
164
|
+
# @param word [String] The word
|
|
165
|
+
# @param max_distance [Integer] Maximum edit distance
|
|
166
|
+
# @return [Array<String>] Keyboard variants
|
|
167
|
+
def keyboard_variants(word, max_distance)
|
|
168
|
+
return [] if word.nil? || word.empty?
|
|
169
|
+
|
|
170
|
+
word = word.downcase
|
|
171
|
+
variants = Set.new([word])
|
|
172
|
+
|
|
173
|
+
max_distance.times do
|
|
174
|
+
new_variants = Set.new
|
|
175
|
+
|
|
176
|
+
variants.each do |variant|
|
|
177
|
+
# Generate all single-key substitutions
|
|
178
|
+
variant.each_char.with_index do |char, i|
|
|
179
|
+
neighbors(char).each do |neighbor|
|
|
180
|
+
new_word = variant[0...i] + neighbor + variant[(i + 1)..]
|
|
181
|
+
new_variants.add(new_word)
|
|
182
|
+
|
|
183
|
+
# Also try insertions and deletions
|
|
184
|
+
new_variants.add(variant[0...i] + variant[(i + 1)..]) # Delete
|
|
185
|
+
new_variants.add(variant[0...i] + neighbor + variant[i..]) # Insert
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Adjacent transposition: swap each neighboring
|
|
190
|
+
# character pair (a single 1-cost edit).
|
|
191
|
+
(0...variant.length - 1).each do |i|
|
|
192
|
+
swapped = variant[0...i] + variant[i + 1] + variant[i] + variant[(i + 2)..]
|
|
193
|
+
new_variants.add(swapped)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
variants = new_variants
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
variants.to_a
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
require "set" if RUBY_VERSION < "3.0"
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# N-gram suggestion strategy.
|
|
7
|
+
#
|
|
8
|
+
# Generates suggestions by finding words with high n-gram similarity.
|
|
9
|
+
# N-grams are contiguous sequences of n characters.
|
|
10
|
+
#
|
|
11
|
+
# @example Creating an n-gram strategy
|
|
12
|
+
# strategy = NgramStrategy.new(n: 3)
|
|
13
|
+
# result = strategy.generate(context)
|
|
14
|
+
class NgramStrategy < BaseStrategy
|
|
15
|
+
# Create a new n-gram strategy.
|
|
16
|
+
#
|
|
17
|
+
# @param name [String, Symbol] Name of the strategy
|
|
18
|
+
# @param config [Hash] Configuration options
|
|
19
|
+
# @option config [Integer] n N-gram size (default: 3)
|
|
20
|
+
# @option config [Float] min_similarity Minimum similarity threshold (0-1)
|
|
21
|
+
# @option config [Integer] max_results Maximum results to return
|
|
22
|
+
def initialize(name: :ngram, **config)
|
|
23
|
+
super
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Generate suggestions based on n-gram similarity.
|
|
27
|
+
#
|
|
28
|
+
# @param context [Context] The suggestion context
|
|
29
|
+
# @return [SuggestionSet] Suggestions with high n-gram similarity
|
|
30
|
+
def generate(context)
|
|
31
|
+
word = context.word
|
|
32
|
+
n = get_config(:n, 3)
|
|
33
|
+
min_sim = get_config(:min_similarity, 0.3)
|
|
34
|
+
min_typo_similarity = get_config(:min_typo_similarity, 0.70) # Filter by typo correction similarity
|
|
35
|
+
|
|
36
|
+
return create_suggestion_set([]) if word.length < n
|
|
37
|
+
|
|
38
|
+
# Get n-grams for input word
|
|
39
|
+
word_ngrams = extract_ngrams(word, n)
|
|
40
|
+
word_length = word.length
|
|
41
|
+
|
|
42
|
+
# Calculate n-gram similarity for each dictionary word
|
|
43
|
+
results = {}
|
|
44
|
+
each_word_with_length(context) do |dict_word, dict_length|
|
|
45
|
+
next if dict_word == word
|
|
46
|
+
next if dict_length < n
|
|
47
|
+
|
|
48
|
+
# Length pre-gate: similarity is a Jaccard coefficient
|
|
49
|
+
# over n-gram multisets, so it is at most
|
|
50
|
+
# min-gram-count / max-gram-count — each word of length L
|
|
51
|
+
# contributes L - n + 1 grams. A word whose length bound
|
|
52
|
+
# already sits under min_similarity can never pass the
|
|
53
|
+
# real test, so skip it before extracting a single
|
|
54
|
+
# n-gram. Mathematically the same set of words passes
|
|
55
|
+
# (same gate as ngram.rs in kotoshu-rs). The length is
|
|
56
|
+
# read from the dictionary's memoized sweep index before
|
|
57
|
+
# any per-word work when the backend maintains one.
|
|
58
|
+
min_grams = (dict_length < word_length ? dict_length : word_length) - (n - 1)
|
|
59
|
+
max_grams = (dict_length > word_length ? dict_length : word_length) - (n - 1)
|
|
60
|
+
next if min_grams.to_f / max_grams < min_sim
|
|
61
|
+
|
|
62
|
+
similarity = ngram_similarity(word_ngrams, dict_word, n)
|
|
63
|
+
next if similarity < min_sim
|
|
64
|
+
|
|
65
|
+
# Also check typo correction similarity for filtering
|
|
66
|
+
typo_sim = calculate_ngram_similarity(word, dict_word)
|
|
67
|
+
next if typo_sim < min_typo_similarity
|
|
68
|
+
|
|
69
|
+
# Convert similarity to distance (higher similarity = lower distance)
|
|
70
|
+
dist = ((1 - similarity) * 10).to_i
|
|
71
|
+
next if dist.zero?
|
|
72
|
+
|
|
73
|
+
results[dict_word] ||= dist
|
|
74
|
+
results[dict_word] = dist if dist < results[dict_word]
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Convert to suggestions sorted by similarity
|
|
78
|
+
sorted_words = results.sort_by { |_, dist| dist }.map(&:first)
|
|
79
|
+
create_suggestion_set(sorted_words, distances: results, original_word: word)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Check if this strategy should handle the context.
|
|
83
|
+
#
|
|
84
|
+
# @param context [Context] The suggestion context
|
|
85
|
+
# @return [Boolean] True if the word needs correction
|
|
86
|
+
def handles?(context)
|
|
87
|
+
return false unless enabled?
|
|
88
|
+
|
|
89
|
+
!dictionary_lookup(context, context.word)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
# Yield each dictionary word paired with its char length, in
|
|
95
|
+
# word-list order. Over a Dictionary::Base backend the pairs
|
|
96
|
+
# come from the dictionary's memoized sweep index
|
|
97
|
+
# ({Dictionary::Base#sweep_index}) — the length gate reads the
|
|
98
|
+
# indexed value before any per-word work, and the sweep never
|
|
99
|
+
# copies the word list. Ad-hoc Hash/Array dictionaries compute
|
|
100
|
+
# the length live, exactly as before.
|
|
101
|
+
#
|
|
102
|
+
# @param context [Context] The suggestion context
|
|
103
|
+
# @yield [String, Integer] Each word and its char length
|
|
104
|
+
# @return [void]
|
|
105
|
+
def each_word_with_length(context)
|
|
106
|
+
dictionary = context.dictionary
|
|
107
|
+
if dictionary.is_a?(Kotoshu::Dictionary::Base)
|
|
108
|
+
dictionary.sweep_index.each_with_length { |*pair| yield(*pair) }
|
|
109
|
+
else
|
|
110
|
+
dictionary_words(context).each { |dict_word| yield dict_word, dict_word.length }
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Extract n-grams from a word.
|
|
115
|
+
#
|
|
116
|
+
# @param word [String] The word
|
|
117
|
+
# @param n [Integer] N-gram size
|
|
118
|
+
# @return [Hash] N-gram to count mapping
|
|
119
|
+
def extract_ngrams(word, n)
|
|
120
|
+
ngrams = Hash.new(0)
|
|
121
|
+
|
|
122
|
+
(word.length - n + 1).times do |i|
|
|
123
|
+
ngram = word[i...i + n]
|
|
124
|
+
ngrams[ngram] += 1
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
ngrams
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Calculate n-gram similarity between two words.
|
|
131
|
+
#
|
|
132
|
+
# Uses the Jaccard similarity coefficient:
|
|
133
|
+
# similarity = |intersection| / |union|
|
|
134
|
+
#
|
|
135
|
+
# @param word_ngrams [Hash] N-grams for the first word
|
|
136
|
+
# @param other_word [String] The second word
|
|
137
|
+
# @param n [Integer] N-gram size
|
|
138
|
+
# @return [Float] Similarity score (0-1)
|
|
139
|
+
def ngram_similarity(word_ngrams, other_word, n)
|
|
140
|
+
other_ngrams = extract_ngrams(other_word, n)
|
|
141
|
+
|
|
142
|
+
# Intersection and union in one pass over both maps. The
|
|
143
|
+
# original built the key union as an intermediate Array
|
|
144
|
+
# (`word_ngrams.keys | other_ngrams.keys`) — one allocation
|
|
145
|
+
# per dictionary word that dominated the sweep once the
|
|
146
|
+
# length gate landed.
|
|
147
|
+
intersection = 0
|
|
148
|
+
union = 0
|
|
149
|
+
word_ngrams.each do |ngram, count|
|
|
150
|
+
other_count = other_ngrams[ngram] || 0
|
|
151
|
+
intersection += count < other_count ? count : other_count
|
|
152
|
+
union += count > other_count ? count : other_count
|
|
153
|
+
end
|
|
154
|
+
other_ngrams.each do |ngram, count|
|
|
155
|
+
union += count unless word_ngrams.key?(ngram)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
return 0.0 if union.zero?
|
|
159
|
+
|
|
160
|
+
intersection.to_f / union
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
module Strategies
|
|
6
|
+
# Phonetic suggestion strategy.
|
|
7
|
+
#
|
|
8
|
+
# Generates suggestions by finding words with similar phonetic codes
|
|
9
|
+
# using algorithms like Soundex and Metaphone.
|
|
10
|
+
#
|
|
11
|
+
# @example Creating a phonetic strategy
|
|
12
|
+
# strategy = PhoneticStrategy.new(algorithm: :soundex)
|
|
13
|
+
# result = strategy.generate(context)
|
|
14
|
+
class PhoneticStrategy < BaseStrategy
|
|
15
|
+
# Supported algorithms.
|
|
16
|
+
ALGORITHMS = %i[soundex metaphone].freeze
|
|
17
|
+
|
|
18
|
+
# Create a new phonetic strategy.
|
|
19
|
+
#
|
|
20
|
+
# @param name [String, Symbol] Name of the strategy
|
|
21
|
+
# @param config [Hash] Configuration options
|
|
22
|
+
# @option config [Symbol] algorithm The algorithm to use (:soundex or :metaphone)
|
|
23
|
+
# @option config [Integer] max_results Maximum results to return
|
|
24
|
+
def initialize(name: :phonetic, **config)
|
|
25
|
+
super
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Generate suggestions based on phonetic similarity.
|
|
29
|
+
#
|
|
30
|
+
# @param context [Context] The suggestion context
|
|
31
|
+
# @return [SuggestionSet] Suggestions with same phonetic code
|
|
32
|
+
def generate(context)
|
|
33
|
+
word = context.word
|
|
34
|
+
algorithm = get_config(:algorithm, :soundex)
|
|
35
|
+
max_dist = 2
|
|
36
|
+
|
|
37
|
+
# Get phonetic code for input word
|
|
38
|
+
word_code = phonetic_code(word, algorithm)
|
|
39
|
+
|
|
40
|
+
# Find words with same phonetic code
|
|
41
|
+
results = []
|
|
42
|
+
each_phonetic_pair(context, algorithm) do |dict_word, dict_code|
|
|
43
|
+
next if dict_word == word
|
|
44
|
+
next unless dict_code == word_code
|
|
45
|
+
|
|
46
|
+
dist = edit_distance(word, dict_word)
|
|
47
|
+
next if dist > max_dist || dist.zero?
|
|
48
|
+
|
|
49
|
+
results << [dict_word, dist]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Sort by distance and convert to suggestions
|
|
53
|
+
sorted_words = results.sort_by { |_, dist| dist }.map(&:first)
|
|
54
|
+
create_suggestion_set(sorted_words)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Check if this strategy should handle the context.
|
|
58
|
+
#
|
|
59
|
+
# @param context [Context] The suggestion context
|
|
60
|
+
# @return [Boolean] True if the word needs correction
|
|
61
|
+
def handles?(context)
|
|
62
|
+
return false unless enabled?
|
|
63
|
+
|
|
64
|
+
!dictionary_lookup(context, context.word)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
# Yield each dictionary word paired with its phonetic code, in
|
|
70
|
+
# word-list order. For Soundex over a Dictionary::Base backend
|
|
71
|
+
# the codes come from the dictionary's memoized sweep index
|
|
72
|
+
# ({Dictionary::Base#sweep_index}) — one Soundex per word for
|
|
73
|
+
# the dictionary's lifetime instead of one per sweep. Every
|
|
74
|
+
# other combination (ad-hoc Hash/Array dictionaries, Metaphone)
|
|
75
|
+
# computes each code live, exactly as before.
|
|
76
|
+
#
|
|
77
|
+
# @param context [Context] The suggestion context
|
|
78
|
+
# @param algorithm [Symbol] The phonetic algorithm
|
|
79
|
+
# @yield [String, String] Each word and its phonetic code
|
|
80
|
+
# @return [void]
|
|
81
|
+
def each_phonetic_pair(context, algorithm)
|
|
82
|
+
dictionary = context.dictionary
|
|
83
|
+
if algorithm == :soundex && dictionary.is_a?(Kotoshu::Dictionary::Base)
|
|
84
|
+
dictionary.sweep_index.each_with_soundex { |*pair| yield(*pair) }
|
|
85
|
+
else
|
|
86
|
+
dictionary_words(context).each do |dict_word|
|
|
87
|
+
yield dict_word, phonetic_code(dict_word, algorithm)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Get phonetic code for a word.
|
|
93
|
+
#
|
|
94
|
+
# @param word [String] The word
|
|
95
|
+
# @param algorithm [Symbol] The algorithm to use
|
|
96
|
+
# @return [String] The phonetic code
|
|
97
|
+
def phonetic_code(word, algorithm = :soundex)
|
|
98
|
+
case algorithm
|
|
99
|
+
when :soundex
|
|
100
|
+
soundex_code(word)
|
|
101
|
+
when :metaphone
|
|
102
|
+
metaphone_code(word)
|
|
103
|
+
else
|
|
104
|
+
soundex_code(word)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Calculate Soundex code for a word.
|
|
109
|
+
#
|
|
110
|
+
# Soundex is a phonetic algorithm developed by Robert C. Russell
|
|
111
|
+
# and Margaret King Odell in the early 1900s. The implementation
|
|
112
|
+
# lives in {Algorithms::Soundex} and is shared with
|
|
113
|
+
# {Suggestions::SweepIndex}, which memoizes the code per
|
|
114
|
+
# dictionary word.
|
|
115
|
+
#
|
|
116
|
+
# @param word [String] The word
|
|
117
|
+
# @return [String] The Soundex code (letter + 3 digits)
|
|
118
|
+
#
|
|
119
|
+
# @example
|
|
120
|
+
# soundex_code("Robert") # => "R163"
|
|
121
|
+
# soundex_code("Rupert") # => "R163"
|
|
122
|
+
# soundex_code("Ashcraft") # => "A261"
|
|
123
|
+
def soundex_code(word)
|
|
124
|
+
Algorithms::Soundex.code(word)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Calculate Metaphone code for a word.
|
|
128
|
+
#
|
|
129
|
+
# Metaphone is an improved phonetic algorithm developed by
|
|
130
|
+
# Lawrence Philips in 1990.
|
|
131
|
+
#
|
|
132
|
+
# @param word [String] The word
|
|
133
|
+
# @return [String] The Metaphone code
|
|
134
|
+
#
|
|
135
|
+
# @example
|
|
136
|
+
# metaphone_code("Schmidt") # => "XMT"
|
|
137
|
+
# metaphone_code("Smith") # => "SM0"
|
|
138
|
+
def metaphone_code(word)
|
|
139
|
+
return "" if word.nil? || word.empty?
|
|
140
|
+
|
|
141
|
+
word = word.upcase.gsub(/[^A-Z]/, "")
|
|
142
|
+
return "" if word.empty?
|
|
143
|
+
|
|
144
|
+
# Metaphone rules (simplified implementation)
|
|
145
|
+
code = ""
|
|
146
|
+
i = 0
|
|
147
|
+
length = word.length
|
|
148
|
+
|
|
149
|
+
while i < length && code.length < 4
|
|
150
|
+
char = word[i]
|
|
151
|
+
next_char = i + 1 < length ? word[i + 1] : ""
|
|
152
|
+
|
|
153
|
+
case char
|
|
154
|
+
when "A", "E", "I", "O", "U"
|
|
155
|
+
# Vowels are only encoded at the beginning
|
|
156
|
+
code += char if i.zero?
|
|
157
|
+
when "B"
|
|
158
|
+
code += "B"
|
|
159
|
+
when "C"
|
|
160
|
+
if next_char == "H" && i + 2 < length && %w[A E I O U].include?(word[i + 2])
|
|
161
|
+
# "CH" followed by vowel => "X"
|
|
162
|
+
code += "X"
|
|
163
|
+
i += 1
|
|
164
|
+
elsif next_char == "I" && i + 2 < length && word[i + 2] == "A"
|
|
165
|
+
# "CIA" => "X"
|
|
166
|
+
code += "X"
|
|
167
|
+
i += 2
|
|
168
|
+
elsif %w[S G].include?(next_char)
|
|
169
|
+
# "CS", "CG" => "X"
|
|
170
|
+
code += "X"
|
|
171
|
+
i += 1
|
|
172
|
+
else
|
|
173
|
+
code += "K"
|
|
174
|
+
end
|
|
175
|
+
when "D"
|
|
176
|
+
if next_char == "G" && i + 2 < length && %w[I E Y].include?(word[i + 2])
|
|
177
|
+
# "DG" followed by I, E, Y => "J"
|
|
178
|
+
code += "J"
|
|
179
|
+
i += 1
|
|
180
|
+
else
|
|
181
|
+
code += "T"
|
|
182
|
+
end
|
|
183
|
+
when "F"
|
|
184
|
+
code += "F"
|
|
185
|
+
when "G"
|
|
186
|
+
if next_char == "H"
|
|
187
|
+
# "GH" => silent unless at beginning or after vowel
|
|
188
|
+
if i.zero?
|
|
189
|
+
code += "K"
|
|
190
|
+
i += 1
|
|
191
|
+
end
|
|
192
|
+
elsif next_char == "N"
|
|
193
|
+
# "GN" => "N" (silent G)
|
|
194
|
+
i += 1
|
|
195
|
+
elsif next_char == "N" && i + 2 < length && word[i + 2] == "E" && i + 3 < length && word[i + 3] == "D"
|
|
196
|
+
# "GNED" => "N" (silent G)
|
|
197
|
+
i += 3
|
|
198
|
+
else
|
|
199
|
+
code += "K"
|
|
200
|
+
end
|
|
201
|
+
when "H"
|
|
202
|
+
# H is silent unless at beginning
|
|
203
|
+
code += "H" if i.zero?
|
|
204
|
+
when "J"
|
|
205
|
+
code += "J"
|
|
206
|
+
when "K"
|
|
207
|
+
code += "K"
|
|
208
|
+
i += 1 if next_char == "N" # "KN" => "N"
|
|
209
|
+
when "L"
|
|
210
|
+
code += "L"
|
|
211
|
+
when "M"
|
|
212
|
+
code += "M"
|
|
213
|
+
when "N"
|
|
214
|
+
code += "N"
|
|
215
|
+
when "P"
|
|
216
|
+
if next_char == "H"
|
|
217
|
+
# "PH" => "F"
|
|
218
|
+
code += "F"
|
|
219
|
+
i += 1
|
|
220
|
+
else
|
|
221
|
+
code += "P"
|
|
222
|
+
end
|
|
223
|
+
when "Q"
|
|
224
|
+
code += "K"
|
|
225
|
+
when "R"
|
|
226
|
+
code += "R"
|
|
227
|
+
when "S"
|
|
228
|
+
if next_char == "H"
|
|
229
|
+
# "SH" => "X"
|
|
230
|
+
code += "X"
|
|
231
|
+
i += 1
|
|
232
|
+
elsif next_char == "I" && i + 2 < length && word[i + 2] == "O"
|
|
233
|
+
# "SIO" or "SIA" => "X"
|
|
234
|
+
code += "X"
|
|
235
|
+
i += 2
|
|
236
|
+
else
|
|
237
|
+
code += "S"
|
|
238
|
+
end
|
|
239
|
+
when "T"
|
|
240
|
+
if next_char == "I" && i + 2 < length && %w[O A].include?(word[i + 2])
|
|
241
|
+
# "TIO" or "TIA" => "X"
|
|
242
|
+
code += "X"
|
|
243
|
+
i += 2
|
|
244
|
+
elsif next_char == "H"
|
|
245
|
+
# "TH" => "0"
|
|
246
|
+
code += "0"
|
|
247
|
+
i += 1
|
|
248
|
+
else
|
|
249
|
+
code += "T"
|
|
250
|
+
end
|
|
251
|
+
when "V"
|
|
252
|
+
code += "F"
|
|
253
|
+
when "W", "Y"
|
|
254
|
+
# W and Y are semi-vowels, only encode at beginning
|
|
255
|
+
code += char if i.zero?
|
|
256
|
+
when "X"
|
|
257
|
+
code += "KS"
|
|
258
|
+
when "Z"
|
|
259
|
+
code += "S"
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
i += 1
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
code[0...4] # Max 4 characters
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Calculate Damerau-Levenshtein edit distance.
|
|
269
|
+
#
|
|
270
|
+
# Delegates to Algorithms::EditDistance so phonetically equal
|
|
271
|
+
# candidates are measured with the same metric as the edit
|
|
272
|
+
# distance strategy — an adjacent transposition ("recieve" ->
|
|
273
|
+
# "receive") costs 1, not 2.
|
|
274
|
+
#
|
|
275
|
+
# @param str1 [String] First string
|
|
276
|
+
# @param str2 [String] Second string
|
|
277
|
+
# @return [Integer] Edit distance
|
|
278
|
+
def edit_distance(str1, str2)
|
|
279
|
+
Algorithms::EditDistance.distance(str1, str2)
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|