kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Algorithms
|
|
5
|
+
# Capitalization handling for different languages.
|
|
6
|
+
#
|
|
7
|
+
# Ported from Spylls (Python) capitalization.py
|
|
8
|
+
#
|
|
9
|
+
# This module provides capitalization detection and conversion for different
|
|
10
|
+
# language casing rules, including special handling for Turkic and German languages.
|
|
11
|
+
module Capitalization
|
|
12
|
+
# Type of capitalization detected by Casing.guess.
|
|
13
|
+
#
|
|
14
|
+
# NO:: all lowercase ("foo")
|
|
15
|
+
# INIT:: titlecase, only initial letter is capitalized ("Foo")
|
|
16
|
+
# ALL:: all uppercase ("FOO")
|
|
17
|
+
# HUH:: mixed capitalization ("fooBar")
|
|
18
|
+
# HUHINIT:: mixed capitalization, first letter is capitalized ("FooBar")
|
|
19
|
+
module Type
|
|
20
|
+
NO = :no
|
|
21
|
+
INIT = :init
|
|
22
|
+
ALL = :all
|
|
23
|
+
HUH = :huh
|
|
24
|
+
HUHINIT = :huhinit
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Base class for casing-related algorithms specific for dictionary's language.
|
|
28
|
+
#
|
|
29
|
+
# This is a class (not a set of functions) because it needs to have
|
|
30
|
+
# subclasses for specific language casing, which have only some aspects
|
|
31
|
+
# different from generic one.
|
|
32
|
+
class Casing
|
|
33
|
+
# Guess word's capitalization. Redefined in GermanCasing.
|
|
34
|
+
#
|
|
35
|
+
# @param word [String] The word to analyze
|
|
36
|
+
# @return [Symbol] One of the Type constants
|
|
37
|
+
def guess(word)
|
|
38
|
+
return Type::NO if word.downcase == word
|
|
39
|
+
return Type::ALL if word.upcase == word
|
|
40
|
+
return Type::INIT if word[0].upcase == word[0] && word[1..].downcase == word[1..]
|
|
41
|
+
|
|
42
|
+
if word[0].upcase == word[0]
|
|
43
|
+
Type::HUHINIT
|
|
44
|
+
else
|
|
45
|
+
Type::HUH
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Lowercases the word. Returns list of possible lowercasings for all
|
|
50
|
+
# casing classes to behave consistently.
|
|
51
|
+
#
|
|
52
|
+
# In GermanCasing (and only there), lowercasing word like "STRASSE"
|
|
53
|
+
# produces two possibilities: "strasse" and "ße" (ß is most of the time
|
|
54
|
+
# upcased to SS, so we can't decide which of downcased words is "right"
|
|
55
|
+
# and need to check both).
|
|
56
|
+
#
|
|
57
|
+
# Also redefined in TurkicCasing, because in Turkic languages lowercase
|
|
58
|
+
# "i" is uppercased as "İ", and uppercase "I" is downcased as "ı".
|
|
59
|
+
#
|
|
60
|
+
# @param word [String] The word to lowercase
|
|
61
|
+
# @return [Array<String>] List of possible lowercasings
|
|
62
|
+
def lower(word)
|
|
63
|
+
# Can't be properly lowercased in non-Turkic collation
|
|
64
|
+
return [] if word.nil? || word.empty? || word[0] == 'İ'
|
|
65
|
+
|
|
66
|
+
# Turkic "lowercase dot i" to latinic "i", just in case
|
|
67
|
+
[word.downcase.gsub('i̇', 'i')]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Uppercase the word. Redefined in TurkicCasing, because in Turkic
|
|
71
|
+
# languages lowercase "i" is uppercased as "İ", and uppercase "I"
|
|
72
|
+
# is downcased as "ı".
|
|
73
|
+
#
|
|
74
|
+
# @param word [String] The word to uppercase
|
|
75
|
+
# @return [String] Uppercased word
|
|
76
|
+
def upper(word)
|
|
77
|
+
word.upcase
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Capitalize (convert word to all lowercase and first letter uppercase).
|
|
81
|
+
# Returns a list of results for same reasons as lower.
|
|
82
|
+
#
|
|
83
|
+
# @param word [String] The word to capitalize
|
|
84
|
+
# @return [Enumerator<String>] Enum of capitalized variants
|
|
85
|
+
def capitalize(word)
|
|
86
|
+
return enum_for(:capitalize, word) unless block_given?
|
|
87
|
+
|
|
88
|
+
if word.length == 1
|
|
89
|
+
yield upper(word[0])
|
|
90
|
+
else
|
|
91
|
+
upper_first = upper(word[0])
|
|
92
|
+
lower(word[1..]).each do |lowered|
|
|
93
|
+
yield upper_first + lowered
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Just change the case of the first letter to lower.
|
|
99
|
+
# Returns a list of results for same reasons as lower.
|
|
100
|
+
#
|
|
101
|
+
# @param word [String] The word to process
|
|
102
|
+
# @return [Enumerator<String>] Enum of variants with lowercased first letter
|
|
103
|
+
def lowerfirst(word)
|
|
104
|
+
return enum_for(:lowerfirst, word) unless block_given?
|
|
105
|
+
|
|
106
|
+
lower(word[0]).each do |lowered|
|
|
107
|
+
yield lowered + word[1..]
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Returns hypotheses of how the word might have been cased (in dictionary),
|
|
112
|
+
# if we consider it is spelled correctly.
|
|
113
|
+
#
|
|
114
|
+
# Example: If word is "Kitten", hypotheses are "kitten", "Kitten".
|
|
115
|
+
#
|
|
116
|
+
# @param word [String] The word to analyze
|
|
117
|
+
# @return [Array<Symbol, Array<String>>] Pair of [captype, variants]
|
|
118
|
+
def variants(word)
|
|
119
|
+
captype = guess(word)
|
|
120
|
+
|
|
121
|
+
result = case captype
|
|
122
|
+
when Type::NO
|
|
123
|
+
[word]
|
|
124
|
+
when Type::INIT
|
|
125
|
+
[word, *lower(word)]
|
|
126
|
+
when Type::HUHINIT
|
|
127
|
+
[word, *lowerfirst(word).to_a]
|
|
128
|
+
when Type::HUH
|
|
129
|
+
[word]
|
|
130
|
+
when Type::ALL
|
|
131
|
+
[word, *lower(word), *capitalize(word).to_a]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
[captype, result]
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Returns hypotheses of how the word might have been cased if it is a
|
|
138
|
+
# misspelling.
|
|
139
|
+
#
|
|
140
|
+
# Example: "DiCtionary" (HUHINIT capitalization) produces hypotheses
|
|
141
|
+
# "DiCtionary", "diCtionary", "dictionary", "Dictionary", and all of
|
|
142
|
+
# them are checked by Suggest.
|
|
143
|
+
#
|
|
144
|
+
# @param word [String] The word to analyze
|
|
145
|
+
# @return [Array<Symbol, Array<String>>] Pair of [captype, variants]
|
|
146
|
+
def corrections(word)
|
|
147
|
+
captype = guess(word)
|
|
148
|
+
|
|
149
|
+
result = case captype
|
|
150
|
+
when Type::NO
|
|
151
|
+
[word]
|
|
152
|
+
when Type::INIT
|
|
153
|
+
[word, *lower(word)]
|
|
154
|
+
when Type::HUHINIT
|
|
155
|
+
[word, *lowerfirst(word).to_a, *lower(word), *capitalize(word).to_a]
|
|
156
|
+
when Type::HUH
|
|
157
|
+
[word, *lower(word)]
|
|
158
|
+
when Type::ALL
|
|
159
|
+
[word, *lower(word), *capitalize(word).to_a]
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
[captype, result]
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Used by suggest: by known (valid) suggestion, and initial word's
|
|
166
|
+
# capitalization, produce proper suggestion capitalization.
|
|
167
|
+
#
|
|
168
|
+
# Example: If misspelling was "Kiten" (INIT capitalization),
|
|
169
|
+
# found suggestion "kitten", then this method makes it "Kitten".
|
|
170
|
+
#
|
|
171
|
+
# @param word [String] The valid suggestion word
|
|
172
|
+
# @param cap [Symbol] Original word's capitalization type
|
|
173
|
+
# @return [String] Properly capitalized suggestion
|
|
174
|
+
def coerce(word, cap)
|
|
175
|
+
case cap
|
|
176
|
+
when Type::INIT, Type::HUHINIT
|
|
177
|
+
upper(word[0]) + word[1..]
|
|
178
|
+
when Type::ALL
|
|
179
|
+
upper(word)
|
|
180
|
+
else
|
|
181
|
+
word
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Redefines upper and lower, because in Turkic languages lowercase "i"
|
|
187
|
+
# is uppercased as "İ", and uppercase "I" is downcased as "ı".
|
|
188
|
+
#
|
|
189
|
+
# Example:
|
|
190
|
+
# turkic = Kotoshu::Algorithms::Capitalization::TurkicCasing.new
|
|
191
|
+
# turkic.lower('Izmir') # => ['ızmir']
|
|
192
|
+
# turkic.upper('Izmir') # => 'IZMİR'
|
|
193
|
+
class TurkicCasing < Casing
|
|
194
|
+
U2L = {
|
|
195
|
+
'İ' => 'i',
|
|
196
|
+
'I' => 'ı'
|
|
197
|
+
}.freeze
|
|
198
|
+
|
|
199
|
+
L2U = {
|
|
200
|
+
'i' => 'İ',
|
|
201
|
+
'ı' => 'I'
|
|
202
|
+
}.freeze
|
|
203
|
+
|
|
204
|
+
# Translate uppercase Turkic characters to lowercase.
|
|
205
|
+
#
|
|
206
|
+
# @param word [String] The word to lowercase
|
|
207
|
+
# @return [Array<String>] List of lowercased variants
|
|
208
|
+
def lower(word)
|
|
209
|
+
translated = word.chars.map { |c| U2L[c] || c }.join
|
|
210
|
+
super(translated)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Translate lowercase Turkic characters to uppercase.
|
|
214
|
+
#
|
|
215
|
+
# @param word [String] The word to uppercase
|
|
216
|
+
# @return [String] Uppercased word
|
|
217
|
+
def upper(word)
|
|
218
|
+
translated = word.chars.map { |c| L2U[c] || c }.join
|
|
219
|
+
super(translated)
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Redefines lower because in German "SS" can be lowercased both as "ss" and "ß".
|
|
224
|
+
#
|
|
225
|
+
# Example:
|
|
226
|
+
# german = Kotoshu::Algorithms::Capitalization::GermanCasing.new
|
|
227
|
+
# german.lower('STRASSE') # => ['straße', 'strasse']
|
|
228
|
+
class GermanCasing < Casing
|
|
229
|
+
# Generate sharp S (ß) variants for all "ss" occurrences.
|
|
230
|
+
#
|
|
231
|
+
# @param text [String] The text to process
|
|
232
|
+
# @param start [Integer] Starting position for search
|
|
233
|
+
# @return [Array<String>] All variants with ß replacements
|
|
234
|
+
def sharp_s_variants(text, start = 0)
|
|
235
|
+
pos = text.index('ss', start)
|
|
236
|
+
return [] unless pos
|
|
237
|
+
|
|
238
|
+
replaced = text[0...pos] + 'ß' + text[(pos + 2)..]
|
|
239
|
+
[replaced,
|
|
240
|
+
*sharp_s_variants(replaced, pos + 1),
|
|
241
|
+
*sharp_s_variants(text, pos + 2)]
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Lowercase word, generating both "ss" and "ß" variants where applicable.
|
|
245
|
+
#
|
|
246
|
+
# @param word [String] The word to lowercase
|
|
247
|
+
# @return [Array<String>] List of lowercased variants
|
|
248
|
+
def lower(word)
|
|
249
|
+
lowered = super.first
|
|
250
|
+
return [lowered] unless word.include?('SS')
|
|
251
|
+
|
|
252
|
+
[*sharp_s_variants(lowered), lowered]
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Guess word's capitalization, accounting for German ß handling.
|
|
256
|
+
#
|
|
257
|
+
# In German uppercased words, ß (which is lowercase, and usually uppercased
|
|
258
|
+
# as SS) is allowed: "straße" => "STRAßE"
|
|
259
|
+
#
|
|
260
|
+
# @param word [String] The word to analyze
|
|
261
|
+
# @return [Symbol] One of the Type constants
|
|
262
|
+
def guess(word)
|
|
263
|
+
result = super
|
|
264
|
+
|
|
265
|
+
# Check if removing ß makes it ALL caps
|
|
266
|
+
if word.include?('ß')
|
|
267
|
+
word_without_ss = word.gsub('ß', '')
|
|
268
|
+
return Type::ALL if super(word_without_ss) == Type::ALL
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
result
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Algorithms
|
|
5
|
+
# Damerau-Levenshtein edit distance.
|
|
6
|
+
#
|
|
7
|
+
# Counts the minimum number of operations (insertion, deletion,
|
|
8
|
+
# substitution, or transposition of adjacent characters) needed
|
|
9
|
+
# to transform one string into another. The transposition
|
|
10
|
+
# extension distinguishes this from plain Levenshtein — a
|
|
11
|
+
# transposition (e.g. "teh" → "the") costs 1 instead of 2.
|
|
12
|
+
#
|
|
13
|
+
# Extracted from EditDistanceStrategy so that the algorithm is
|
|
14
|
+
# reusable independent of the strategy pipeline and testable
|
|
15
|
+
# without send-to-private.
|
|
16
|
+
module EditDistance
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# Compute the Damerau-Levenshtein distance between two strings.
|
|
20
|
+
#
|
|
21
|
+
# @param str1 [String] First string
|
|
22
|
+
# @param str2 [String] Second string
|
|
23
|
+
# @return [Integer] Edit distance (0 when str1 == str2)
|
|
24
|
+
def distance(str1, str2)
|
|
25
|
+
return str2.length if str1.empty?
|
|
26
|
+
return str1.length if str2.empty?
|
|
27
|
+
|
|
28
|
+
len1 = str1.length
|
|
29
|
+
len2 = str2.length
|
|
30
|
+
|
|
31
|
+
d = Array.new(len1 + 1) { Array.new(len2 + 1, 0) }
|
|
32
|
+
|
|
33
|
+
(0..len1).each { |i| d[i][0] = i }
|
|
34
|
+
(0..len2).each { |j| d[0][j] = j }
|
|
35
|
+
|
|
36
|
+
(1..len1).each do |i|
|
|
37
|
+
(1..len2).each do |j|
|
|
38
|
+
cost = str1[i - 1] == str2[j - 1] ? 0 : 1
|
|
39
|
+
|
|
40
|
+
d[i][j] = [
|
|
41
|
+
d[i - 1][j] + 1, # deletion
|
|
42
|
+
d[i][j - 1] + 1, # insertion
|
|
43
|
+
d[i - 1][j - 1] + cost # substitution
|
|
44
|
+
].min
|
|
45
|
+
|
|
46
|
+
next unless i > 1 && j > 1 &&
|
|
47
|
+
str1[i - 1] == str2[j - 2] &&
|
|
48
|
+
str1[i - 2] == str2[j - 1]
|
|
49
|
+
|
|
50
|
+
d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
d[len1][len2]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Compute edit distance with early-exit threshold.
|
|
58
|
+
#
|
|
59
|
+
# Returns nil when the true distance exceeds +threshold+. Uses
|
|
60
|
+
# the row-minimum early-termination technique: after each DP
|
|
61
|
+
# row is computed, if every cell in the row exceeds threshold,
|
|
62
|
+
# the final distance must exceed threshold (the last-row cell
|
|
63
|
+
# is bounded below by the row minimum) — so we bail without
|
|
64
|
+
# computing the rest.
|
|
65
|
+
#
|
|
66
|
+
# Combined with the length pre-filter (|len1-len2| > threshold
|
|
67
|
+
# implies distance > threshold), this prunes clearly-different
|
|
68
|
+
# pairs in O(threshold * min(len1, len2)) instead of the full
|
|
69
|
+
# O(len1 * len2) Damerau-Levenshtein DP.
|
|
70
|
+
#
|
|
71
|
+
# Note: the full matrix is kept (not 2-row DP) because the
|
|
72
|
+
# Damerau transposition step needs d[i-2][j-2] which a 2-row
|
|
73
|
+
# implementation doesn't retain. The early termination is the
|
|
74
|
+
# primary win, not memory reduction.
|
|
75
|
+
#
|
|
76
|
+
# @param str1 [String] First string
|
|
77
|
+
# @param str2 [String] Second string
|
|
78
|
+
# @param threshold [Integer] Maximum distance to report (assumed >= 0)
|
|
79
|
+
# @return [Integer, nil] Distance if ≤ threshold, else nil
|
|
80
|
+
def distance_with_threshold(str1, str2, threshold)
|
|
81
|
+
return 0 if str1 == str2
|
|
82
|
+
# The length filter must run before the empty-string shortcuts:
|
|
83
|
+
# an empty str1 gives distance str2.length, which still has to
|
|
84
|
+
# honor the nil-above-threshold contract.
|
|
85
|
+
return nil if (str1.length - str2.length).abs > threshold
|
|
86
|
+
return str2.length if str1.empty?
|
|
87
|
+
return str1.length if str2.empty?
|
|
88
|
+
|
|
89
|
+
len1 = str1.length
|
|
90
|
+
len2 = str2.length
|
|
91
|
+
|
|
92
|
+
d = Array.new(len1 + 1) { Array.new(len2 + 1, 0) }
|
|
93
|
+
(0..len1).each { |i| d[i][0] = i }
|
|
94
|
+
(0..len2).each { |j| d[0][j] = j }
|
|
95
|
+
|
|
96
|
+
(1..len1).each do |i|
|
|
97
|
+
row_min = Float::INFINITY
|
|
98
|
+
|
|
99
|
+
(1..len2).each do |j|
|
|
100
|
+
cost = str1[i - 1] == str2[j - 1] ? 0 : 1
|
|
101
|
+
|
|
102
|
+
d[i][j] = [
|
|
103
|
+
d[i - 1][j] + 1, # deletion
|
|
104
|
+
d[i][j - 1] + 1, # insertion
|
|
105
|
+
d[i - 1][j - 1] + cost # substitution
|
|
106
|
+
].min
|
|
107
|
+
|
|
108
|
+
# Damerau transposition (needs d[i-2][j-2] which is why
|
|
109
|
+
# we keep the full matrix).
|
|
110
|
+
if i > 1 && j > 1 &&
|
|
111
|
+
str1[i - 1] == str2[j - 2] &&
|
|
112
|
+
str1[i - 2] == str2[j - 1]
|
|
113
|
+
d[i][j] = [d[i][j], d[i - 2][j - 2] + 1].min
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
row_min = d[i][j] if d[i][j] < row_min
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Row minimum > threshold ⇒ final answer > threshold.
|
|
120
|
+
return nil if row_min > threshold
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
result = d[len1][len2]
|
|
124
|
+
result <= threshold ? result : nil
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Algorithms
|
|
5
|
+
module Lookup
|
|
6
|
+
# AffixForm is a hypothesis of how some word might be split into
|
|
7
|
+
# stem, suffixes and prefixes.
|
|
8
|
+
#
|
|
9
|
+
# It always has full text and stem, and may have up to two suffixes
|
|
10
|
+
# and up to two prefixes.
|
|
11
|
+
#
|
|
12
|
+
# The following is always true (considering absent affixes as empty):
|
|
13
|
+
# prefix + prefix2 + stem + suffix2 + suffix = text
|
|
14
|
+
#
|
|
15
|
+
# prefix2/suffix2 are "secondary", so if the word has only one suffix,
|
|
16
|
+
# it is stored in suffix and suffix2 is nil.
|
|
17
|
+
class AffixForm
|
|
18
|
+
# @return [String] Full word text
|
|
19
|
+
attr_reader :text
|
|
20
|
+
|
|
21
|
+
# @return [String] Stem (word without affixes)
|
|
22
|
+
attr_reader :stem
|
|
23
|
+
|
|
24
|
+
# @return [Hash, nil] Prefix affix data
|
|
25
|
+
attr_reader :prefix
|
|
26
|
+
|
|
27
|
+
# @return [Hash, nil] Suffix affix data
|
|
28
|
+
attr_reader :suffix
|
|
29
|
+
|
|
30
|
+
# @return [Hash, nil] Secondary prefix affix data
|
|
31
|
+
attr_reader :prefix2
|
|
32
|
+
|
|
33
|
+
# @return [Hash, nil] Secondary suffix affix data
|
|
34
|
+
attr_reader :suffix2
|
|
35
|
+
|
|
36
|
+
# @return [Hash, nil] Dictionary entry for stem
|
|
37
|
+
attr_reader :in_dictionary
|
|
38
|
+
|
|
39
|
+
def initialize(text, stem,
|
|
40
|
+
prefix: nil, suffix: nil,
|
|
41
|
+
prefix2: nil, suffix2: nil,
|
|
42
|
+
in_dictionary: nil)
|
|
43
|
+
@text = text
|
|
44
|
+
@stem = stem
|
|
45
|
+
@prefix = prefix
|
|
46
|
+
@suffix = suffix
|
|
47
|
+
@prefix2 = prefix2
|
|
48
|
+
@suffix2 = suffix2
|
|
49
|
+
@in_dictionary = in_dictionary
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Create a copy with changes.
|
|
53
|
+
#
|
|
54
|
+
# @param changes [Hash] Changes to apply
|
|
55
|
+
# @return [AffixForm] New affix form with changes applied
|
|
56
|
+
def replace(**changes)
|
|
57
|
+
self.class.new(
|
|
58
|
+
changes.fetch(:text, @text),
|
|
59
|
+
changes.fetch(:stem, @stem),
|
|
60
|
+
prefix: changes.fetch(:prefix, @prefix),
|
|
61
|
+
suffix: changes.fetch(:suffix, @suffix),
|
|
62
|
+
prefix2: changes.fetch(:prefix2, @prefix2),
|
|
63
|
+
suffix2: changes.fetch(:suffix2, @suffix2),
|
|
64
|
+
in_dictionary: changes.fetch(:in_dictionary, @in_dictionary)
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Check if this form has any affixes.
|
|
69
|
+
#
|
|
70
|
+
# @return [Boolean]
|
|
71
|
+
def has_affixes?
|
|
72
|
+
!@suffix.nil? || !@prefix.nil?
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Check if this is a base form (no affixes).
|
|
76
|
+
#
|
|
77
|
+
# @return [Boolean]
|
|
78
|
+
def is_base?
|
|
79
|
+
!has_affixes?
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Suffixes applied to this form, outermost first.
|
|
83
|
+
#
|
|
84
|
+
# @return [Array<Hash>] Suffix affix data
|
|
85
|
+
def suffixes
|
|
86
|
+
[@suffix, @suffix2].compact
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Flags the form carries: the dictionary entry's, plus those of the
|
|
90
|
+
# outermost prefix and suffix.
|
|
91
|
+
#
|
|
92
|
+
# Secondary affixes are deliberately excluded, matching Spylls. A
|
|
93
|
+
# secondary affix sits between the stem and the affix that was
|
|
94
|
+
# stripped last; its flags gate that stripping (see #desuffix's
|
|
95
|
+
# required_flags) rather than describing the finished word.
|
|
96
|
+
#
|
|
97
|
+
# @return [Set<String>] Combined flags
|
|
98
|
+
def flags
|
|
99
|
+
flags = @in_dictionary ? Set.new(@in_dictionary[:flags] || []) : Set.new
|
|
100
|
+
flags.merge(@prefix[:flags] || []) if @prefix
|
|
101
|
+
flags.merge(@suffix[:flags] || []) if @suffix
|
|
102
|
+
flags
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Get all affixes (excluding nils).
|
|
106
|
+
#
|
|
107
|
+
# @return [Array<Hash>] List of affix data
|
|
108
|
+
def all_affixes
|
|
109
|
+
[@prefix2, @prefix, @suffix, @suffix2].compact
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# String representation.
|
|
113
|
+
#
|
|
114
|
+
# @return [String]
|
|
115
|
+
def to_s
|
|
116
|
+
return @text if is_base?
|
|
117
|
+
|
|
118
|
+
parts = []
|
|
119
|
+
parts << @prefix.inspect if @prefix
|
|
120
|
+
parts << @prefix2.inspect if @prefix2
|
|
121
|
+
parts << @stem
|
|
122
|
+
parts << @suffix2.inspect if @suffix2
|
|
123
|
+
parts << @suffix.inspect if @suffix
|
|
124
|
+
|
|
125
|
+
"AffixForm(#{@text} = #{parts.join(' + ')})"
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
alias inspect to_s
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|