kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Russian language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Supports multiple dialects: ru-RU, ru-BY, ru-KZ, ru-KG, ru-MD
|
|
8
|
+
#
|
|
9
|
+
# Full Hunspell integration with spell checking, POS tagging, and grammar rules
|
|
10
|
+
# specifically handling Russian Cyrillic script and case system.
|
|
11
|
+
class Russian < Language::Base
|
|
12
|
+
# Russian spell checker with Hunspell integration.
|
|
13
|
+
class SpellChecker < Components::SpellChecker
|
|
14
|
+
attr_reader :aff_path, :dic_path, :script
|
|
15
|
+
|
|
16
|
+
def initialize(aff_path:, dic_path:, script: :cyrillic, encoding: 'UTF-8')
|
|
17
|
+
@aff_path = aff_path
|
|
18
|
+
@dic_path = dic_path
|
|
19
|
+
@script = script
|
|
20
|
+
@encoding = encoding
|
|
21
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def check(word)
|
|
25
|
+
return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
|
|
26
|
+
|
|
27
|
+
first_form = @lookuper.good_forms(word).first
|
|
28
|
+
if first_form
|
|
29
|
+
{ found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
|
|
30
|
+
else
|
|
31
|
+
{ found: false, stem: nil, flags: [] }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def suggest(word, max_suggestions: 10)
|
|
36
|
+
return [] if word.nil? || word.empty?
|
|
37
|
+
|
|
38
|
+
first_form = @lookuper.good_forms(word).first
|
|
39
|
+
return [] if first_form
|
|
40
|
+
|
|
41
|
+
generate_suggestions(word, max_suggestions).take(max_suggestions)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def correct?(word)
|
|
45
|
+
check(word)[:found]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def lookuper
|
|
49
|
+
@lookuper
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def calculate_distance(a, b)
|
|
55
|
+
return a.length if b.empty?
|
|
56
|
+
return b.length if a.empty?
|
|
57
|
+
|
|
58
|
+
matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
|
|
59
|
+
(1..b.length).each { |j| matrix[0][j] = j }
|
|
60
|
+
(1..a.length).each do |i|
|
|
61
|
+
(1..b.length).each do |j|
|
|
62
|
+
cost = a[i - 1] == b[j - 1] ? 0 : 1
|
|
63
|
+
matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
matrix[a.length][b.length]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def calculate_score(original, suggestion, rank)
|
|
70
|
+
distance = calculate_distance(original, suggestion)
|
|
71
|
+
max_len = [original.length, suggestion.length].max
|
|
72
|
+
distance_score = 1.0 - (distance.to_f / max_len)
|
|
73
|
+
rank_penalty = rank * 0.05
|
|
74
|
+
[distance_score - rank_penalty, 0.0].max
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def generate_suggestions(word, _max_suggestions)
|
|
78
|
+
variations = []
|
|
79
|
+
|
|
80
|
+
# Russian character substitutions (common Cyrillic errors)
|
|
81
|
+
cyrillic_substitutions = {
|
|
82
|
+
'а' => %w[о и е я],
|
|
83
|
+
'о' => %w[а е и],
|
|
84
|
+
'е' => %w[и э а],
|
|
85
|
+
'и' => %w[е е],
|
|
86
|
+
'п' => %w[т к],
|
|
87
|
+
'т' => %w[п д],
|
|
88
|
+
'к' => %w[г х],
|
|
89
|
+
'н' => %w[т п],
|
|
90
|
+
'с' => %w[з ш],
|
|
91
|
+
'ш' => %w[с щ],
|
|
92
|
+
'щ' => %w[ш],
|
|
93
|
+
'б' => %w[п в],
|
|
94
|
+
'в' => %w[б ф],
|
|
95
|
+
'ф' => %w[в в],
|
|
96
|
+
'д' => %w[т],
|
|
97
|
+
'г' => %w[к х],
|
|
98
|
+
'х' => %w[г к],
|
|
99
|
+
'я' => %w[а е],
|
|
100
|
+
'ю' => %w[у],
|
|
101
|
+
'ё' => %w[е],
|
|
102
|
+
'ж' => %w[з ш],
|
|
103
|
+
'з' => %w[с ж],
|
|
104
|
+
'ь' => %w[ъ],
|
|
105
|
+
'ъ' => %w[ь],
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
word.chars.each_with_index do |char, i|
|
|
109
|
+
next unless cyrillic_substitutions.key?(char.downcase)
|
|
110
|
+
|
|
111
|
+
cyrillic_substitutions[char.downcase].each do |sub|
|
|
112
|
+
substituted = word.dup
|
|
113
|
+
substituted[i] = sub
|
|
114
|
+
variations << substituted if @lookuper.good_forms(substituted).first
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Doubled and deleted letters
|
|
119
|
+
word.chars.each_with_index do |char, i|
|
|
120
|
+
next if i == 0
|
|
121
|
+
|
|
122
|
+
doubled = word.dup
|
|
123
|
+
doubled.insert(i, char)
|
|
124
|
+
variations << doubled if @lookuper.good_forms(doubled).first
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
(0...word.length).each do |i|
|
|
128
|
+
deleted = word.dup
|
|
129
|
+
deleted.slice!(i)
|
|
130
|
+
next if deleted.empty?
|
|
131
|
+
|
|
132
|
+
variations << deleted if @lookuper.good_forms(deleted).first
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
variations.uniq!
|
|
136
|
+
variations.map do |suggestion|
|
|
137
|
+
{ word: suggestion, distance: calculate_distance(word, suggestion),
|
|
138
|
+
score: calculate_score(word, suggestion, 0) }
|
|
139
|
+
end.sort_by { |s| s[:distance] }
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Russian tokenizer with abbreviation handling.
|
|
144
|
+
class Tokenizer < Language::Tokenizer::RussianTokenizer
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Russian POS tagger.
|
|
148
|
+
class POSTagger < Components::PosTagger
|
|
149
|
+
FLAG_TO_POS = {
|
|
150
|
+
'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
|
|
151
|
+
'S' => 'NOUN', 'Sub' => 'NOUN',
|
|
152
|
+
'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
|
|
153
|
+
'VBP' => 'VERB', 'VBZ' => 'VERB',
|
|
154
|
+
'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
|
|
155
|
+
'Adj' => 'ADJ',
|
|
156
|
+
'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
|
|
157
|
+
'Adv' => 'ADV',
|
|
158
|
+
'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
|
|
159
|
+
'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
|
|
160
|
+
'WP' => 'PRON', 'WP$' => 'PRON_POSS',
|
|
161
|
+
'Pro' => 'PRON',
|
|
162
|
+
'I' => 'PREP', 'IN' => 'PREP',
|
|
163
|
+
'Präp' => 'PREP',
|
|
164
|
+
'C' => 'CONJ', 'CC' => 'CONJ',
|
|
165
|
+
'Conj' => 'CONJ',
|
|
166
|
+
'U' => 'PART', 'RP' => 'PART',
|
|
167
|
+
'Pt' => 'PART',
|
|
168
|
+
'INTJ' => 'INTJ', 'UH' => 'INTJ',
|
|
169
|
+
'Int' => 'INTJ',
|
|
170
|
+
'CD' => 'NUM',
|
|
171
|
+
'FW' => 'X',
|
|
172
|
+
'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
|
|
173
|
+
'?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
|
|
174
|
+
}.freeze
|
|
175
|
+
|
|
176
|
+
attr_reader :aff_path, :dic_path, :script
|
|
177
|
+
|
|
178
|
+
def initialize(aff_path:, dic_path:, script: :cyrillic, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
|
|
179
|
+
@aff_path = aff_path
|
|
180
|
+
@dic_path = dic_path
|
|
181
|
+
@script = script
|
|
182
|
+
@encoding = encoding
|
|
183
|
+
@flag_mapping = flag_mapping
|
|
184
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
185
|
+
@lookup_cache = {}
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def tag(tokens)
|
|
189
|
+
return [] if tokens.nil? || tokens.empty?
|
|
190
|
+
|
|
191
|
+
tokens.map do |token|
|
|
192
|
+
word = token[:token]
|
|
193
|
+
if word.nil? || word.empty?
|
|
194
|
+
token.merge(pos_tag: nil, lemma: nil)
|
|
195
|
+
else
|
|
196
|
+
lookup_result = lookup_with_pos(word)
|
|
197
|
+
token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def flag_mapping
|
|
203
|
+
@flag_mapping
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def flag_mapping=(mapping)
|
|
207
|
+
@flag_mapping = mapping
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def clear_cache
|
|
211
|
+
@lookup_cache.clear
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def cache_size
|
|
215
|
+
@lookup_cache.size
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
private
|
|
219
|
+
|
|
220
|
+
def lookup_with_pos(word)
|
|
221
|
+
return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
|
|
222
|
+
return @lookup_cache[word] if @lookup_cache.key?(word)
|
|
223
|
+
|
|
224
|
+
first_form = @lookuper.good_forms(word).first
|
|
225
|
+
pos_tag = derive_pos_tag(first_form)
|
|
226
|
+
cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
|
|
227
|
+
@lookup_cache[word] = cache_result
|
|
228
|
+
cache_result
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def derive_pos_tag(result)
|
|
232
|
+
return nil unless result
|
|
233
|
+
|
|
234
|
+
flags = result.flags&.to_a || []
|
|
235
|
+
return guess_pos_from_affix(result) if flags.empty?
|
|
236
|
+
|
|
237
|
+
flags.each do |flag|
|
|
238
|
+
pos_tag = flag_to_pos(flag)
|
|
239
|
+
return pos_tag if pos_tag
|
|
240
|
+
end
|
|
241
|
+
guess_pos_from_affix(result)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def flag_to_pos(flag)
|
|
245
|
+
return @flag_mapping[flag] if @flag_mapping.key?(flag)
|
|
246
|
+
|
|
247
|
+
first_char = flag[0]
|
|
248
|
+
@flag_mapping[first_char]
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def guess_pos_from_affix(result)
|
|
252
|
+
suffix = result.suffix
|
|
253
|
+
return guess_pos_from_suffix(suffix) if suffix
|
|
254
|
+
|
|
255
|
+
nil
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def guess_pos_from_suffix(suffix)
|
|
259
|
+
# Russian suffix patterns
|
|
260
|
+
return 'VERB' if suffix.match?(/^(ть|ти|чь|л|ла|ло|ли|ют|ют|ешь|ишь|им|ите|ат|ят)$/)
|
|
261
|
+
return 'ADV' if suffix.match?(/^(о|е|и)$/)
|
|
262
|
+
return 'NOUN' if suffix.match?(/^(ость|ение|ание|ка|ник|чик|щик|ство|тель|ение|ство)$/)
|
|
263
|
+
return 'ADJ' if suffix.match?(/^(ый|ий|ой|ое|ая|ое|ые|их|ем|им|ом|ого|ому)$/)
|
|
264
|
+
|
|
265
|
+
nil
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Russian grammar rules module.
|
|
270
|
+
module GrammarRules
|
|
271
|
+
class Rule
|
|
272
|
+
attr_reader :id, :name, :description
|
|
273
|
+
|
|
274
|
+
def initialize(id, name, description)
|
|
275
|
+
@id = id
|
|
276
|
+
@name = name
|
|
277
|
+
@description = description
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def check(tokens)
|
|
281
|
+
raise NotImplementedError, "#{self.class} must implement #check"
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Rule: Verbal aspect consistency
|
|
286
|
+
class VerbalAspectRule < Rule
|
|
287
|
+
IMPERFECTIVE_SUFFIXES = %w[ать ять].freeze
|
|
288
|
+
PERFECTIVE_SUFFIXES = %w[ить по].freeze
|
|
289
|
+
|
|
290
|
+
def initialize
|
|
291
|
+
super('RU_VERBAL_ASPECT', 'Verbal Aspect', 'Russian verbs should use consistent aspect (imperfective/perfective).')
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def check(_tokens)
|
|
295
|
+
# Simplified implementation
|
|
296
|
+
[]
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Rule: Case agreement
|
|
301
|
+
class CaseAgreementRule < Rule
|
|
302
|
+
def initialize
|
|
303
|
+
super('RU_CASE_AGREEMENT', 'Case Agreement', 'Nouns, adjectives, and verbs must agree in case.')
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def check(_tokens)
|
|
307
|
+
# Simplified implementation
|
|
308
|
+
[]
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
class RuleRegistry
|
|
313
|
+
class << self
|
|
314
|
+
def default_rules
|
|
315
|
+
[VerbalAspectRule.new, CaseAgreementRule.new]
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def get_rule(id)
|
|
319
|
+
default_rules.find { |rule| rule.id == id }
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Registration
|
|
326
|
+
register "ru"
|
|
327
|
+
register "ru-RU"
|
|
328
|
+
register "ru-BY"
|
|
329
|
+
register "ru-KZ"
|
|
330
|
+
register "ru-KG"
|
|
331
|
+
register "ru-MD"
|
|
332
|
+
|
|
333
|
+
HUNSPELL_DICTIONARIES = {
|
|
334
|
+
'ru-RU' => {
|
|
335
|
+
aff: 'spec/integrational/fixtures/ru_RU.aff',
|
|
336
|
+
dic: 'spec/integrational/fixtures/ru_RU.dic'
|
|
337
|
+
}
|
|
338
|
+
}.freeze
|
|
339
|
+
|
|
340
|
+
VARIANT_NAMES = {
|
|
341
|
+
'RU' => 'Russian',
|
|
342
|
+
'BY' => 'Belarusian',
|
|
343
|
+
'KZ' => 'Kazakh',
|
|
344
|
+
'KG' => 'Kyrgyz',
|
|
345
|
+
'MD' => 'Moldovan'
|
|
346
|
+
}.freeze
|
|
347
|
+
|
|
348
|
+
def initialize(code: "ru", name: "Russian", variant: nil)
|
|
349
|
+
variant ||= extract_region_code(code)
|
|
350
|
+
super
|
|
351
|
+
@hunspell_paths = resolve_hunspell_paths(code)
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def description
|
|
355
|
+
return name unless variant
|
|
356
|
+
|
|
357
|
+
variant_name = VARIANT_NAMES[variant] || variant
|
|
358
|
+
"#{name} (#{variant_name})"
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def tokenizer
|
|
362
|
+
@tokenizer ||= Tokenizer.new
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
def normalizer
|
|
366
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def dictionary_class
|
|
370
|
+
Dictionary::UnixWords
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def default_dictionary_paths
|
|
374
|
+
case code
|
|
375
|
+
when "ru-RU"
|
|
376
|
+
["/usr/share/dict/russian"]
|
|
377
|
+
else
|
|
378
|
+
["/usr/share/dict/words"]
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def script_type
|
|
383
|
+
:cyrillic
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
def create_spell_checker
|
|
387
|
+
SpellChecker.new(
|
|
388
|
+
aff_path: @hunspell_paths[:aff],
|
|
389
|
+
dic_path: @hunspell_paths[:dic],
|
|
390
|
+
script: :cyrillic
|
|
391
|
+
)
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def create_tokenizer
|
|
395
|
+
Tokenizer.new
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def create_pos_tagger
|
|
399
|
+
POSTagger.new(
|
|
400
|
+
aff_path: @hunspell_paths[:aff],
|
|
401
|
+
dic_path: @hunspell_paths[:dic],
|
|
402
|
+
script: :cyrillic,
|
|
403
|
+
flag_mapping: POSTagger::FLAG_TO_POS
|
|
404
|
+
)
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
private
|
|
408
|
+
|
|
409
|
+
def extract_region_code(code)
|
|
410
|
+
return nil unless code.include?("-")
|
|
411
|
+
|
|
412
|
+
code.split("-", 2).last.upcase
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
def resolve_hunspell_paths(code)
|
|
416
|
+
HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['ru-RU']
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
end
|
|
420
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Slovak language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Slovak diacritics (á ä č ď é í ľ ĺ ň ó ô ŕ š ť
|
|
9
|
+
# ú ý ž) are dead keys over the US QWERTY grid
|
|
10
|
+
# (Keyboard::Layouts::Latin::Slovak), the Polish programmers'
|
|
11
|
+
# pattern the eval harness also models.
|
|
12
|
+
class Slovak < LatinBase
|
|
13
|
+
register "sk"
|
|
14
|
+
register "sk-SK"
|
|
15
|
+
|
|
16
|
+
def initialize(code: "sk", name: "Slovak")
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def default_dictionary_paths
|
|
21
|
+
["/usr/share/dict/slovak"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Slovenian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. The Slovenian letters č š ž (plus the Croatian
|
|
9
|
+
# đ ć the grid also carries) are real keys on the South-Slavic
|
|
10
|
+
# QWERTZ physical layout (Keyboard::Layouts::Slovenian), shared
|
|
11
|
+
# with Croatian.
|
|
12
|
+
class Slovenian < LatinBase
|
|
13
|
+
register "sl"
|
|
14
|
+
register "sl-SI"
|
|
15
|
+
|
|
16
|
+
def initialize(code: "sl", name: "Slovenian")
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def default_dictionary_paths
|
|
21
|
+
["/usr/share/dict/slovenian"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Serbian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Cyrillic composition like Bulgarian/Ukrainian: Serbian is
|
|
8
|
+
# written in both scripts, but Cyrillic is the constitutional
|
|
9
|
+
# primary and the script the staged dictionary carries — the
|
|
10
|
+
# module registers sr for the Cyrillic grid
|
|
11
|
+
# (Keyboard::Layouts::SerbianCyrillic); Serbian Latin (sr-Latn)
|
|
12
|
+
# has its own module over the distinct staged Latin dictionary
|
|
13
|
+
# (plan 110, Kotoshu::Languages::SerbianLatin). The Serbian
|
|
14
|
+
# letters љ њ ђ ћ џ ј are real keys on that grid.
|
|
15
|
+
class Serbian < LatinBase
|
|
16
|
+
register "sr"
|
|
17
|
+
register "sr-RS"
|
|
18
|
+
|
|
19
|
+
def initialize(code: "sr", name: "Serbian")
|
|
20
|
+
super
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Cyrillic-script tokenizer.
|
|
24
|
+
#
|
|
25
|
+
# @return [Language::Tokenizer::CyrillicTokenizer]
|
|
26
|
+
def tokenizer
|
|
27
|
+
@tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def script_type
|
|
31
|
+
:cyrillic
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def default_dictionary_paths
|
|
35
|
+
["/usr/share/dict/serbian"]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Serbian Latin implementation (plan 110).
|
|
6
|
+
#
|
|
7
|
+
# The Latin script variant of Serbian, the last named gap in the
|
|
8
|
+
# corpus after plan 107 opened the staged manifest. The data
|
|
9
|
+
# decided the wiring: the dictionaries manifest stages a distinct
|
|
10
|
+
# sr-Latn dictionary (LibreOffice, wooorm/dictionaries), so
|
|
11
|
+
# sr-Latn resolves its own download rather than riding the
|
|
12
|
+
# Cyrillic sr dictionary — and because sr and sr-Latn differ by
|
|
13
|
+
# script, not vocabulary, a script-subtag code must survive
|
|
14
|
+
# ResourceManager language normalization (plan 110) for the
|
|
15
|
+
# staged Latin files to be reachable at all.
|
|
16
|
+
#
|
|
17
|
+
# Thin composition like its Croatian sibling: shared Latin
|
|
18
|
+
# tokenizer (\p{Latin} covers š đ č ć ž), base normalizer,
|
|
19
|
+
# registry entry. The five Latin digraph letters are real keys on
|
|
20
|
+
# the South-Slavic QWERTZ grid
|
|
21
|
+
# (Keyboard::Layouts::SerbianLatin, the physical arrangement the
|
|
22
|
+
# hr and sl modules share).
|
|
23
|
+
class SerbianLatin < LatinBase
|
|
24
|
+
register "sr-Latn"
|
|
25
|
+
|
|
26
|
+
def initialize(code: "sr-Latn", name: "Serbian (Latin)")
|
|
27
|
+
super
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def default_dictionary_paths
|
|
31
|
+
["/usr/share/dict/serbian"]
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Swedish language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Swedish å ä ö are real keys on the Nordic physical layout (Keyboard::Layouts::Latin::Swedish).
|
|
9
|
+
class Swedish < LatinBase
|
|
10
|
+
register "sv"
|
|
11
|
+
register "sv-SE"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "sv", name: "Swedish")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/swedish"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Turkish language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition plus one piece of real care: Turkish case
|
|
8
|
+
# folding. Turkish has a separate dotted/dotless i pair (I/ı and
|
|
9
|
+
# i/İ), so a Unicode-default downcase maps uppercase words onto
|
|
10
|
+
# letter sequences no Turkish dictionary contains. The Turkish
|
|
11
|
+
# normalizer uses Ruby's :turkic case mapping (I -> ı, İ -> i),
|
|
12
|
+
# so İZMİR looks up as izmir and ISTANBUL as ıstanbul.
|
|
13
|
+
#
|
|
14
|
+
# The ı ğ ü ş i ö ç letters are real keys on the Turkish-Q
|
|
15
|
+
# physical layout (Keyboard::Layouts::TurkishQ).
|
|
16
|
+
class Turkish < LatinBase
|
|
17
|
+
register "tr"
|
|
18
|
+
register "tr-TR"
|
|
19
|
+
|
|
20
|
+
def initialize(code: "tr", name: "Turkish")
|
|
21
|
+
super
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Turkish-aware normalizer (dotless-i case folding).
|
|
25
|
+
#
|
|
26
|
+
# @return [Language::Normalizer::Turkish]
|
|
27
|
+
def normalizer
|
|
28
|
+
@normalizer ||= Language::Normalizer::Turkish.new
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def default_dictionary_paths
|
|
32
|
+
["/usr/share/dict/turkish"]
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Ukrainian language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Cyrillic composition: the Cyrillic tokenizer keeps
|
|
8
|
+
# Ukrainian-script words (і ї є ґ included; the apostrophe stays a
|
|
9
|
+
# word character for names like Мар'яна). The base normalizer
|
|
10
|
+
# suffices — Ukrainian case folding has no special pairs.
|
|
11
|
+
#
|
|
12
|
+
# Keyboard proximity uses the Ukrainian JCUKEN grid
|
|
13
|
+
# (Keyboard::Layouts::Ukrainian).
|
|
14
|
+
class Ukrainian < LatinBase
|
|
15
|
+
register "uk"
|
|
16
|
+
register "uk-UA"
|
|
17
|
+
|
|
18
|
+
def initialize(code: "uk", name: "Ukrainian")
|
|
19
|
+
super
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Cyrillic-script tokenizer.
|
|
23
|
+
#
|
|
24
|
+
# @return [Language::Tokenizer::CyrillicTokenizer]
|
|
25
|
+
def tokenizer
|
|
26
|
+
@tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def script_type
|
|
30
|
+
:cyrillic
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def default_dictionary_paths
|
|
34
|
+
["/usr/share/dict/ukrainian"]
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Vietnamese language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Vietnamese tone and vowel marks are dead-key sequences over ASCII letters (VIQR-style typing).
|
|
9
|
+
class Vietnamese < LatinBase
|
|
10
|
+
register "vi"
|
|
11
|
+
register "vi-VN"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "vi", name: "Vietnamese")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/vietnamese"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|