kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,530 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Readers
|
|
5
|
+
# Affix data class for Hunspell affix rules.
|
|
6
|
+
#
|
|
7
|
+
# This class represents a prefix or suffix affix rule.
|
|
8
|
+
#
|
|
9
|
+
# @attr flag [String] The flag character identifying this rule
|
|
10
|
+
# @attr crossproduct [Boolean] Whether this is a cross-product rule
|
|
11
|
+
# @attr strip [String] Characters to strip from the word
|
|
12
|
+
# @attr add [String] Characters to add to the word
|
|
13
|
+
# @attr condition [String] Condition for applying this rule
|
|
14
|
+
# @attr flags [Set<String>] Additional flags
|
|
15
|
+
#
|
|
16
|
+
# @example Creating a suffix affix
|
|
17
|
+
# Affix.new(
|
|
18
|
+
# type: :suffix,
|
|
19
|
+
# flag: 'H',
|
|
20
|
+
# crossproduct: false,
|
|
21
|
+
# strip: 'y',
|
|
22
|
+
# add: 'ieth',
|
|
23
|
+
# condition: 'y',
|
|
24
|
+
# flags: Set.new
|
|
25
|
+
# )
|
|
26
|
+
class Affix
|
|
27
|
+
attr_reader :type, :flag, :crossproduct, :strip, :add, :condition, :flags
|
|
28
|
+
|
|
29
|
+
# Create a new affix.
|
|
30
|
+
#
|
|
31
|
+
# @param type [Symbol] :prefix or :suffix
|
|
32
|
+
# @param flag [String] Flag character
|
|
33
|
+
# @param crossproduct [Boolean] Whether cross-product
|
|
34
|
+
# @param strip [String] Characters to strip
|
|
35
|
+
# @param add [String] Characters to add
|
|
36
|
+
# @param condition [String] Condition regex
|
|
37
|
+
# @param flags [Set<String>] Additional flags
|
|
38
|
+
def initialize(type:, flag:, crossproduct:, strip:, add:, condition:, flags: Set.new)
|
|
39
|
+
@type = type
|
|
40
|
+
@flag = flag
|
|
41
|
+
@crossproduct = crossproduct
|
|
42
|
+
@strip = strip
|
|
43
|
+
@add = add
|
|
44
|
+
@condition = condition
|
|
45
|
+
@flags = flags
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Check if this is a prefix.
|
|
49
|
+
#
|
|
50
|
+
# @return [Boolean] True if prefix
|
|
51
|
+
def prefix?
|
|
52
|
+
@type == :prefix
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Check if this is a suffix.
|
|
56
|
+
#
|
|
57
|
+
# @return [Boolean] True if suffix
|
|
58
|
+
def suffix?
|
|
59
|
+
@type == :suffix
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# String representation.
|
|
63
|
+
#
|
|
64
|
+
# @return [String] String representation
|
|
65
|
+
def to_s
|
|
66
|
+
type_str = prefix? ? 'Prefix' : 'Suffix'
|
|
67
|
+
"#{type_str}(#{@add}: #{@flag}#{'×' if @crossproduct}/#{@flags.to_a.join(',')}, on #{condition})"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Inspect string.
|
|
71
|
+
#
|
|
72
|
+
# @return [String] Inspect string
|
|
73
|
+
def inspect
|
|
74
|
+
to_s
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Break pattern for word splitting.
|
|
79
|
+
#
|
|
80
|
+
# @attr pattern [String] The break pattern
|
|
81
|
+
# @attr matcher [Regexp] Compiled matcher for the pattern
|
|
82
|
+
class BreakPattern
|
|
83
|
+
attr_reader :pattern, :matcher
|
|
84
|
+
|
|
85
|
+
# Create a new break pattern.
|
|
86
|
+
#
|
|
87
|
+
# @param pattern [String] The pattern string
|
|
88
|
+
def initialize(pattern)
|
|
89
|
+
@pattern = pattern
|
|
90
|
+
# Special chars like #, -, * should be escaped, but ^ and $ should be treated as pattern anchors
|
|
91
|
+
regex_pattern = Regexp.escape(pattern).gsub('\\^', '^').gsub('\\$', '$')
|
|
92
|
+
@matcher = if regex_pattern.start_with?('^') || regex_pattern.end_with?('$')
|
|
93
|
+
Regexp.new("(#{regex_pattern})")
|
|
94
|
+
else
|
|
95
|
+
Regexp.new(".(#{regex_pattern}).")
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Ignore characters for lookup/suggest.
|
|
101
|
+
#
|
|
102
|
+
# @attr chars [String] Characters to ignore
|
|
103
|
+
# @attr translation_table [Hash] Translation table for removal
|
|
104
|
+
class Ignore
|
|
105
|
+
attr_reader :chars, :translation_table
|
|
106
|
+
|
|
107
|
+
# Create a new ignore set.
|
|
108
|
+
#
|
|
109
|
+
# @param chars [String] Characters to ignore
|
|
110
|
+
def initialize(chars)
|
|
111
|
+
@chars = chars
|
|
112
|
+
# Create translation table that removes these characters
|
|
113
|
+
@translation_table = chars.each_char.each_with_index.to_h
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Remove ignored characters from string.
|
|
117
|
+
#
|
|
118
|
+
# @param str [String] Input string
|
|
119
|
+
# @return [String] String with ignored chars removed
|
|
120
|
+
def remove(str)
|
|
121
|
+
str.chars.reject { |c| @translation_table.key?(c) }.join
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Replacement pattern for suggestions.
|
|
126
|
+
#
|
|
127
|
+
# @attr pattern [String] The pattern to match
|
|
128
|
+
# @attr replacement [String] The replacement string
|
|
129
|
+
# @attr matcher [Regexp] Compiled matcher for the pattern
|
|
130
|
+
class RepPattern
|
|
131
|
+
attr_reader :pattern, :replacement, :matcher
|
|
132
|
+
|
|
133
|
+
# Create a new replacement pattern.
|
|
134
|
+
#
|
|
135
|
+
# @param pattern [String] The pattern string
|
|
136
|
+
# @param replacement [String] The replacement string
|
|
137
|
+
def initialize(pattern, replacement)
|
|
138
|
+
@pattern = pattern
|
|
139
|
+
@replacement = replacement
|
|
140
|
+
@matcher = Regexp.new(pattern)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Conversion table for ICONV/OCONV.
|
|
145
|
+
#
|
|
146
|
+
# @attr pairs [Array<Array<String>>] Array of [pattern, replacement] pairs
|
|
147
|
+
class ConvTable
|
|
148
|
+
attr_reader :pairs
|
|
149
|
+
|
|
150
|
+
# Create a new conversion table.
|
|
151
|
+
#
|
|
152
|
+
# @param pairs [Array<Array<String>>] Array of [pattern, replacement] pairs
|
|
153
|
+
def initialize(pairs)
|
|
154
|
+
@pairs = pairs
|
|
155
|
+
@table = pairs.map { |pat1, pat2| compile_row(pat1, pat2) }.sort_by { |search, _| search.length }
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Apply conversions to word.
|
|
159
|
+
#
|
|
160
|
+
# Note: Python's `re.match(string, pos)` anchors at pos, but Ruby's
|
|
161
|
+
# `Regexp#match?` searches from pos onward. We must check that the
|
|
162
|
+
# match actually begins at pos, otherwise short conversions fire on
|
|
163
|
+
# later positions in the word and produce nonsense like "ÉÉÉÉ" for
|
|
164
|
+
# "bébé".
|
|
165
|
+
#
|
|
166
|
+
# Spylls uses Python's stable `sorted(..., key=lambda r: len(r[0]))`
|
|
167
|
+
# which preserves declaration order for ties. Ruby's `sort_by` is
|
|
168
|
+
# unstable, so we add the table index as a secondary key to mirror
|
|
169
|
+
# Spylls. Without this, Nepali's ICONV reorders the `ZWNJ$ → ZWNJ`
|
|
170
|
+
# no-op rule behind `ZWNJ → U+FFF0`, causing the trailing-ZWNJ word
|
|
171
|
+
# to be normalized to U+FFF0 (and then dropped by IGNORE) so it
|
|
172
|
+
# matches the dictionary.
|
|
173
|
+
#
|
|
174
|
+
# @param word [String] Input word
|
|
175
|
+
# @return [String] Converted word
|
|
176
|
+
def call(word)
|
|
177
|
+
pos = 0
|
|
178
|
+
result = ''
|
|
179
|
+
|
|
180
|
+
while pos < word.length
|
|
181
|
+
matches = @table.each_with_index.filter_map do |(search, pattern, _), idx|
|
|
182
|
+
m = pattern.match(word, pos)
|
|
183
|
+
next unless m && m.begin(0) == pos
|
|
184
|
+
|
|
185
|
+
[search, idx]
|
|
186
|
+
end.sort_by { |s, idx| [-s.length, idx] }
|
|
187
|
+
|
|
188
|
+
if matches.any?
|
|
189
|
+
search, idx = matches.first
|
|
190
|
+
_, _, replacement = @table[idx]
|
|
191
|
+
result += replacement
|
|
192
|
+
pos += search.length
|
|
193
|
+
else
|
|
194
|
+
result += word[pos]
|
|
195
|
+
pos += 1
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
result
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
private
|
|
203
|
+
|
|
204
|
+
def compile_row(pat1, pat2)
|
|
205
|
+
pat1_clean = pat1.gsub('_', '')
|
|
206
|
+
pat1_re = pat1_clean.dup
|
|
207
|
+
pat1_re = "^#{pat1_re}" if pat1.start_with?('_')
|
|
208
|
+
pat1_re = "#{pat1_re}$" if pat1.end_with?('_')
|
|
209
|
+
|
|
210
|
+
[pat1_clean, Regexp.new(pat1_re), pat2.gsub('_', ' ')]
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Compound rule pattern.
|
|
215
|
+
#
|
|
216
|
+
# @attr text [String] The rule text
|
|
217
|
+
# @attr flags [Set<String>] Flags in this rule
|
|
218
|
+
# @attr re [Regexp] Compiled regex for full matching
|
|
219
|
+
# @attr partial_re [Regexp] Compiled regex for prefix matching
|
|
220
|
+
class CompoundRule
|
|
221
|
+
attr_reader :text, :flags, :re, :partial_re
|
|
222
|
+
|
|
223
|
+
# Create a new compound rule.
|
|
224
|
+
#
|
|
225
|
+
# @param text [String] The rule text (e.g., "A*B?CD")
|
|
226
|
+
def initialize(text)
|
|
227
|
+
@text = text
|
|
228
|
+
# Parse flags from rule text
|
|
229
|
+
if text.include?('(')
|
|
230
|
+
@flags = text.scan(/\((.+?)\)/).flatten.to_set
|
|
231
|
+
parts = text.scan(/\([^*?]+?\)[*?]?/)
|
|
232
|
+
else
|
|
233
|
+
@flags = text.gsub(/[*?]/, '').chars.to_set
|
|
234
|
+
# A flag character may itself be a regex metacharacter. Escape each
|
|
235
|
+
# flag so it compiles as a literal, keeping a trailing * or ?
|
|
236
|
+
# operator intact. The Swedish dictionary uses ) as a flag
|
|
237
|
+
# (COMPOUNDRULE )k, dictionaries pin 1829a3e).
|
|
238
|
+
parts = text.scan(/[^*?][*?]?/).map do |part|
|
|
239
|
+
"#{Regexp.escape(part[0])}#{part[1..]}"
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Full-match regex: the entire flag-combination string must match.
|
|
244
|
+
@re = Regexp.new("\\A#{parts.join}\\z")
|
|
245
|
+
|
|
246
|
+
# Partial-match regex: any prefix of the rule is accepted. Built
|
|
247
|
+
# by making each trailing part optional, from the end backwards:
|
|
248
|
+
# parts ["A","B","C"] → "A(B(C?)?)?" which matches "A", "AB", "ABC".
|
|
249
|
+
@partial_re = if parts.empty?
|
|
250
|
+
Regexp.new('\\A\\z')
|
|
251
|
+
else
|
|
252
|
+
Regexp.new("\\A#{build_partial(parts)}\\z")
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Check if flag sets fully match this rule.
|
|
257
|
+
#
|
|
258
|
+
# @param flag_sets [Array<Set<String>>] Array of flag sets
|
|
259
|
+
# @return [Boolean] True if the entire flag-combination matches
|
|
260
|
+
def fullmatch(flag_sets)
|
|
261
|
+
relevant_flags = flag_sets.map { |f| @flags.intersection(f).to_a }
|
|
262
|
+
return false if relevant_flags.empty? || relevant_flags.any?(&:empty?)
|
|
263
|
+
|
|
264
|
+
relevant_flags[0].product(*relevant_flags[1..]).any? do |fc|
|
|
265
|
+
@re.match?(fc.join)
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Check if flag sets form a valid prefix of this rule.
|
|
270
|
+
#
|
|
271
|
+
# Used during compounds_by_rules recursion to prune branches that
|
|
272
|
+
# can never lead to a full match.
|
|
273
|
+
#
|
|
274
|
+
# @param flag_sets [Array<Set<String>>] Array of flag sets
|
|
275
|
+
# @return [Boolean] True if a prefix of the rule matches
|
|
276
|
+
def partial_match(flag_sets)
|
|
277
|
+
relevant_flags = flag_sets.map { |f| @flags.intersection(f).to_a }
|
|
278
|
+
return false if relevant_flags.empty? || relevant_flags.any?(&:empty?)
|
|
279
|
+
|
|
280
|
+
relevant_flags[0].product(*relevant_flags[1..]).any? do |fc|
|
|
281
|
+
@partial_re.match?(fc.join)
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
private
|
|
286
|
+
|
|
287
|
+
# Build a regex where each trailing part becomes optional.
|
|
288
|
+
#
|
|
289
|
+
# ["A", "B", "C"] → "A(B(C?)?)?"
|
|
290
|
+
def build_partial(parts)
|
|
291
|
+
parts.reverse.reduce(nil) do |inner, part|
|
|
292
|
+
inner ? "#{part}(#{inner})?" : "#{part}?"
|
|
293
|
+
end.gsub('??', '?')
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Compound pattern for checking compound words.
|
|
298
|
+
#
|
|
299
|
+
# @attr left [String] Left side pattern
|
|
300
|
+
# @attr right [String] Right side pattern
|
|
301
|
+
# @attr replacement [String, nil] Optional replacement
|
|
302
|
+
class CompoundPattern
|
|
303
|
+
attr_reader :left, :right, :replacement, :left_stem, :left_flag, :right_stem, :right_flag,
|
|
304
|
+
:left_no_affix
|
|
305
|
+
|
|
306
|
+
# Create a new compound pattern.
|
|
307
|
+
#
|
|
308
|
+
# Mirrors Hunspell's parse_checkcpdtable: each side is the literal text
|
|
309
|
+
# up to an optional "/flag", and neither side is rewritten at parse
|
|
310
|
+
# time. In particular a "0" stays a "0" — cpdpat_check is what gives
|
|
311
|
+
# the left side's leading zero its meaning, and the right side never
|
|
312
|
+
# gets one.
|
|
313
|
+
#
|
|
314
|
+
# @param left [String] Left side pattern
|
|
315
|
+
# @param right [String] Right side pattern
|
|
316
|
+
# @param replacement [String, nil] Optional replacement
|
|
317
|
+
def initialize(left, right, replacement = nil)
|
|
318
|
+
@left = left
|
|
319
|
+
@right = right
|
|
320
|
+
@replacement = replacement
|
|
321
|
+
|
|
322
|
+
# The separator from partition('/') distinguishes "no slash" (no flag
|
|
323
|
+
# specified → nil, so the matcher skips the flag check) from a slash
|
|
324
|
+
# with an empty flag.
|
|
325
|
+
@left_stem, sep, @left_flag = left.partition('/')
|
|
326
|
+
@left_flag = nil if sep.empty?
|
|
327
|
+
# Hunspell tests only the first character (`pattern[0] == '0'`).
|
|
328
|
+
@left_no_affix = @left_stem.start_with?('0')
|
|
329
|
+
|
|
330
|
+
@right_stem, sep, @right_flag = right.partition('/')
|
|
331
|
+
@right_flag = nil if sep.empty?
|
|
332
|
+
# isSubset treats "." as a single-character wildcard. Precomputed so
|
|
333
|
+
# the overwhelmingly common literal case stays a plain start_with?.
|
|
334
|
+
@right_wildcard = @right_stem.include?('.')
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# Whether this pattern also spells out a simplified compound form.
|
|
338
|
+
#
|
|
339
|
+
# @return [Boolean] True if a non-empty replacement was given
|
|
340
|
+
def replacement?
|
|
341
|
+
!@replacement.nil? && !@replacement.empty?
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# How the two members are actually written when this pattern's
|
|
345
|
+
# replacement stands in for the junction between them.
|
|
346
|
+
#
|
|
347
|
+
# `foo` and `bar` under `CHECKCOMPOUNDPATTERN o b z` are written
|
|
348
|
+
# `fozar`, not `foobar`. Checks that read the word as the user typed it
|
|
349
|
+
# need this rather than the concatenation of the members.
|
|
350
|
+
#
|
|
351
|
+
# Without a replacement the members are simply written one after the
|
|
352
|
+
# other, so that is what this returns.
|
|
353
|
+
#
|
|
354
|
+
# @param left_text [String] Surface text of the left member
|
|
355
|
+
# @param right_text [String] Surface text of the right member
|
|
356
|
+
# @return [String] The simplified spelling
|
|
357
|
+
def surface(left_text, right_text)
|
|
358
|
+
return left_text + right_text unless replacement?
|
|
359
|
+
|
|
360
|
+
# Trim by length, not by content. Hunspell builds the members by
|
|
361
|
+
# splicing the two stems in at fixed offsets (affixmgr.cxx:1684), so
|
|
362
|
+
# undoing it is a matter of position. Matching on the text instead
|
|
363
|
+
# fails whenever a stem is not literal — a "." wildcard on the right,
|
|
364
|
+
# or a leading "0" on the left, would trim nothing or trim a real
|
|
365
|
+
# character that happened to look like the stem.
|
|
366
|
+
left_text[0...(left_text.length - @left_stem.length)].to_s +
|
|
367
|
+
@replacement + right_text[@right_stem.length..].to_s
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# Check if this pattern matches the given left and right parts.
|
|
371
|
+
#
|
|
372
|
+
# Ported from Hunspell's cpdpat_check. Both sides are matched against
|
|
373
|
+
# the members as written, not against their stems: the pattern
|
|
374
|
+
# describes the seam the reader sees.
|
|
375
|
+
#
|
|
376
|
+
# The left side has three readings, in Hunspell's order:
|
|
377
|
+
# empty means "only the flags matter"; a leading "0" means "the text
|
|
378
|
+
# running into the seam is the bare dictionary root", which a
|
|
379
|
+
# zero-width affix still satisfies; anything else is literal text the
|
|
380
|
+
# left member must end with.
|
|
381
|
+
#
|
|
382
|
+
# @param left_part [AffixForm] Left part with text, stem, flags
|
|
383
|
+
# @param right_part [AffixForm] Right part with text, flags
|
|
384
|
+
# @return [Boolean] True if matches
|
|
385
|
+
def match?(left_part, right_part)
|
|
386
|
+
return false unless right_matches?(right_part.text)
|
|
387
|
+
return false if @left_flag && !left_part.flags.include?(@left_flag)
|
|
388
|
+
return false if @right_flag && !right_part.flags.include?(@right_flag)
|
|
389
|
+
|
|
390
|
+
left_matches?(left_part)
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
private
|
|
394
|
+
|
|
395
|
+
# Hunspell matches the right side with isSubset, a shared affix-key
|
|
396
|
+
# helper in which "." stands for any one character. The behaviour is
|
|
397
|
+
# undocumented for this directive and absent from Spylls, but it is
|
|
398
|
+
# what the reference implementation does.
|
|
399
|
+
#
|
|
400
|
+
# @param text [String] Surface text of the right member
|
|
401
|
+
# @return [Boolean] Whether the right side's text condition holds
|
|
402
|
+
def right_matches?(text)
|
|
403
|
+
return text.start_with?(@right_stem) unless @right_wildcard
|
|
404
|
+
return false if text.length < @right_stem.length
|
|
405
|
+
|
|
406
|
+
@right_stem.each_char.with_index.all? { |char, i| char == '.' || char == text[i] }
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
# @param left_part [AffixForm] Left part of the junction
|
|
410
|
+
# @return [Boolean] Whether the left side's text condition holds
|
|
411
|
+
def left_matches?(left_part)
|
|
412
|
+
return true if @left_stem.empty?
|
|
413
|
+
return left_part.text.end_with?(left_part.stem) if @left_no_affix
|
|
414
|
+
|
|
415
|
+
left_part.text.end_with?(@left_stem)
|
|
416
|
+
end
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# Phonetic table for PHONE directive.
|
|
420
|
+
#
|
|
421
|
+
# Domain object wrapping the parsed PHONE rules. Rules are indexed by
|
|
422
|
+
# their first letter so the metaphone algorithm can look up candidates
|
|
423
|
+
# in O(1) per character position.
|
|
424
|
+
#
|
|
425
|
+
# @attr rules [Hash<String, Array<Rule>>] First-char → rule list
|
|
426
|
+
class PhonetTable
|
|
427
|
+
attr_reader :table, :rules
|
|
428
|
+
|
|
429
|
+
# Pattern for matching phonetic rules.
|
|
430
|
+
# Updated to support extended ASCII (Latin-1) characters like É, À, etc.
|
|
431
|
+
RULE_PATTERN = /^(?<letters>[[:alpha:]]+)(\((?<optional>[[:alpha:]]+)\))?(?<lookahead>-+)?(?<flags>[\^$<]*)(?<priority>\d)?$/
|
|
432
|
+
|
|
433
|
+
# Rule class for phonetic transformations.
|
|
434
|
+
#
|
|
435
|
+
# @attr search [Regexp] Search pattern
|
|
436
|
+
# @attr replacement [String] Replacement string
|
|
437
|
+
# @attr start [Boolean] Match at start
|
|
438
|
+
# @attr end [Boolean] Match at end
|
|
439
|
+
# @attr priority [Integer] Rule priority
|
|
440
|
+
# @attr followup [Boolean] Follow-up rule
|
|
441
|
+
Rule = Struct.new(:search, :replacement, :start, :end, :priority, :followup, keyword_init: true) do
|
|
442
|
+
# Length of the match if this rule fires at +pos+ in +word+,
|
|
443
|
+
# or nil if the rule doesn't match. The metaphone walker uses
|
|
444
|
+
# the length to advance its position past the consumed input.
|
|
445
|
+
#
|
|
446
|
+
# @param word [String] Word to match against
|
|
447
|
+
# @param pos [Integer] Position in word
|
|
448
|
+
# @return [Integer, nil] Length of matched substring, or nil
|
|
449
|
+
def match_length(word, pos)
|
|
450
|
+
return nil if start && pos.positive?
|
|
451
|
+
|
|
452
|
+
match_data = if self[:end]
|
|
453
|
+
# Anchor at end of word: strip the prefix so
|
|
454
|
+
# the regex can match without considering the
|
|
455
|
+
# preceding characters.
|
|
456
|
+
search.match(word[pos..])
|
|
457
|
+
else
|
|
458
|
+
search.match(word, pos)
|
|
459
|
+
end
|
|
460
|
+
return nil unless match_data
|
|
461
|
+
|
|
462
|
+
match_data.to_s.length
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
# Check if rule matches at position.
|
|
466
|
+
#
|
|
467
|
+
# @param word [String] Word to check
|
|
468
|
+
# @param pos [Integer] Position in word
|
|
469
|
+
# @return [Boolean] True if matches
|
|
470
|
+
def match?(word, pos)
|
|
471
|
+
!match_length(word, pos).nil?
|
|
472
|
+
end
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# Create a new phonetic table.
|
|
476
|
+
#
|
|
477
|
+
# @param table [Array<Array<String>>] Array of [pattern, replacement] pairs
|
|
478
|
+
def initialize(table)
|
|
479
|
+
@table = table
|
|
480
|
+
@rules = Hash.new { |h, k| h[k] = [] }
|
|
481
|
+
|
|
482
|
+
table.each do |search, replacement|
|
|
483
|
+
@rules[search[0]] << parse_rule(search, replacement)
|
|
484
|
+
end
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
# Whether this table contains any rules.
|
|
488
|
+
#
|
|
489
|
+
# @return [Boolean]
|
|
490
|
+
def empty?
|
|
491
|
+
@table.empty?
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# Parse a phonetic rule.
|
|
495
|
+
#
|
|
496
|
+
# @param search [String] Search pattern
|
|
497
|
+
# @param replacement [String] Replacement string ("_" means silent/empty,
|
|
498
|
+
# per aspell's phonetic convention which Hunspell inherited)
|
|
499
|
+
# @return [Rule] Parsed rule
|
|
500
|
+
def parse_rule(search, replacement)
|
|
501
|
+
match = RULE_PATTERN.match(search)
|
|
502
|
+
raise ArgumentError, "Not a proper rule: #{search.inspect}" unless match
|
|
503
|
+
|
|
504
|
+
text = match['letters'].chars
|
|
505
|
+
text << "[#{match['optional']}]" if match['optional']
|
|
506
|
+
|
|
507
|
+
if match['lookahead']
|
|
508
|
+
lookahead_len = match['lookahead'].length
|
|
509
|
+
regex = text[0...-lookahead_len].join + "(?=#{text[-lookahead_len..].join})"
|
|
510
|
+
else
|
|
511
|
+
regex = text.join
|
|
512
|
+
end
|
|
513
|
+
|
|
514
|
+
# Aspell/Hunspell phonetic convention: "_" in the replacement column
|
|
515
|
+
# means "produce nothing" (silent). Normalizing here means downstream
|
|
516
|
+
# code can append the replacement verbatim.
|
|
517
|
+
normalized_replacement = replacement == '_' ? '' : replacement
|
|
518
|
+
|
|
519
|
+
Rule.new(
|
|
520
|
+
search: Regexp.new(regex),
|
|
521
|
+
replacement: normalized_replacement,
|
|
522
|
+
start: match['flags']&.include?('^'),
|
|
523
|
+
end: match['flags']&.include?('$'),
|
|
524
|
+
priority: match['priority']&.to_i || 5,
|
|
525
|
+
followup: !match['lookahead'].nil?
|
|
526
|
+
)
|
|
527
|
+
end
|
|
528
|
+
end
|
|
529
|
+
end
|
|
530
|
+
end
|