kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,841 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Algorithms
|
|
5
|
+
# Main "is this word correct?" algorithm implementation.
|
|
6
|
+
#
|
|
7
|
+
# Ported from Spylls (Python) lookup.py
|
|
8
|
+
#
|
|
9
|
+
# On a bird's-eye view level:
|
|
10
|
+
# * Word correctness check is an attempt to analyze word form
|
|
11
|
+
# (maybe it has this suffix? maybe it has this prefix? maybe it
|
|
12
|
+
# consists of several words?)
|
|
13
|
+
# * The word is considered correct if at least one form is found that
|
|
14
|
+
# has valid suffixes/prefixes from .aff file and valid stem from
|
|
15
|
+
# .dic file, and they are all compatible with each other.
|
|
16
|
+
#
|
|
17
|
+
# To follow algorithm details, start reading from Lookup.call method.
|
|
18
|
+
module Lookup
|
|
19
|
+
autoload :CompoundChecks, "kotoshu/algorithms/lookup/compound_checks"
|
|
20
|
+
autoload :AffixForm, "kotoshu/algorithms/lookup/affix_form"
|
|
21
|
+
autoload :CompoundForm, "kotoshu/algorithms/lookup/compound_form"
|
|
22
|
+
|
|
23
|
+
NUMBER_REGEXP = /^\d+(\.\d+)?$/
|
|
24
|
+
|
|
25
|
+
# Position of word part in compound word.
|
|
26
|
+
#
|
|
27
|
+
# Used when checking whether a word could be part of a compound
|
|
28
|
+
# (specifically its begin/middle/end).
|
|
29
|
+
module CompoundPos
|
|
30
|
+
BEGIN_POS = :begin
|
|
31
|
+
MIDDLE = :middle
|
|
32
|
+
END_POS = :end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Main word correctness lookup class.
|
|
36
|
+
#
|
|
37
|
+
# Typically, you would not use this directly.
|
|
38
|
+
#
|
|
39
|
+
# Example:
|
|
40
|
+
# dictionary = Kotoshu::Dictionary.load('en_US')
|
|
41
|
+
# lookuper = dictionary.lookuper
|
|
42
|
+
#
|
|
43
|
+
# lookuper.call('spylls') # => false
|
|
44
|
+
# lookuper.call('spells') # => true
|
|
45
|
+
#
|
|
46
|
+
# lookuper.good_forms('spells') do |form|
|
|
47
|
+
# puts form
|
|
48
|
+
# end
|
|
49
|
+
# # AffixForm(spells = spells)
|
|
50
|
+
# # AffixForm(spells = spell + Suffix(s: S×, on [[^sxzhy]]$))
|
|
51
|
+
class Lookuper
|
|
52
|
+
include CompoundChecks
|
|
53
|
+
|
|
54
|
+
# @return [Hash] Aff data structure (from aff file)
|
|
55
|
+
attr_reader :aff
|
|
56
|
+
|
|
57
|
+
# @return [Hash] Dic data structure (from dic file)
|
|
58
|
+
attr_reader :dic
|
|
59
|
+
|
|
60
|
+
def initialize(aff, dic)
|
|
61
|
+
@aff = aff
|
|
62
|
+
@dic = dic
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The outermost word correctness check.
|
|
66
|
+
#
|
|
67
|
+
# Basically, prepares word for check (converting/removing chars), and
|
|
68
|
+
# then checks whether any good word form can be produced with good_forms.
|
|
69
|
+
# If there is none, also tries to break word by break-points.
|
|
70
|
+
#
|
|
71
|
+
# @param word [String] Word to check
|
|
72
|
+
# @param capitalization [Boolean] If false, check only exact capitalization
|
|
73
|
+
# @param allow_nosuggest [Boolean] If false, don't consider NOSUGGEST words as correct
|
|
74
|
+
# @return [Boolean] Whether word is correct
|
|
75
|
+
def call(word, capitalization: true, allow_nosuggest: true)
|
|
76
|
+
# Check if word is correct
|
|
77
|
+
is_correct = ->(w) do
|
|
78
|
+
good_forms(w, capitalization: capitalization, allow_nosuggest: allow_nosuggest).any?
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# If all entries matching the word have FORBIDDENWORD flag, word can't be correct
|
|
82
|
+
if @aff[:FORBIDDENWORD] && @dic[:has_flag]&.call(word, @aff[:FORBIDDENWORD], for_all: true)
|
|
83
|
+
return false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Convert word with ICONV table
|
|
87
|
+
word_to_check = @aff[:ICONV] ? @aff[:ICONV].call(word) : word
|
|
88
|
+
|
|
89
|
+
# Remove ignored characters
|
|
90
|
+
if @aff[:IGNORE]
|
|
91
|
+
ignore_chars = @aff[:IGNORE]
|
|
92
|
+
word_to_check = word_to_check.chars.reject { |c| ignore_chars.include?(c) }.join
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Numbers are always good
|
|
96
|
+
return true if NUMBER_REGEXP.match?(word_to_check)
|
|
97
|
+
|
|
98
|
+
# Try breaking word by break patterns
|
|
99
|
+
break_word(word_to_check).each do |parts|
|
|
100
|
+
if parts.all? { |part| part.empty? || is_correct.call(part) }
|
|
101
|
+
return true
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
false
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Recursively produce all possible lists of word breaking by break
|
|
109
|
+
# patterns (like dashes).
|
|
110
|
+
#
|
|
111
|
+
# Example: "pre-processed-meat" would produce:
|
|
112
|
+
# ["pre-processed-meat"]
|
|
113
|
+
# ["pre", "processed-meat"]
|
|
114
|
+
# ["pre", "processed", "meat"]
|
|
115
|
+
# ["pre-processed", "meat"]
|
|
116
|
+
#
|
|
117
|
+
# This is necessary because dictionary might contain "pre-processed"
|
|
118
|
+
# as a separate entry.
|
|
119
|
+
#
|
|
120
|
+
# @param text [String] Text to break
|
|
121
|
+
# @param depth [Integer] Current recursion depth
|
|
122
|
+
# @yield [Array<String>] Each possible breaking
|
|
123
|
+
# @return [Enumerator] If no block given
|
|
124
|
+
def break_word(text, depth = 0)
|
|
125
|
+
return enum_for(:break_word, text, depth) unless block_given?
|
|
126
|
+
return if depth > 10
|
|
127
|
+
|
|
128
|
+
# Return whole text as first option
|
|
129
|
+
yield [text]
|
|
130
|
+
|
|
131
|
+
break_patterns = @aff[:BREAK] || []
|
|
132
|
+
break_patterns.each do |pattern|
|
|
133
|
+
str = text.to_s
|
|
134
|
+
pos = 0
|
|
135
|
+
|
|
136
|
+
while (match_data = pattern[:matcher].match(str, pos))
|
|
137
|
+
start = str[0...match_data.begin(1)]
|
|
138
|
+
rest = str[match_data.end(1)..]
|
|
139
|
+
|
|
140
|
+
break_word(rest, depth + 1) do |breaking|
|
|
141
|
+
yield [start, *breaking]
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
pos = match_data.end(0)
|
|
145
|
+
break if pos >= str.length
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The main producer of correct word forms.
|
|
151
|
+
#
|
|
152
|
+
# Produces all ways the proposed string might correspond to dictionary/
|
|
153
|
+
# affixes. If there is at least one, the word is correctly spelled.
|
|
154
|
+
#
|
|
155
|
+
# Example:
|
|
156
|
+
# lookuper.good_forms('building') do |form|
|
|
157
|
+
# puts form
|
|
158
|
+
# end
|
|
159
|
+
# # AffixForm(building = building) # noun
|
|
160
|
+
# # AffixForm(building = build + Suffix(ing: G×, on [[^e]]$)) # verb
|
|
161
|
+
#
|
|
162
|
+
# @param word [String] Word to check
|
|
163
|
+
# @param capitalization [Boolean] If false, use only exact capitalization
|
|
164
|
+
# @param allow_nosuggest [Boolean] If false, exclude NOSUGGEST words
|
|
165
|
+
# @param affix_forms [Boolean] If false, only return compound forms
|
|
166
|
+
# @param compound_forms [Boolean] If false, only return affix forms
|
|
167
|
+
# @yield [AffixForm, CompoundForm] Each valid word form
|
|
168
|
+
def good_forms(word,
|
|
169
|
+
capitalization: true,
|
|
170
|
+
allow_nosuggest: true,
|
|
171
|
+
affix_forms: true,
|
|
172
|
+
compound_forms: true, &block)
|
|
173
|
+
unless block
|
|
174
|
+
return enum_for(:good_forms, word,
|
|
175
|
+
capitalization: capitalization,
|
|
176
|
+
allow_nosuggest: allow_nosuggest,
|
|
177
|
+
affix_forms: affix_forms,
|
|
178
|
+
compound_forms: compound_forms)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Get capitalization variants
|
|
182
|
+
if capitalization
|
|
183
|
+
captype, variants = @aff[:casing].variants(word)
|
|
184
|
+
else
|
|
185
|
+
captype = @aff[:casing].guess(word)
|
|
186
|
+
variants = [word]
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Check each variant
|
|
190
|
+
variants.each do |variant|
|
|
191
|
+
if affix_forms
|
|
192
|
+
affix_forms_internal(variant, captype: captype, allow_nosuggest: allow_nosuggest) do |form|
|
|
193
|
+
# Special German ß handling
|
|
194
|
+
if @aff[:CHECKSHARPS] && @aff[:KEEPCASE]
|
|
195
|
+
stem = form.in_dictionary ? form.in_dictionary[:stem] : form.stem
|
|
196
|
+
if stem.include?('ß') &&
|
|
197
|
+
captype == Capitalization::Type::ALL &&
|
|
198
|
+
word.include?('ß') &&
|
|
199
|
+
form.flags.include?(@aff[:KEEPCASE])
|
|
200
|
+
next
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
yield form
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
if compound_forms
|
|
209
|
+
compound_forms_internal(variant, captype: captype, allow_nosuggest: allow_nosuggest, &block)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Check if the word is correct without yielding forms.
|
|
215
|
+
#
|
|
216
|
+
# Convenience method for simple correctness checks.
|
|
217
|
+
#
|
|
218
|
+
# @param word [String] Word to check
|
|
219
|
+
# @param capitalization [Boolean] Check capitalization variants
|
|
220
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
221
|
+
# @param affix_forms [Boolean] Check affix forms
|
|
222
|
+
# @param compound_forms [Boolean] Check compound forms
|
|
223
|
+
# @return [Boolean] Whether word is correct
|
|
224
|
+
def correct?(word,
|
|
225
|
+
capitalization: true,
|
|
226
|
+
allow_nosuggest: true,
|
|
227
|
+
affix_forms: true,
|
|
228
|
+
compound_forms: true)
|
|
229
|
+
good_forms(word,
|
|
230
|
+
capitalization: capitalization,
|
|
231
|
+
allow_nosuggest: allow_nosuggest,
|
|
232
|
+
affix_forms: affix_forms,
|
|
233
|
+
compound_forms: compound_forms).any?
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Alias for better readability
|
|
237
|
+
alias is_correct? correct?
|
|
238
|
+
|
|
239
|
+
private
|
|
240
|
+
|
|
241
|
+
# Internal affix forms generator.
|
|
242
|
+
#
|
|
243
|
+
# @param word [String] Word to process
|
|
244
|
+
# @param captype [Symbol] Capitalization type
|
|
245
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
246
|
+
# @param with_forbidden [Boolean] When true, also yield forms whose
|
|
247
|
+
# homonym carries FORBIDDENWORD (used by compound_forms_internal to
|
|
248
|
+
# detect forbidden base words). When false (default), forbidden
|
|
249
|
+
# homonyms are skipped per-homonym — not by aborting the whole
|
|
250
|
+
# search, so a non-forbidden homonym of the same stem can still
|
|
251
|
+
# match.
|
|
252
|
+
# @param compoundpos [Symbol, nil] When called from compounds_by_flags,
|
|
253
|
+
# the position in compound (BEGIN_POS / MIDDLE / END_POS). Drives
|
|
254
|
+
# suffix/prefix allowance in produce_affix_forms and compound
|
|
255
|
+
# position flag checks in is_good_form.
|
|
256
|
+
# @param prefix_flags [Array<String>] Flags a prefix must carry to be
|
|
257
|
+
# valid inside a compound (COMPOUNDPERMITFLAG).
|
|
258
|
+
# @param suffix_flags [Array<String>] Flags a suffix must carry to be
|
|
259
|
+
# valid inside a compound (COMPOUNDPERMITFLAG).
|
|
260
|
+
# @param forbidden_flags [Array<String>] Flags that disqualify an
|
|
261
|
+
# affix inside a compound (COMPOUNDFORBIDFLAG).
|
|
262
|
+
# @yield [AffixForm] Each valid affix form
|
|
263
|
+
def affix_forms_internal(word, captype:, allow_nosuggest:, with_forbidden: false,
|
|
264
|
+
compoundpos: nil, prefix_flags: [], suffix_flags: [],
|
|
265
|
+
forbidden_flags: [])
|
|
266
|
+
unless block_given?
|
|
267
|
+
return enum_for(:affix_forms_internal, word,
|
|
268
|
+
captype: captype,
|
|
269
|
+
allow_nosuggest: allow_nosuggest,
|
|
270
|
+
with_forbidden: with_forbidden,
|
|
271
|
+
compoundpos: compoundpos,
|
|
272
|
+
prefix_flags: prefix_flags,
|
|
273
|
+
suffix_flags: suffix_flags,
|
|
274
|
+
forbidden_flags: forbidden_flags)
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
produce_affix_forms(word, compoundpos: compoundpos,
|
|
278
|
+
prefix_flags: prefix_flags,
|
|
279
|
+
suffix_flags: suffix_flags,
|
|
280
|
+
forbidden_flags: forbidden_flags).each do |form|
|
|
281
|
+
found = false
|
|
282
|
+
homonyms = @dic[:homonyms]&.call(form.stem) || []
|
|
283
|
+
|
|
284
|
+
# FORBIDDENWORD: in compound context (compoundpos set) OR when the
|
|
285
|
+
# form has affixes, if ANY homonym of the stem carries
|
|
286
|
+
# FORBIDDENWORD, the entire form is rejected. This mirrors
|
|
287
|
+
# Spylls lookup.py — a forbidden stem must not appear as a
|
|
288
|
+
# compound part (even without affixes) nor as an affixed form.
|
|
289
|
+
if !with_forbidden && @aff[:FORBIDDENWORD] &&
|
|
290
|
+
(compoundpos || form.has_affixes?) &&
|
|
291
|
+
homonyms.any? { |h| (h[:flags] || []).include?(@aff[:FORBIDDENWORD]) }
|
|
292
|
+
next
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
homonyms.each do |homonym|
|
|
296
|
+
candidate = form.replace(in_dictionary: homonym)
|
|
297
|
+
if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
298
|
+
compoundpos: compoundpos)
|
|
299
|
+
found = true
|
|
300
|
+
yield candidate
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# FORCEUCASE: when checking the beginning of a compound and the
|
|
305
|
+
# original word is capitalized, also try lowercased stem homonyms
|
|
306
|
+
# so that compound parts that must be uppercased can still match.
|
|
307
|
+
if compoundpos == CompoundPos::BEGIN_POS && @aff[:FORCEUCASE] &&
|
|
308
|
+
captype == Capitalization::Type::INIT
|
|
309
|
+
lower_homonyms = @dic[:homonyms]&.call(form.stem.downcase) || []
|
|
310
|
+
lower_homonyms.each do |homonym|
|
|
311
|
+
candidate = form.replace(in_dictionary: homonym)
|
|
312
|
+
if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
313
|
+
compoundpos: compoundpos)
|
|
314
|
+
found = true
|
|
315
|
+
yield candidate
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
# Skip the case-insensitive fallback when any path already matched
|
|
321
|
+
# for this form, when the form lives in a compound slot (compound
|
|
322
|
+
# parts go through their own dispatch), or when the original
|
|
323
|
+
# word's captype is not ALL — the lowercase index is only meant
|
|
324
|
+
# for ALLCAPS queries whose dictionary stem differs in case.
|
|
325
|
+
next if found
|
|
326
|
+
next if compoundpos
|
|
327
|
+
next if captype != Capitalization::Type::ALL
|
|
328
|
+
next unless @aff[:casing].guess(word) == Capitalization::Type::NO
|
|
329
|
+
|
|
330
|
+
# ALLCAPS case-insensitive fallback: when the original input was
|
|
331
|
+
# ALL CAPS but the dictionary stem has a different case (e.g.
|
|
332
|
+
# user typed "UNICEF'S" / captype=ALL but the variant being
|
|
333
|
+
# checked is "unicef's" / captype=NO), look up homonyms in the
|
|
334
|
+
# lowercase index. Mirrors Spylls lookup.py:423-436.
|
|
335
|
+
ignorecase_homonyms = @dic[:homonyms]&.call(form.stem, ignorecase: true) || []
|
|
336
|
+
ignorecase_homonyms.each do |homonym|
|
|
337
|
+
forbidden = @aff[:FORBIDDENWORD] &&
|
|
338
|
+
form.has_affixes? &&
|
|
339
|
+
(homonym[:flags] || []).include?(@aff[:FORBIDDENWORD])
|
|
340
|
+
next if forbidden && !with_forbidden
|
|
341
|
+
|
|
342
|
+
candidate = form.replace(in_dictionary: homonym)
|
|
343
|
+
if is_good_form(candidate, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
344
|
+
compoundpos: compoundpos)
|
|
345
|
+
yield candidate
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Internal compound forms generator.
|
|
352
|
+
#
|
|
353
|
+
# @param word [String] Word to process
|
|
354
|
+
# @param captype [Symbol] Capitalization type
|
|
355
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
356
|
+
# @yield [CompoundForm] Each valid compound form
|
|
357
|
+
def compound_forms_internal(word, captype:, allow_nosuggest:)
|
|
358
|
+
unless block_given?
|
|
359
|
+
return enum_for(:compound_forms_internal, word, captype: captype,
|
|
360
|
+
allow_nosuggest: allow_nosuggest)
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Check if any affix form has FORBIDDENWORD
|
|
364
|
+
if @aff[:FORBIDDENWORD]
|
|
365
|
+
forbidden_found = false
|
|
366
|
+
affix_forms_internal(word, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
367
|
+
with_forbidden: true) do |form|
|
|
368
|
+
if form.flags.include?(@aff[:FORBIDDENWORD])
|
|
369
|
+
forbidden_found = true
|
|
370
|
+
break
|
|
371
|
+
end
|
|
372
|
+
end
|
|
373
|
+
return if forbidden_found
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
# Try compounds by flags
|
|
377
|
+
if @aff[:COMPOUNDBEGIN] || @aff[:COMPOUNDFLAG]
|
|
378
|
+
compounds_by_flags(word, captype: captype, allow_nosuggest: allow_nosuggest) do |compound|
|
|
379
|
+
yield compound unless is_bad_compound(compound, captype)
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
# Try compounds by rules
|
|
384
|
+
if @aff[:COMPOUNDRULE]
|
|
385
|
+
compounds_by_rules(word, allow_nosuggest: allow_nosuggest) do |compound|
|
|
386
|
+
yield compound unless is_bad_compound(compound, captype)
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
# Produce all possible affix forms for a word.
|
|
392
|
+
#
|
|
393
|
+
# @param word [String] Word to process
|
|
394
|
+
# @param compoundpos [Symbol, nil] Position in compound
|
|
395
|
+
# @param prefix_flags [Array<String>] Required prefix flags
|
|
396
|
+
# @param suffix_flags [Array<String>] Required suffix flags
|
|
397
|
+
# @param forbidden_flags [Array<String>] Forbidden affix flags
|
|
398
|
+
# @yield [AffixForm] Each possible affix form
|
|
399
|
+
def produce_affix_forms(word,
|
|
400
|
+
compoundpos: nil,
|
|
401
|
+
prefix_flags: [],
|
|
402
|
+
suffix_flags: [],
|
|
403
|
+
forbidden_flags: [], &block)
|
|
404
|
+
unless block
|
|
405
|
+
return enum_for(:produce_affix_forms, word,
|
|
406
|
+
compoundpos: compoundpos,
|
|
407
|
+
prefix_flags: prefix_flags,
|
|
408
|
+
suffix_flags: suffix_flags,
|
|
409
|
+
forbidden_flags: forbidden_flags)
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# "Whole word" is always an option
|
|
413
|
+
yield AffixForm.new(word, word)
|
|
414
|
+
|
|
415
|
+
# Check if suffixes/prefixes are allowed
|
|
416
|
+
suffix_allowed = compoundpos.nil? || compoundpos == CompoundPos::END_POS || !suffix_flags.empty?
|
|
417
|
+
prefix_allowed = compoundpos.nil? || compoundpos == CompoundPos::BEGIN_POS || !prefix_flags.empty?
|
|
418
|
+
|
|
419
|
+
# Generate suffix forms
|
|
420
|
+
if suffix_allowed
|
|
421
|
+
desuffix(word, required_flags: suffix_flags, forbidden_flags: forbidden_flags, &block)
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# Generate prefix forms
|
|
425
|
+
if prefix_allowed
|
|
426
|
+
deprefix(word, required_flags: prefix_flags, forbidden_flags: forbidden_flags) do |form|
|
|
427
|
+
yield form
|
|
428
|
+
|
|
429
|
+
# Try prefix + suffix if allowed
|
|
430
|
+
if suffix_allowed && form.prefix && form.prefix[:crossproduct]
|
|
431
|
+
desuffix(form.stem,
|
|
432
|
+
required_flags: suffix_flags,
|
|
433
|
+
forbidden_flags: forbidden_flags,
|
|
434
|
+
crossproduct: true) do |form2|
|
|
435
|
+
yield form2.replace(text: form.text, prefix: form.prefix)
|
|
436
|
+
end
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
# Remove suffixes from word.
|
|
443
|
+
#
|
|
444
|
+
# @param word [String] Word to process
|
|
445
|
+
# @param required_flags [Array<String>] Required suffix flags
|
|
446
|
+
# @param forbidden_flags [Array<String>] Forbidden suffix flags
|
|
447
|
+
# @param nested [Boolean] Whether this is a nested call
|
|
448
|
+
# @param crossproduct [Boolean] Whether suffix must have crossproduct
|
|
449
|
+
# @yield [AffixForm] Each form with suffix removed
|
|
450
|
+
def desuffix(word, required_flags: [], forbidden_flags: [], nested: false, crossproduct: false)
|
|
451
|
+
unless block_given?
|
|
452
|
+
return enum_for(:desuffix, word,
|
|
453
|
+
required_flags: required_flags,
|
|
454
|
+
forbidden_flags: forbidden_flags,
|
|
455
|
+
nested: nested,
|
|
456
|
+
crossproduct: crossproduct)
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
suffixes_index = @aff[:suffixes_index] || {}
|
|
460
|
+
word_reversed = word.reverse
|
|
461
|
+
|
|
462
|
+
# Spylls's Trie.lookup yields root payloads (empty-add suffixes)
|
|
463
|
+
# before walking the path, so suffixes with add="" are always
|
|
464
|
+
# considered. The hash index drops them unless we explicitly
|
|
465
|
+
# include the "" bucket.
|
|
466
|
+
candidates = (suffixes_index[''] || []) + (suffixes_index[word_reversed[0]] || [])
|
|
467
|
+
candidates.each do |suffix|
|
|
468
|
+
# Check if suffix is valid
|
|
469
|
+
next if crossproduct && !suffix[:crossproduct]
|
|
470
|
+
next unless (required_flags - (suffix[:flags] || [])).empty?
|
|
471
|
+
next unless (forbidden_flags & (suffix[:flags] || [])).empty?
|
|
472
|
+
|
|
473
|
+
# Check if suffix matches
|
|
474
|
+
if word.end_with?(suffix[:affix])
|
|
475
|
+
# Remove suffix and add strip value. Note: when affix is "",
|
|
476
|
+
# `word[0...-0]` would be `word[0...0]` = "" — so handle the
|
|
477
|
+
# empty case explicitly to keep the whole word as the base.
|
|
478
|
+
base = suffix[:affix].empty? ? word : word[0...-suffix[:affix].length]
|
|
479
|
+
strip = suffix[:affix_data] ? suffix[:affix_data][:strip] : ''
|
|
480
|
+
stem = base + strip
|
|
481
|
+
|
|
482
|
+
# Check condition (only if condition_checker is present)
|
|
483
|
+
next if suffix[:condition_checker] && !suffix[:condition_checker].matches?(stem)
|
|
484
|
+
|
|
485
|
+
yield AffixForm.new(word, stem, suffix: suffix)
|
|
486
|
+
|
|
487
|
+
# Try removing another suffix (one level only)
|
|
488
|
+
unless nested
|
|
489
|
+
desuffix(stem,
|
|
490
|
+
required_flags: [suffix[:flag], *required_flags],
|
|
491
|
+
forbidden_flags: forbidden_flags,
|
|
492
|
+
nested: true,
|
|
493
|
+
crossproduct: crossproduct) do |form2|
|
|
494
|
+
yield form2.replace(suffix2: suffix, text: word)
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
# Remove prefixes from word.
|
|
502
|
+
#
|
|
503
|
+
# @param word [String] Word to process
|
|
504
|
+
# @param required_flags [Array<String>] Required prefix flags
|
|
505
|
+
# @param forbidden_flags [Array<String>] Forbidden prefix flags
|
|
506
|
+
# @param nested [Boolean] Whether this is a nested call
|
|
507
|
+
# @yield [AffixForm] Each form with prefix removed
|
|
508
|
+
def deprefix(word, required_flags: [], forbidden_flags: [], nested: false)
|
|
509
|
+
unless block_given?
|
|
510
|
+
return enum_for(:deprefix, word,
|
|
511
|
+
required_flags: required_flags,
|
|
512
|
+
forbidden_flags: forbidden_flags,
|
|
513
|
+
nested: nested)
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
prefixes_index = @aff[:prefixes_index] || {}
|
|
517
|
+
|
|
518
|
+
# Mirror the suffix side: prefixes with add="" live under the ""
|
|
519
|
+
# bucket and must always be considered.
|
|
520
|
+
candidates = (prefixes_index[''] || []) + (prefixes_index[word[0]] || [])
|
|
521
|
+
candidates.each do |prefix|
|
|
522
|
+
# Check if prefix is valid
|
|
523
|
+
next unless (required_flags - (prefix[:flags] || [])).empty?
|
|
524
|
+
next unless (forbidden_flags & (prefix[:flags] || [])).empty?
|
|
525
|
+
|
|
526
|
+
# Check if prefix matches
|
|
527
|
+
if word.start_with?(prefix[:affix])
|
|
528
|
+
# Remove prefix and re-add the strip value at the START.
|
|
529
|
+
# (For suffixes the strip is appended; for prefixes it's
|
|
530
|
+
# prepended — the strip/affix are mirrors of each other
|
|
531
|
+
# and the strip lives on the same edge of the stem as the
|
|
532
|
+
# affix does on the candidate.)
|
|
533
|
+
strip = prefix[:affix_data] ? prefix[:affix_data][:strip] : ''
|
|
534
|
+
stem = strip + word[prefix[:affix].length..]
|
|
535
|
+
|
|
536
|
+
# Check condition (only if condition_checker is present)
|
|
537
|
+
next if prefix[:condition_checker] && !prefix[:condition_checker].matches?(stem)
|
|
538
|
+
|
|
539
|
+
yield AffixForm.new(word, stem, prefix: prefix)
|
|
540
|
+
|
|
541
|
+
# Try removing another prefix if COMPLEXPREFIXES is set
|
|
542
|
+
unless nested || !@aff[:COMPLEXPREFIXES]
|
|
543
|
+
deprefix(stem,
|
|
544
|
+
required_flags: [prefix[:flag], *required_flags],
|
|
545
|
+
forbidden_flags: forbidden_flags,
|
|
546
|
+
nested: true) do |form2|
|
|
547
|
+
yield form2.replace(prefix2: prefix, text: word)
|
|
548
|
+
end
|
|
549
|
+
end
|
|
550
|
+
end
|
|
551
|
+
end
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
# Check if an affix form is valid.
|
|
555
|
+
#
|
|
556
|
+
# When compoundpos is nil (non-compound check), the form must not
|
|
557
|
+
# carry ONLYINCOMPOUND. When compoundpos is set (compound part
|
|
558
|
+
# check), ONLYINCOMPOUND is allowed and instead the form must carry
|
|
559
|
+
# COMPOUNDFLAG or the position-specific flag (COMPOUNDBEGIN /
|
|
560
|
+
# COMPOUNDMIDDLE / COMPOUNDEND). This mirrors Spylls lookup.py
|
|
561
|
+
# is_good_form.
|
|
562
|
+
#
|
|
563
|
+
# @param form [AffixForm] Form to check
|
|
564
|
+
# @param captype [Symbol] Original word's capitalization type
|
|
565
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
566
|
+
# @param compoundpos [Symbol, nil] Position in compound, or nil
|
|
567
|
+
# @return [Boolean] Whether form is valid
|
|
568
|
+
def is_good_form(form, captype:, allow_nosuggest:, compoundpos: nil)
|
|
569
|
+
return false unless form.in_dictionary
|
|
570
|
+
|
|
571
|
+
root_flags = form.in_dictionary[:flags] || []
|
|
572
|
+
all_flags = form.flags
|
|
573
|
+
|
|
574
|
+
# Check NOSUGGEST
|
|
575
|
+
if !allow_nosuggest && @aff[:NOSUGGEST] && root_flags.include?(@aff[:NOSUGGEST])
|
|
576
|
+
return false
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
# Check KEEPCASE
|
|
580
|
+
if @aff[:KEEPCASE] && root_flags.include?(@aff[:KEEPCASE])
|
|
581
|
+
stem_captype = @aff[:casing].guess(form.in_dictionary[:stem])
|
|
582
|
+
return false if captype != stem_captype && !(@aff[:CHECKSHARPS] && form.in_dictionary[:stem].include?('ß'))
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
# Check NEEDAFFIX
|
|
586
|
+
if @aff[:NEEDAFFIX]
|
|
587
|
+
if root_flags.include?(@aff[:NEEDAFFIX]) && !form.has_affixes?
|
|
588
|
+
return false
|
|
589
|
+
end
|
|
590
|
+
if form.has_affixes? && form.all_affixes.all? { |a| (a[:flags] || []).include?(@aff[:NEEDAFFIX]) }
|
|
591
|
+
return false
|
|
592
|
+
end
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
# Check prefix flag compatibility
|
|
596
|
+
if form.prefix && !all_flags.include?(form.prefix[:flag])
|
|
597
|
+
return false
|
|
598
|
+
end
|
|
599
|
+
|
|
600
|
+
# Check suffix flag compatibility
|
|
601
|
+
if form.suffix && !all_flags.include?(form.suffix[:flag])
|
|
602
|
+
return false
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
# Check CIRCUMFIX
|
|
606
|
+
if @aff[:CIRCUMFIX]
|
|
607
|
+
suffix_has = form.suffix ? (form.suffix[:flags] || []).include?(@aff[:CIRCUMFIX]) : false
|
|
608
|
+
prefix_has = form.prefix ? (form.prefix[:flags] || []).include?(@aff[:CIRCUMFIX]) : false
|
|
609
|
+
return false if suffix_has != prefix_has
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
# Compound position checks
|
|
613
|
+
if compoundpos.nil?
|
|
614
|
+
# Non-compound: reject ONLYINCOMPOUND words
|
|
615
|
+
return !all_flags.include?(@aff[:ONLYINCOMPOUND])
|
|
616
|
+
end
|
|
617
|
+
|
|
618
|
+
return false if compoundpos == CompoundPos::END_POS && barred_from_compound_end?(form)
|
|
619
|
+
|
|
620
|
+
# Compound: must carry COMPOUNDFLAG or the position-specific flag.
|
|
621
|
+
# ONLYINCOMPOUND is allowed here (it just means "not valid outside
|
|
622
|
+
# compounds") — the compound position flag is what authorizes the
|
|
623
|
+
# part to appear at this slot.
|
|
624
|
+
return true if @aff[:COMPOUNDFLAG] && all_flags.include?(@aff[:COMPOUNDFLAG])
|
|
625
|
+
return true if compoundpos == CompoundPos::BEGIN_POS && @aff[:COMPOUNDBEGIN] &&
|
|
626
|
+
all_flags.include?(@aff[:COMPOUNDBEGIN])
|
|
627
|
+
return true if compoundpos == CompoundPos::MIDDLE && @aff[:COMPOUNDMIDDLE] &&
|
|
628
|
+
all_flags.include?(@aff[:COMPOUNDMIDDLE])
|
|
629
|
+
return true if compoundpos == CompoundPos::END_POS && @aff[:COMPOUNDEND] &&
|
|
630
|
+
all_flags.include?(@aff[:COMPOUNDEND])
|
|
631
|
+
|
|
632
|
+
false
|
|
633
|
+
end
|
|
634
|
+
|
|
635
|
+
# Whether a suffix stops this form from closing a compound.
|
|
636
|
+
#
|
|
637
|
+
# Ported from Hunspell's suffix_check (affixmgr.cxx:2832): a suffix
|
|
638
|
+
# is barred at the compound end only when it carries ONLYINCOMPOUND
|
|
639
|
+
# and nothing else rescues it. Two things do. A zero-width suffix is
|
|
640
|
+
# handled by a separate branch upstream that has no such guard, and
|
|
641
|
+
# a prefix on the form is an explicit exemption (`ppfx` upstream).
|
|
642
|
+
# That is how
|
|
643
|
+
# German reaches "Arbeitscomputern", through its decapitalising
|
|
644
|
+
# prefix rather than through any compound-position flag.
|
|
645
|
+
#
|
|
646
|
+
# @param form [AffixForm] Form being placed at the compound end
|
|
647
|
+
# @return [Boolean] Whether a suffix bars it from ending the compound
|
|
648
|
+
def barred_from_compound_end?(form)
|
|
649
|
+
only_in_compound = @aff[:ONLYINCOMPOUND]
|
|
650
|
+
return false unless only_in_compound
|
|
651
|
+
return false if form.prefix
|
|
652
|
+
|
|
653
|
+
form.suffixes.any? do |suffix|
|
|
654
|
+
!suffix[:affix].to_s.empty? && (suffix[:flags] || []).include?(only_in_compound)
|
|
655
|
+
end
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
# Generate compound forms by flags.
|
|
659
|
+
#
|
|
660
|
+
# @param word_rest [String] Remaining word to process
|
|
661
|
+
# @param captype [Symbol] Capitalization type
|
|
662
|
+
# @param depth [Integer] Current recursion depth
|
|
663
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
664
|
+
# @yield [CompoundForm] Each valid compound form
|
|
665
|
+
def compounds_by_flags(word_rest, captype:, depth: 0, allow_nosuggest: true)
|
|
666
|
+
unless block_given?
|
|
667
|
+
return enum_for(:compounds_by_flags, word_rest,
|
|
668
|
+
captype: captype,
|
|
669
|
+
depth: depth,
|
|
670
|
+
allow_nosuggest: allow_nosuggest)
|
|
671
|
+
end
|
|
672
|
+
|
|
673
|
+
aff = @aff
|
|
674
|
+
compound_min = aff[:COMPOUNDMIN] || 3
|
|
675
|
+
compound_word_max = aff[:COMPOUNDWORDMAX]
|
|
676
|
+
compound_permit_flag = aff[:COMPOUNDPERMITFLAG]
|
|
677
|
+
compound_forbid_flag = aff[:COMPOUNDFORBIDFLAG]
|
|
678
|
+
|
|
679
|
+
forbidden_flags = compound_forbid_flag ? [compound_forbid_flag] : []
|
|
680
|
+
permit_flags = compound_permit_flag ? [compound_permit_flag] : []
|
|
681
|
+
|
|
682
|
+
# Check if rest can be compound end. At END position, suffixes are
|
|
683
|
+
# always allowed (compoundpos=END_POS in produce_affix_forms), but
|
|
684
|
+
# prefixes still need COMPOUNDPERMITFLAG.
|
|
685
|
+
if depth.positive?
|
|
686
|
+
affix_forms_internal(word_rest, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
687
|
+
compoundpos: CompoundPos::END_POS,
|
|
688
|
+
prefix_flags: permit_flags,
|
|
689
|
+
forbidden_flags: forbidden_flags) do |form|
|
|
690
|
+
yield CompoundForm.new([form])
|
|
691
|
+
end
|
|
692
|
+
end
|
|
693
|
+
|
|
694
|
+
# Check compounding limits
|
|
695
|
+
return if word_rest.length < compound_min * 2
|
|
696
|
+
return if compound_word_max && depth >= compound_word_max
|
|
697
|
+
|
|
698
|
+
compoundpos = depth.zero? ? CompoundPos::BEGIN_POS : CompoundPos::MIDDLE
|
|
699
|
+
# At BEGIN_POS, prefixes are allowed by default (no permit flag
|
|
700
|
+
# needed); at MIDDLE, prefixes need the permit flag. Suffixes at
|
|
701
|
+
# both BEGIN and MIDDLE need the permit flag — when there is no
|
|
702
|
+
# COMPOUNDPERMITFLAG in the .aff, suffix_flags is empty and
|
|
703
|
+
# produce_affix_forms blocks all suffixes inside compounds.
|
|
704
|
+
prefix_flags = compoundpos == CompoundPos::BEGIN_POS ? [] : permit_flags
|
|
705
|
+
|
|
706
|
+
each_compound_junction(word_rest, compound_min) do |left, right, left_text, pattern|
|
|
707
|
+
affix_forms_internal(left, captype: captype, allow_nosuggest: allow_nosuggest,
|
|
708
|
+
compoundpos: compoundpos,
|
|
709
|
+
prefix_flags: prefix_flags,
|
|
710
|
+
suffix_flags: permit_flags,
|
|
711
|
+
forbidden_flags: forbidden_flags) do |form|
|
|
712
|
+
compounds_by_flags(right, captype: captype, depth: depth + 1,
|
|
713
|
+
allow_nosuggest: allow_nosuggest) do |partial|
|
|
714
|
+
next if pattern && !pattern.match?(form, partial.parts.first)
|
|
715
|
+
|
|
716
|
+
part = left_text ? form.replace(text: left_text) : form
|
|
717
|
+
yield CompoundForm.new([part, *partial.parts],
|
|
718
|
+
[pattern, *partial.junction_patterns])
|
|
719
|
+
end
|
|
720
|
+
end
|
|
721
|
+
end
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
# CHECKCOMPOUNDPATTERN entries that spell a simplified compound form.
|
|
725
|
+
#
|
|
726
|
+
# @return [Array<Readers::CompoundPattern>] Patterns with a replacement
|
|
727
|
+
def replacement_patterns
|
|
728
|
+
@replacement_patterns ||= (@aff[:CHECKCOMPOUNDPATTERN] || []).select(&:replacement?)
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
# Every cut of the word worth trying as a compound boundary.
|
|
732
|
+
#
|
|
733
|
+
# Yields `(left, right, left_text, pattern)` rather than an object:
|
|
734
|
+
# this is the innermost loop of the whole checker, and one allocation
|
|
735
|
+
# per split position is measurable on deeply compounding languages.
|
|
736
|
+
#
|
|
737
|
+
# `left_text` is the text to record on the left member when it
|
|
738
|
+
# differs from the text looked up, and `pattern` is set only for a
|
|
739
|
+
# cut a CHECKCOMPOUNDPATTERN replacement rebuilt. Both are nil for
|
|
740
|
+
# the ordinary case, which is nearly all of them.
|
|
741
|
+
#
|
|
742
|
+
# The window is COMPOUNDMIN either side of the cut, measured on the
|
|
743
|
+
# word as written. Hunspell fixes it the same way, before it tries
|
|
744
|
+
# any replacement (setcminmax at affixmgr.cxx:1644, the replacement
|
|
745
|
+
# loop nested inside at :1660). A replacement can therefore stand for
|
|
746
|
+
# members longer than the text it replaced, which is why hunspell.5
|
|
747
|
+
# warns that COMPOUNDMIN "doesn't work correctly with the compound
|
|
748
|
+
# word alternation, so it may need to set COMPOUNDMIN to lower
|
|
749
|
+
# value". Matching that quirk beats being quietly more permissive
|
|
750
|
+
# than the dictionaries were written against.
|
|
751
|
+
#
|
|
752
|
+
# @param word_rest [String] Remaining word being split
|
|
753
|
+
# @param compound_min [Integer] COMPOUNDMIN
|
|
754
|
+
# @yield [String, String, String, Readers::CompoundPattern] Left
|
|
755
|
+
# member, remaining text, surface text or nil, pattern or nil
|
|
756
|
+
def each_compound_junction(word_rest, compound_min)
|
|
757
|
+
simplified_triple = @aff[:SIMPLIFIEDTRIPLE]
|
|
758
|
+
patterns = replacement_patterns
|
|
759
|
+
|
|
760
|
+
(compound_min...(word_rest.length - compound_min + 1)).each do |pos|
|
|
761
|
+
beg = word_rest[0...pos]
|
|
762
|
+
rest = word_rest[pos..]
|
|
763
|
+
yield beg, rest, nil, nil
|
|
764
|
+
|
|
765
|
+
if simplified_triple && !beg.empty? && !rest.empty? && beg[-1] == rest[0]
|
|
766
|
+
yield beg + beg[-1], rest, beg, nil
|
|
767
|
+
end
|
|
768
|
+
|
|
769
|
+
patterns.each do |pattern|
|
|
770
|
+
replacement = pattern.replacement
|
|
771
|
+
next unless word_rest[pos, replacement.length] == replacement
|
|
772
|
+
|
|
773
|
+
yield beg + pattern.left_stem,
|
|
774
|
+
pattern.right_stem + word_rest[(pos + replacement.length)..],
|
|
775
|
+
nil, pattern
|
|
776
|
+
end
|
|
777
|
+
end
|
|
778
|
+
end
|
|
779
|
+
|
|
780
|
+
# Generate compound forms by rules.
|
|
781
|
+
#
|
|
782
|
+
# @param word_rest [String] Remaining word to process
|
|
783
|
+
# @param prev_parts [Array<Hash>] Previously processed parts
|
|
784
|
+
# @param rules [Array<Hash>] Valid compound rules
|
|
785
|
+
# @param allow_nosuggest [Boolean] Include NOSUGGEST words
|
|
786
|
+
# @yield [CompoundForm] Each valid compound form
|
|
787
|
+
def compounds_by_rules(word_rest, prev_parts: [], rules: nil, allow_nosuggest: true)
|
|
788
|
+
unless block_given?
|
|
789
|
+
return enum_for(:compounds_by_rules, word_rest,
|
|
790
|
+
prev_parts: prev_parts,
|
|
791
|
+
rules: rules,
|
|
792
|
+
allow_nosuggest: allow_nosuggest)
|
|
793
|
+
end
|
|
794
|
+
|
|
795
|
+
aff = @aff
|
|
796
|
+
compound_min = aff[:COMPOUNDMIN] || 3
|
|
797
|
+
compound_word_max = aff[:COMPOUNDWORDMAX]
|
|
798
|
+
compound_rules = aff[:COMPOUNDRULE] || []
|
|
799
|
+
|
|
800
|
+
rules ||= compound_rules
|
|
801
|
+
|
|
802
|
+
# Check if rest can be a complete compound
|
|
803
|
+
if prev_parts.any?
|
|
804
|
+
homonyms = @dic[:homonyms]&.call(word_rest) || []
|
|
805
|
+
homonyms.each do |homonym|
|
|
806
|
+
parts = [*prev_parts, homonym]
|
|
807
|
+
flag_sets = parts.map { |p| p[:flags] || [] }
|
|
808
|
+
|
|
809
|
+
if compound_rules.any? { |rule| rule[:full_match]&.call(flag_sets) }
|
|
810
|
+
yield CompoundForm.new([AffixForm.new(word_rest, word_rest)])
|
|
811
|
+
end
|
|
812
|
+
end
|
|
813
|
+
end
|
|
814
|
+
|
|
815
|
+
# Check limits
|
|
816
|
+
return if word_rest.length < compound_min * 2
|
|
817
|
+
return if compound_word_max && prev_parts.length >= compound_word_max
|
|
818
|
+
|
|
819
|
+
# Try all possible split positions
|
|
820
|
+
(compound_min...(word_rest.length - compound_min + 1)).each do |pos|
|
|
821
|
+
beg = word_rest[0...pos]
|
|
822
|
+
homonyms = @dic[:homonyms]&.call(beg) || []
|
|
823
|
+
|
|
824
|
+
homonyms.each do |homonym|
|
|
825
|
+
parts = [*prev_parts, homonym]
|
|
826
|
+
flag_sets = parts.map { |p| p[:flags] || [] }
|
|
827
|
+
|
|
828
|
+
matching_rules = compound_rules.select { |rule| rule[:partial_match]&.call(flag_sets) }
|
|
829
|
+
next if matching_rules.empty?
|
|
830
|
+
|
|
831
|
+
compounds_by_rules(word_rest[pos..], prev_parts: parts, rules: matching_rules,
|
|
832
|
+
allow_nosuggest: allow_nosuggest) do |partial|
|
|
833
|
+
yield CompoundForm.new([AffixForm.new(beg, beg), *partial.parts])
|
|
834
|
+
end
|
|
835
|
+
end
|
|
836
|
+
end
|
|
837
|
+
end
|
|
838
|
+
end
|
|
839
|
+
end
|
|
840
|
+
end
|
|
841
|
+
end
|