kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,584 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Readers
|
|
5
|
+
# Builder for creating Lookup::Lookuper instances from Hunspell data.
|
|
6
|
+
#
|
|
7
|
+
# This class can either read from files or accept pre-read aff/dic data.
|
|
8
|
+
#
|
|
9
|
+
# @example Building a lookuper from files
|
|
10
|
+
# builder = LookupBuilder.new('en_US.aff', 'en_US.dic')
|
|
11
|
+
# lookuper = builder.build
|
|
12
|
+
#
|
|
13
|
+
# @example Building a lookuper from pre-read data
|
|
14
|
+
# aff_reader = AffReader.new('en_US.aff')
|
|
15
|
+
# aff_data = aff_reader.read
|
|
16
|
+
# dic_reader = DicReader.new('en_US.dic')
|
|
17
|
+
# words = dic_reader.read
|
|
18
|
+
# builder = LookupBuilder.from_data(aff_data, words)
|
|
19
|
+
# lookuper = builder.build
|
|
20
|
+
class LookupBuilder
|
|
21
|
+
# Hunspell upstream defaults for suggester directives. These apply
|
|
22
|
+
# when the .aff file is silent on the directive. See the Hunspell
|
|
23
|
+
# documentation at https://hunspell.github.io/ and the spylls
|
|
24
|
+
# `Aff` defaults for reference.
|
|
25
|
+
DEFAULT_MAX_NGRAM_SUGS = 4
|
|
26
|
+
DEFAULT_MAX_CPD_SUGS = 3
|
|
27
|
+
DEFAULT_MAX_DIFF = 5
|
|
28
|
+
|
|
29
|
+
attr_reader :aff_path, :dic_path, :encoding, :aff_data, :words, :script
|
|
30
|
+
|
|
31
|
+
# Create a new LookupBuilder from file paths.
|
|
32
|
+
#
|
|
33
|
+
# @param aff_path [String, nil] Path to the .aff file (nil when
|
|
34
|
+
# constructing via .from_data with pre-read aff/dic data)
|
|
35
|
+
# @param dic_path [String, nil] Path to the .dic file
|
|
36
|
+
# @param encoding [String] File encoding (default: 'UTF-8')
|
|
37
|
+
# @param script [Symbol] The script type for condition checking (default: :latin)
|
|
38
|
+
# @param aff_data [Hash, nil] Pre-read aff data (skips file read
|
|
39
|
+
# in #build when provided)
|
|
40
|
+
# @param words [Array<Word>, nil] Pre-read word entries
|
|
41
|
+
def initialize(aff_path, dic_path, encoding: 'UTF-8', script: :latin,
|
|
42
|
+
aff_data: nil, words: nil)
|
|
43
|
+
@aff_path = aff_path
|
|
44
|
+
@dic_path = dic_path
|
|
45
|
+
@encoding = encoding
|
|
46
|
+
@script = script
|
|
47
|
+
@aff_data = aff_data
|
|
48
|
+
@words = words
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Create a new LookupBuilder from pre-read data.
|
|
52
|
+
#
|
|
53
|
+
# @param aff_data [Hash] Raw aff data from AffReader
|
|
54
|
+
# @param words [Array<Word>] Word entries from DicReader
|
|
55
|
+
# @return [LookupBuilder] A new builder instance
|
|
56
|
+
def self.from_data(aff_data, words)
|
|
57
|
+
new(nil, nil, aff_data: aff_data, words: words)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Build the Lookuper instance.
|
|
61
|
+
#
|
|
62
|
+
# @return [Algorithms::Lookup::Lookuper] The lookuper instance
|
|
63
|
+
def build
|
|
64
|
+
# Read files if data not already provided
|
|
65
|
+
aff_data_to_use = @aff_data || read_aff_data
|
|
66
|
+
words_to_use = @words || read_dic_data(aff_data_to_use)
|
|
67
|
+
|
|
68
|
+
# Build the aff structure for Lookuper
|
|
69
|
+
aff = build_aff_structure(aff_data_to_use)
|
|
70
|
+
|
|
71
|
+
# Build the dic structure for Lookuper. Dictionary `ph:` morph data
|
|
72
|
+
# is a source of REP entries (Hunspell 1.7+), so build_dic_structure
|
|
73
|
+
# also enriches aff[:REP] — see PhRepExtractor.
|
|
74
|
+
dic = build_dic_structure(words_to_use, aff: aff)
|
|
75
|
+
|
|
76
|
+
# Create and return the Lookuper
|
|
77
|
+
Algorithms::Lookup::Lookuper.new(aff, dic)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
# Read aff data from file.
|
|
83
|
+
#
|
|
84
|
+
# @return [Hash] Raw aff data
|
|
85
|
+
def read_aff_data
|
|
86
|
+
aff_reader = AffReader.new(@aff_path, encoding: @encoding)
|
|
87
|
+
aff_reader.read
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Read dic data from file.
|
|
91
|
+
#
|
|
92
|
+
# @param aff_data [Hash] Aff data for flag format info
|
|
93
|
+
# @return [Array<Word>] Word entries
|
|
94
|
+
def read_dic_data(aff_data)
|
|
95
|
+
dic_reader = DicReader.new(@dic_path,
|
|
96
|
+
encoding: @encoding,
|
|
97
|
+
flag_format: aff_data['FLAG'] || 'short',
|
|
98
|
+
flag_synonyms: aff_data['AF'] || {})
|
|
99
|
+
dic_reader.read
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Select the appropriate Casing class based on the aff file's directives.
|
|
103
|
+
#
|
|
104
|
+
# Hunspell associates a casing strategy with the language declared via
|
|
105
|
+
# LANG or with specific directives like CHECKSHARPS (German ß↔SS) and
|
|
106
|
+
# Turkic languages (i↔İ / ı↔I). We mirror that here.
|
|
107
|
+
def select_casing(aff_data)
|
|
108
|
+
lang = aff_data['LANG'].to_s.downcase
|
|
109
|
+
|
|
110
|
+
if aff_data['CHECKSHARPS']
|
|
111
|
+
Algorithms::Capitalization::GermanCasing.new
|
|
112
|
+
elsif lang.start_with?('tr', 'az', 'crh', 'tt', 'krc', 'kaa')
|
|
113
|
+
Algorithms::Capitalization::TurkicCasing.new
|
|
114
|
+
elsif lang.start_with?('de')
|
|
115
|
+
Algorithms::Capitalization::GermanCasing.new
|
|
116
|
+
else
|
|
117
|
+
Algorithms::Capitalization::Casing.new
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Build the aff data structure for Lookuper.
|
|
122
|
+
#
|
|
123
|
+
# @param aff_data [Hash] Raw aff data from AffReader
|
|
124
|
+
# @return [Hash] Aff structure for Lookuper
|
|
125
|
+
def build_aff_structure(aff_data)
|
|
126
|
+
aff = {}
|
|
127
|
+
|
|
128
|
+
# Capitalization handler — German when CHECKSHARPS is set (the
|
|
129
|
+
# German ß↔SS rule needs special handling: lower("STRASSE") yields
|
|
130
|
+
# both "straße" and "strasse"), Turkic when LANG starts with az/tr,
|
|
131
|
+
# otherwise the standard Casing.
|
|
132
|
+
aff[:casing] = select_casing(aff_data)
|
|
133
|
+
|
|
134
|
+
# IGNORE chars (as Array<String>) — applied to dictionary stems
|
|
135
|
+
# and affix `add` strings at read time so input lookups (which
|
|
136
|
+
# strip IGNORE from the query) match. Without this, prefix "re" +
|
|
137
|
+
# stem "expression" could never match input "reexpression" once
|
|
138
|
+
# the input is reduced to "rxprssn".
|
|
139
|
+
ignore_chars = aff_data['IGNORE']&.chars&.chars || []
|
|
140
|
+
|
|
141
|
+
# Build suffixes index (indexed by first character of reversed suffix)
|
|
142
|
+
suffixes_index = {}
|
|
143
|
+
suffixes_by_flag = Hash.new { |h, k| h[k] = [] }
|
|
144
|
+
aff_data['SFX'].each do |_flag, affix_list|
|
|
145
|
+
affix_list.each do |affix|
|
|
146
|
+
stripped_add = strip_ignore(affix.add, ignore_chars)
|
|
147
|
+
# For suffixes, we need to index by the first char of the REVERSED suffix
|
|
148
|
+
# because the lookup code reverses the word to check suffixes
|
|
149
|
+
reversed_suffix = stripped_add.reverse
|
|
150
|
+
first_char = reversed_suffix[0] || ''
|
|
151
|
+
suffixes_index[first_char] ||= []
|
|
152
|
+
affix_hash = build_affix_hash(affix, script: @script || :latin,
|
|
153
|
+
add: stripped_add)
|
|
154
|
+
suffixes_index[first_char] << affix_hash
|
|
155
|
+
suffixes_by_flag[affix.flag] << affix_hash
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
aff[:suffixes_index] = suffixes_index
|
|
159
|
+
aff[:suffixes_by_flag] = suffixes_by_flag
|
|
160
|
+
|
|
161
|
+
# Build prefixes index (indexed by first character of prefix)
|
|
162
|
+
prefixes_index = {}
|
|
163
|
+
prefixes_by_flag = Hash.new { |h, k| h[k] = [] }
|
|
164
|
+
aff_data['PFX'].each do |_flag, affix_list|
|
|
165
|
+
affix_list.each do |affix|
|
|
166
|
+
stripped_add = strip_ignore(affix.add, ignore_chars)
|
|
167
|
+
first_char = stripped_add[0] || ''
|
|
168
|
+
prefixes_index[first_char] ||= []
|
|
169
|
+
affix_hash = build_affix_hash(affix, script: @script || :latin,
|
|
170
|
+
add: stripped_add)
|
|
171
|
+
prefixes_index[first_char] << affix_hash
|
|
172
|
+
prefixes_by_flag[affix.flag] << affix_hash
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
aff[:prefixes_index] = prefixes_index
|
|
176
|
+
aff[:prefixes_by_flag] = prefixes_by_flag
|
|
177
|
+
|
|
178
|
+
# Single-value flags
|
|
179
|
+
aff[:COMPOUNDMIN] = aff_data['COMPOUNDMIN']
|
|
180
|
+
aff[:COMPOUNDWORDMAX] = aff_data['COMPOUNDWORDMAX']
|
|
181
|
+
aff[:COMPOUNDBEGIN] = aff_data['COMPOUNDBEGIN']
|
|
182
|
+
aff[:COMPOUNDMIDDLE] = aff_data['COMPOUNDMIDDLE']
|
|
183
|
+
aff[:COMPOUNDEND] = aff_data['COMPOUNDEND']
|
|
184
|
+
aff[:COMPOUNDFLAG] = aff_data['COMPOUNDFLAG']
|
|
185
|
+
aff[:COMPOUNDPERMITFLAG] = aff_data['COMPOUNDPERMITFLAG']
|
|
186
|
+
aff[:COMPOUNDFORBIDFLAG] = aff_data['COMPOUNDFORBIDFLAG']
|
|
187
|
+
aff[:COMPOUNDRULE] = build_compound_rules(aff_data['COMPOUNDRULE'])
|
|
188
|
+
aff[:ONLYINCOMPOUND] = aff_data['ONLYINCOMPOUND']
|
|
189
|
+
aff[:COMPLEXPREFIXES] = aff_data['COMPLEXPREFIXES']
|
|
190
|
+
aff[:FORCEUCASE] = aff_data['FORCEUCASE']
|
|
191
|
+
|
|
192
|
+
# Special flags
|
|
193
|
+
aff[:FORBIDDENWORD] = aff_data['FORBIDDENWORD']
|
|
194
|
+
aff[:NOSUGGEST] = aff_data['NOSUGGEST']
|
|
195
|
+
aff[:KEEPCASE] = aff_data['KEEPCASE']
|
|
196
|
+
aff[:NEEDAFFIX] = aff_data['NEEDAFFIX']
|
|
197
|
+
aff[:CIRCUMFIX] = aff_data['CIRCUMFIX']
|
|
198
|
+
aff[:WARN] = aff_data['WARN']
|
|
199
|
+
|
|
200
|
+
# Compound checking flags
|
|
201
|
+
aff[:CHECKCOMPOUNDCASE] = aff_data['CHECKCOMPOUNDCASE']
|
|
202
|
+
aff[:CHECKCOMPOUNDDUP] = aff_data['CHECKCOMPOUNDDUP']
|
|
203
|
+
aff[:CHECKCOMPOUNDREP] = aff_data['CHECKCOMPOUNDREP']
|
|
204
|
+
aff[:CHECKCOMPOUNDTRIPLE] = aff_data['CHECKCOMPOUNDTRIPLE']
|
|
205
|
+
aff[:CHECKCOMPOUNDPATTERN] = aff_data['CHECKCOMPOUNDPATTERN'] || []
|
|
206
|
+
aff[:SIMPLIFIEDTRIPLE] = aff_data['SIMPLIFIEDTRIPLE']
|
|
207
|
+
|
|
208
|
+
# Other directives
|
|
209
|
+
aff[:IGNORE] = aff_data['IGNORE']&.chars&.chars || []
|
|
210
|
+
aff[:BREAK] = build_break_patterns(aff_data['BREAK'])
|
|
211
|
+
aff[:ICONV] = aff_data['ICONV']
|
|
212
|
+
aff[:OCONV] = aff_data['OCONV']
|
|
213
|
+
aff[:REP] = build_rep_table(aff_data['REP'])
|
|
214
|
+
aff[:MAP] = aff_data['MAP'] || []
|
|
215
|
+
aff[:CHECKSHARPS] = aff_data['CHECKSHARPS']
|
|
216
|
+
|
|
217
|
+
# Suggester directives (Hunspell defaults applied where the .aff
|
|
218
|
+
# is silent — these defaults come from the upstream Hunspell
|
|
219
|
+
# manual and match spylls' Aff.objects defaults).
|
|
220
|
+
aff[:TRY] = aff_data['TRY']
|
|
221
|
+
aff[:KEY] = aff_data['KEY']
|
|
222
|
+
aff[:WORDCHARS] = aff_data['WORDCHARS']
|
|
223
|
+
aff[:PHONE] = aff_data['PHONE']
|
|
224
|
+
aff[:NOSPLITSUGS] = aff_data['NOSPLITSUGS']
|
|
225
|
+
aff[:SUGSWITHDOTS] = aff_data['SUGSWITHDOTS']
|
|
226
|
+
aff[:FULLSTRIP] = aff_data['FULLSTRIP']
|
|
227
|
+
aff[:MAXNGRAMSUGS] = aff_data['MAXNGRAMSUGS'] || DEFAULT_MAX_NGRAM_SUGS
|
|
228
|
+
aff[:MAXCPDSUGS] = aff_data['MAXCPDSUGS'] || DEFAULT_MAX_CPD_SUGS
|
|
229
|
+
aff[:MAXDIFF] = aff_data['MAXDIFF'] || DEFAULT_MAX_DIFF
|
|
230
|
+
aff[:ONLYMAXDIFF] = aff_data['ONLYMAXDIFF']
|
|
231
|
+
|
|
232
|
+
aff
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Build the dic data structure for Lookuper.
|
|
236
|
+
#
|
|
237
|
+
# Two stem indexes are maintained, mirroring Spylls's `Dic.index` and
|
|
238
|
+
# `Dic.lowercase_index`:
|
|
239
|
+
#
|
|
240
|
+
# * `word_index` — keyed by the exact stem as it appears in the .dic file.
|
|
241
|
+
# * `lowercase_word_index` — keyed by the lowercased stem. Hunspell's
|
|
242
|
+
# ALLCAPS lookup path uses this to find dictionary entries whose stem
|
|
243
|
+
# casing doesn't match (e.g. querying "unicef" against a "UNICEF" entry).
|
|
244
|
+
#
|
|
245
|
+
# As a side effect, dictionary `ph:` morph data is folded into the
|
|
246
|
+
# aff[:REP] table — Hunspell 1.7+ exposes `ph:` as additional REP
|
|
247
|
+
# patterns so the existing replchars permutation surfaces them. See
|
|
248
|
+
# {PhRepExtractor} for the three supported forms.
|
|
249
|
+
#
|
|
250
|
+
# @param words [Array<Word>] List of word entries
|
|
251
|
+
# @param aff [Hash] Aff structure being built (mutated to add REP)
|
|
252
|
+
# @return [Hash] Dic structure for Lookuper
|
|
253
|
+
def build_dic_structure(words, aff:)
|
|
254
|
+
word_index = Hash.new { |h, k| h[k] = [] }
|
|
255
|
+
lowercase_index = Hash.new { |h, k| h[k] = [] }
|
|
256
|
+
word_list = []
|
|
257
|
+
casing = aff[:casing]
|
|
258
|
+
ignore_chars = aff[:IGNORE] || []
|
|
259
|
+
|
|
260
|
+
words.each do |word|
|
|
261
|
+
ph_tokens = ph_tokens_from(word.morph_data)
|
|
262
|
+
# Hunspell applies IGNORE at read time: any char listed in IGNORE
|
|
263
|
+
# is stripped from dictionary stems so that input words with the
|
|
264
|
+
# same char removed (done in Lookuper#call) match. Without this,
|
|
265
|
+
# the `ignore` fixture's "expression" would never match a dict
|
|
266
|
+
# entry of "expression" — both need to be reduced to "exprssn".
|
|
267
|
+
stem = ignore_chars.any? ? strip_ignore(word.stem, ignore_chars) : word.stem
|
|
268
|
+
entry = {
|
|
269
|
+
stem: stem,
|
|
270
|
+
flags: word.flags.to_a,
|
|
271
|
+
alt_spellings: PhRepExtractor.simple_alt_spellings(stem, ph_tokens)
|
|
272
|
+
}
|
|
273
|
+
word_index[stem] << entry
|
|
274
|
+
word_list << entry
|
|
275
|
+
|
|
276
|
+
# The lowercase index mirrors Spylls's `Dic.lowercase_index`:
|
|
277
|
+
# only words whose casing is not NO (i.e. INIT/ALL/HUH/HUHINIT)
|
|
278
|
+
# contribute a lowercased key, because a NO-cased word's stem is
|
|
279
|
+
# already its own lowercase form and would just duplicate the
|
|
280
|
+
# exact-stem index entry.
|
|
281
|
+
captype = casing.guess(stem)
|
|
282
|
+
if captype != Algorithms::Capitalization::Type::NO
|
|
283
|
+
casing.lower(stem).each do |lowered|
|
|
284
|
+
lowercase_index[lowered] << entry
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
PhRepExtractor.append_to_aff(aff, stem, ph_tokens)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
{
|
|
292
|
+
words: word_list,
|
|
293
|
+
homonyms: ->(w, ignorecase: false) {
|
|
294
|
+
if ignorecase
|
|
295
|
+
lowercase_index[w] || []
|
|
296
|
+
else
|
|
297
|
+
word_index[w] || []
|
|
298
|
+
end
|
|
299
|
+
},
|
|
300
|
+
has_flag: ->(w, flag, for_all: false) {
|
|
301
|
+
entries = word_index[w] || []
|
|
302
|
+
# Spylls's `Dic.has_flag` returns False when there are no homonyms
|
|
303
|
+
# at all — a vacuous `for_all: true` would otherwise block the
|
|
304
|
+
# caller (e.g. the FORBIDDENWORD guard in Lookuper#call) on
|
|
305
|
+
# words whose case doesn't match the dictionary entry.
|
|
306
|
+
return false if entries.empty?
|
|
307
|
+
|
|
308
|
+
if for_all
|
|
309
|
+
entries.all? { |e| (e[:flags] || []).include?(flag) }
|
|
310
|
+
else
|
|
311
|
+
entries.any? { |e| (e[:flags] || []).include?(flag) }
|
|
312
|
+
end
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Select only the `ph:` morphological tokens.
|
|
318
|
+
#
|
|
319
|
+
# @param morph_data [Array<String>, nil] Morphological tokens
|
|
320
|
+
# @return [Array<String>] Raw `ph:` payloads (with the `ph:` prefix stripped)
|
|
321
|
+
def ph_tokens_from(morph_data)
|
|
322
|
+
return [] if morph_data.nil? || morph_data.empty?
|
|
323
|
+
|
|
324
|
+
morph_data.filter_map { |token| token[3..] if token.start_with?('ph:') }
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Remove IGNORE chars from a string. Mirrors Spylls's
|
|
328
|
+
# `word.translate(ignore.tr)` — used at read time for both stems and
|
|
329
|
+
# affix `add` strings so that lookups (which strip IGNORE from the
|
|
330
|
+
# input) match.
|
|
331
|
+
#
|
|
332
|
+
# @param str [String] Input string
|
|
333
|
+
# @param ignore_chars [Array<String>] Chars to remove
|
|
334
|
+
# @return [String] String with ignored chars removed
|
|
335
|
+
def strip_ignore(str, ignore_chars)
|
|
336
|
+
return str if ignore_chars.empty?
|
|
337
|
+
|
|
338
|
+
str.chars.reject { |c| ignore_chars.include?(c) }.join
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Build an affix hash for Lookuper.
|
|
342
|
+
#
|
|
343
|
+
# @param affix [Affix] The affix object
|
|
344
|
+
# @param script [Symbol] The script type for condition checking
|
|
345
|
+
# @param add [String, nil] Pre-stripped `add` string (IGNORE chars
|
|
346
|
+
# removed). When nil, the affix's raw `add` is used.
|
|
347
|
+
# @return [Hash] Affix hash for Lookuper
|
|
348
|
+
def build_affix_hash(affix, script: :latin, add: nil)
|
|
349
|
+
add ||= affix.add
|
|
350
|
+
{
|
|
351
|
+
flag: affix.flag,
|
|
352
|
+
crossproduct: affix.crossproduct,
|
|
353
|
+
strip: affix.strip,
|
|
354
|
+
affix: add,
|
|
355
|
+
condition_checker: compile_condition_matcher(affix.condition,
|
|
356
|
+
script: script,
|
|
357
|
+
type: affix.type),
|
|
358
|
+
affix_data: build_affix_transform(affix.strip, add, type: affix.type),
|
|
359
|
+
flags: affix.flags.to_a
|
|
360
|
+
}
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Compile a condition checker.
|
|
364
|
+
#
|
|
365
|
+
# Hunspell conditions are anchored: for suffixes the matcher checks
|
|
366
|
+
# the END of the stem (e.g. `[^y]$`), for prefixes the START
|
|
367
|
+
# (e.g. `^ij`). Without this distinction, prefix conditions silently
|
|
368
|
+
# never match — Dutch "ijs" with PFX condition `ij` would never get
|
|
369
|
+
# its prefix applied, so "IJs" wouldn't surface.
|
|
370
|
+
#
|
|
371
|
+
# @param condition [String] Condition string from .aff file
|
|
372
|
+
# @param script [Symbol] The script type (:latin, :arabic, etc.)
|
|
373
|
+
# @param type [Symbol] :prefix or :suffix (anchor direction)
|
|
374
|
+
# @return [ConditionChecker, nil] Compiled checker or nil
|
|
375
|
+
def compile_condition_matcher(condition, script: :latin, type: :suffix)
|
|
376
|
+
return nil if condition.nil? || condition.empty?
|
|
377
|
+
|
|
378
|
+
ConditionChecker.compile(condition, script: script, type: type)
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
# Build affix stripping data.
|
|
382
|
+
#
|
|
383
|
+
# Build affix transformation data.
|
|
384
|
+
#
|
|
385
|
+
# @param strip [String] Characters to strip
|
|
386
|
+
# @param add [String] Characters to add
|
|
387
|
+
# @param type [Symbol] :prefix or :suffix
|
|
388
|
+
# @return [Hash] Hash with affix data for transformation
|
|
389
|
+
def build_affix_transform(strip, add, type:)
|
|
390
|
+
return nil if strip.empty? && add.empty?
|
|
391
|
+
|
|
392
|
+
{
|
|
393
|
+
add: add,
|
|
394
|
+
strip: strip || '',
|
|
395
|
+
type: type
|
|
396
|
+
}
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# Build compound rules array.
|
|
400
|
+
#
|
|
401
|
+
# @param rules [Array<CompoundRule>] List of compound rules
|
|
402
|
+
# @return [Array<Hash>] Array of compound rule hashes
|
|
403
|
+
def build_compound_rules(rules)
|
|
404
|
+
return [] if rules.nil? || rules.empty?
|
|
405
|
+
|
|
406
|
+
rules.map do |rule|
|
|
407
|
+
{
|
|
408
|
+
text: rule.text,
|
|
409
|
+
flags: rule.flags,
|
|
410
|
+
full_match: ->(flag_sets) { rule.fullmatch(flag_sets) },
|
|
411
|
+
partial_match: ->(flag_sets) { rule.partial_match(flag_sets) }
|
|
412
|
+
}
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
# Build break patterns array.
|
|
417
|
+
#
|
|
418
|
+
# Spylls's default BREAK is `['-', '^-', '-$']` (applied when the
|
|
419
|
+
# directive is absent). If the .aff file says `BREAK 0` explicitly,
|
|
420
|
+
# the result must be EMPTY — `BREAK 0` is the documented way to
|
|
421
|
+
# disable hyphen breaking. The reader signals "absent" with nil and
|
|
422
|
+
# "explicit zero" with [], so we branch on nil here.
|
|
423
|
+
#
|
|
424
|
+
# @param break_patterns [Array<BreakPattern>, nil] List of break patterns
|
|
425
|
+
# @return [Array<Hash>] Array of break pattern hashes
|
|
426
|
+
def build_break_patterns(break_patterns)
|
|
427
|
+
if break_patterns.nil?
|
|
428
|
+
return [
|
|
429
|
+
{ pattern: '-', matcher: BreakPattern.new('-').matcher },
|
|
430
|
+
{ pattern: '^-', matcher: BreakPattern.new('^-').matcher },
|
|
431
|
+
{ pattern: '-$', matcher: BreakPattern.new('-$').matcher }
|
|
432
|
+
]
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
break_patterns.map do |bp|
|
|
436
|
+
{
|
|
437
|
+
pattern: bp.pattern,
|
|
438
|
+
matcher: bp.matcher
|
|
439
|
+
}
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Build REP table for the Suggester.
|
|
444
|
+
#
|
|
445
|
+
# The Suggester's `replchars` permutation expects each entry as a
|
|
446
|
+
# Hash with `:regexp` (a Regexp) and `:replacement` (a String).
|
|
447
|
+
# AffReader returns `Readers::RepPattern` value objects; this
|
|
448
|
+
# method adapts that representation without poking at the
|
|
449
|
+
# RepPattern's internals — it uses the public accessors.
|
|
450
|
+
#
|
|
451
|
+
# @param rep_patterns [Array<RepPattern>, nil] REP entries
|
|
452
|
+
# @return [Array<Hash{Symbol=>Object}>] REP table for Suggester
|
|
453
|
+
def build_rep_table(rep_patterns)
|
|
454
|
+
return [] if rep_patterns.nil? || rep_patterns.empty?
|
|
455
|
+
|
|
456
|
+
rep_patterns.map do |rp|
|
|
457
|
+
{
|
|
458
|
+
regexp: rp.matcher,
|
|
459
|
+
pattern: rp.pattern,
|
|
460
|
+
replacement: rp.replacement
|
|
461
|
+
}
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
# Extracts REP entries from dictionary `ph:` morphological data.
|
|
467
|
+
#
|
|
468
|
+
# Hunspell 1.7+ treats `ph:` as a phonetic misspelling hint and reuses
|
|
469
|
+
# the existing replchars permutation to surface corrections. To make
|
|
470
|
+
# that work, `ph:` payloads are converted to REP entries at dic-read
|
|
471
|
+
# time. Three payload forms are recognised (mirroring Spylls/Hunspell):
|
|
472
|
+
#
|
|
473
|
+
# * simple — `ph:wich` against stem "which" yields REP(`wich`,
|
|
474
|
+
# `which`) and registers `wich` as an alt_spelling for ngram
|
|
475
|
+
# scoring.
|
|
476
|
+
# * star — `ph:prity*` against stem "pretty" yields REP(`prit`,
|
|
477
|
+
# `prett`): the trailing char of the pattern (and the `*`) is
|
|
478
|
+
# stripped, and the trailing char of the stem is stripped, so the
|
|
479
|
+
# REP entry can attach suffixes on either side (matches "prity" →
|
|
480
|
+
# "pretty" and "pritiest" → "prettiest").
|
|
481
|
+
# * arrow — `ph:hepi->happi` is an explicit pair: REP(`hepi`,
|
|
482
|
+
# `happi`). The stem is irrelevant for arrow form.
|
|
483
|
+
#
|
|
484
|
+
# The class is a pure transformer; side effects on the aff hash are
|
|
485
|
+
# performed by the caller via {append_to_aff}.
|
|
486
|
+
module PhRepExtractor
|
|
487
|
+
module_function
|
|
488
|
+
|
|
489
|
+
# Return only the *simple* payloads — the ones that double as
|
|
490
|
+
# alt_spellings for ngram scoring. Star and arrow forms are excluded
|
|
491
|
+
# because their REP entries don't map 1:1 to the stem and would
|
|
492
|
+
# mislead ngram.
|
|
493
|
+
#
|
|
494
|
+
# @param _stem [String] Dictionary stem (unused, present for symmetry)
|
|
495
|
+
# @param ph_tokens [Array<String>] Raw `ph:` payloads
|
|
496
|
+
# @return [Array<String>] Simple alt spellings
|
|
497
|
+
def simple_alt_spellings(_stem, ph_tokens)
|
|
498
|
+
ph_tokens.reject { |token| token.end_with?('*') || token.include?('->') }
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
# Convert each `ph:` payload into a {RepPattern} and append it to
|
|
502
|
+
# `aff[:REP]`. Mutates the aff hash.
|
|
503
|
+
#
|
|
504
|
+
# @param aff [Hash] Aff structure (mutated)
|
|
505
|
+
# @param stem [String] Dictionary stem
|
|
506
|
+
# @param ph_tokens [Array<String>] Raw `ph:` payloads
|
|
507
|
+
# @return [void]
|
|
508
|
+
def append_to_aff(aff, stem, ph_tokens)
|
|
509
|
+
return if ph_tokens.nil? || ph_tokens.empty?
|
|
510
|
+
|
|
511
|
+
aff[:REP] ||= []
|
|
512
|
+
ph_tokens.each do |token|
|
|
513
|
+
rep = build_rep(stem, token)
|
|
514
|
+
aff[:REP] << rep_hash(rep) if rep
|
|
515
|
+
end
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Build a RepPattern from a single `ph:` payload.
|
|
519
|
+
#
|
|
520
|
+
# @param stem [String] Dictionary stem
|
|
521
|
+
# @param token [String] Raw `ph:` payload
|
|
522
|
+
# @return [RepPattern, nil]
|
|
523
|
+
def build_rep(stem, token)
|
|
524
|
+
if token.end_with?('*')
|
|
525
|
+
star_rep(stem, token)
|
|
526
|
+
elsif token.include?('->')
|
|
527
|
+
arrow_rep(token)
|
|
528
|
+
else
|
|
529
|
+
simple_rep(stem, token)
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
# Convert a RepPattern to the Hash shape used by Permutations.replchars.
|
|
534
|
+
#
|
|
535
|
+
# @param rep [RepPattern]
|
|
536
|
+
# @return [Hash{Symbol=>Object}]
|
|
537
|
+
def rep_hash(rep)
|
|
538
|
+
{
|
|
539
|
+
regexp: rep.matcher,
|
|
540
|
+
pattern: rep.pattern,
|
|
541
|
+
replacement: rep.replacement
|
|
542
|
+
}
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
# `ph:prity*` (against stem "pretty") → REP(`prit`, `prett`).
|
|
546
|
+
#
|
|
547
|
+
# Strip the last two chars of the pattern (`y*`) and the last char
|
|
548
|
+
# of the stem (`y`), so affixed forms on either side can match.
|
|
549
|
+
#
|
|
550
|
+
# @param stem [String] Dictionary stem
|
|
551
|
+
# @param token [String] Star-form payload (e.g. `prity*`)
|
|
552
|
+
# @return [RepPattern, nil] `nil` if pattern or stem too short
|
|
553
|
+
def star_rep(stem, token)
|
|
554
|
+
pattern = token[0...-2]
|
|
555
|
+
replacement = stem[0...-1]
|
|
556
|
+
return nil if pattern.empty? || replacement.empty?
|
|
557
|
+
|
|
558
|
+
RepPattern.new(pattern, replacement)
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# `ph:hepi->happi` → REP(`hepi`, `happi`).
|
|
562
|
+
#
|
|
563
|
+
# @param token [String] Arrow-form payload
|
|
564
|
+
# @return [RepPattern, nil] `nil` if either side is empty
|
|
565
|
+
def arrow_rep(token)
|
|
566
|
+
from, to = token.split('->', 2)
|
|
567
|
+
return nil if from.nil? || to.nil? || from.empty? || to.empty?
|
|
568
|
+
|
|
569
|
+
RepPattern.new(from, to)
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
# `ph:wich` (against stem "which") → REP(`wich`, `which`).
|
|
573
|
+
#
|
|
574
|
+
# @param stem [String] Dictionary stem
|
|
575
|
+
# @param token [String] Simple payload
|
|
576
|
+
# @return [RepPattern, nil] `nil` if pattern is empty
|
|
577
|
+
def simple_rep(stem, token)
|
|
578
|
+
return nil if token.empty?
|
|
579
|
+
|
|
580
|
+
RepPattern.new(token, stem)
|
|
581
|
+
end
|
|
582
|
+
end
|
|
583
|
+
end
|
|
584
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This file is intentionally a no-op stub.
|
|
4
|
+
#
|
|
5
|
+
# History: an earlier draft of the readers layer used `require_relative`
|
|
6
|
+
# here to eagerly load every reader file. The current design uses
|
|
7
|
+
# autoload declared in `lib/kotoshu/readers.rb` (the parent namespace
|
|
8
|
+
# file), so this nested file is redundant — and the `require_relative`
|
|
9
|
+
# calls violated the project rule against `require_relative` in lib/.
|
|
10
|
+
#
|
|
11
|
+
# Loading the original version of this file re-required already-loaded
|
|
12
|
+
# files (harmless but useless) and bypassed the autoload lazy-loading
|
|
13
|
+
# design. Rather than delete the file (project policy preserves files
|
|
14
|
+
# contributors may have local context for), it is reduced to this
|
|
15
|
+
# stub so any future `require` of it succeeds harmlessly.
|
|
16
|
+
#
|
|
17
|
+
# For the actual reader autoloads see: lib/kotoshu/readers.rb
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Hunspell readers module for reading dictionary and affix files.
|
|
5
|
+
module Readers
|
|
6
|
+
autoload :FileReader, "kotoshu/readers/file_reader"
|
|
7
|
+
autoload :StringReader, "kotoshu/readers/file_reader"
|
|
8
|
+
autoload :ZipReader, "kotoshu/readers/file_reader"
|
|
9
|
+
autoload :Affix, "kotoshu/readers/aff_data"
|
|
10
|
+
autoload :BreakPattern, "kotoshu/readers/aff_data"
|
|
11
|
+
autoload :Ignore, "kotoshu/readers/aff_data"
|
|
12
|
+
autoload :RepPattern, "kotoshu/readers/aff_data"
|
|
13
|
+
autoload :ConvTable, "kotoshu/readers/aff_data"
|
|
14
|
+
autoload :CompoundRule, "kotoshu/readers/aff_data"
|
|
15
|
+
autoload :CompoundPattern, "kotoshu/readers/aff_data"
|
|
16
|
+
autoload :PhonetTable, "kotoshu/readers/aff_data"
|
|
17
|
+
autoload :AffReader, "kotoshu/readers/aff_reader"
|
|
18
|
+
autoload :DicReader, "kotoshu/readers/dic_reader"
|
|
19
|
+
autoload :ConditionChecker, "kotoshu/readers/condition_checker"
|
|
20
|
+
autoload :PassthroughConditionChecker, "kotoshu/readers/condition_checker"
|
|
21
|
+
autoload :LatinScriptConditionChecker, "kotoshu/readers/condition_checker"
|
|
22
|
+
autoload :LookupBuilder, "kotoshu/readers/lookup_builder"
|
|
23
|
+
autoload :PhRepExtractor, "kotoshu/readers/lookup_builder"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# The Stage-2 result: every resource {ResourceManager.resolve} found
|
|
5
|
+
# for one language, ready to hand to `Spellchecker.new`.
|
|
6
|
+
#
|
|
7
|
+
# A member is nil for any resource that was not requested in the
|
|
8
|
+
# `want:` list or is not set up. The bundle is cache-only data —
|
|
9
|
+
# building one never touches the network.
|
|
10
|
+
#
|
|
11
|
+
# @example
|
|
12
|
+
# Kotoshu.setup(:en)
|
|
13
|
+
# bundle = Kotoshu::ResourceManager.resolve(language: "en")
|
|
14
|
+
# bundle.dictionary # => #<Kotoshu::Dictionary::Hunspell ...>
|
|
15
|
+
# bundle.has_model? # => false (model not requested)
|
|
16
|
+
ResourceBundle = Struct.new(
|
|
17
|
+
:language, # [String] Language code the bundle was resolved for
|
|
18
|
+
:dictionary, # [Dictionary::Hunspell, nil] Spelling dictionary
|
|
19
|
+
:frequency, # [Object, nil] Frequency data (Kelly tiers)
|
|
20
|
+
:model, # [Object, nil] ONNX embedding model
|
|
21
|
+
:rules, # [Object, nil] Grammar rules (reserved; nil today)
|
|
22
|
+
:cached, # [Boolean] True — resolve is cache-only by contract
|
|
23
|
+
:source_urls, # [Array<String>] URLs the resources came from
|
|
24
|
+
keyword_init: true
|
|
25
|
+
) do
|
|
26
|
+
# Whether the bundle's data came from the cache (always true for
|
|
27
|
+
# ResourceManager-resolved bundles; kept for forward compatibility).
|
|
28
|
+
#
|
|
29
|
+
# @return [Boolean]
|
|
30
|
+
def cached?
|
|
31
|
+
cached ? true : false
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Whether a frequency list is included.
|
|
35
|
+
#
|
|
36
|
+
# @return [Boolean]
|
|
37
|
+
def has_frequency?
|
|
38
|
+
!frequency.nil?
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Whether an ONNX model is included.
|
|
42
|
+
#
|
|
43
|
+
# @return [Boolean]
|
|
44
|
+
def has_model?
|
|
45
|
+
!model.nil?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Whether grammar rules are included (reserved; false today).
|
|
49
|
+
#
|
|
50
|
+
# @return [Boolean]
|
|
51
|
+
def has_rules?
|
|
52
|
+
!rules.nil?
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|