kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Readers
|
|
5
|
+
# AFF file reader for Hunspell affix files.
|
|
6
|
+
#
|
|
7
|
+
# This class reads .aff files and creates an Aff data structure.
|
|
8
|
+
#
|
|
9
|
+
# @example Reading an aff file
|
|
10
|
+
# reader = AffReader.new('en_US.aff')
|
|
11
|
+
# aff = reader.read
|
|
12
|
+
class AffReader
|
|
13
|
+
# Directives that are single boolean flags
|
|
14
|
+
BOOLEAN_DIRECTIVES = %w[
|
|
15
|
+
COMPLEXPREFIXES FULLSTRIP NOSPLITSUGS CHECKSHARPS
|
|
16
|
+
CHECKCOMPOUNDCASE CHECKCOMPOUNDDUP CHECKCOMPOUNDREP CHECKCOMPOUNDTRIPLE
|
|
17
|
+
SIMPLIFIEDTRIPLE ONLYMAXDIFF COMPOUNDMORESUFFIXES
|
|
18
|
+
].freeze
|
|
19
|
+
|
|
20
|
+
# Directives that are single string values
|
|
21
|
+
STRING_DIRECTIVES = %w[SET FLAG KEY TRY WORDCHARS LANG].freeze
|
|
22
|
+
|
|
23
|
+
# Directives that are single integer values
|
|
24
|
+
INTEGER_DIRECTIVES = %w[MAXDIFF MAXNGRAMSUGS MAXCPDSUGS COMPOUNDMIN COMPOUNDWORDMAX].freeze
|
|
25
|
+
|
|
26
|
+
# Directives that are single flag values
|
|
27
|
+
FLAG_DIRECTIVES = %w[
|
|
28
|
+
NOSUGGEST KEEPCASE CIRCUMFIX NEEDAFFIX FORBIDDENWORD WARN
|
|
29
|
+
COMPOUNDFLAG COMPOUNDBEGIN COMPOUNDMIDDLE COMPOUNDEND
|
|
30
|
+
ONLYINCOMPOUND COMPOUNDPERMITFLAG COMPOUNDFORBIDFLAG FORCEUCASE
|
|
31
|
+
SUBSTANDARD SYLLABLENUM COMPOUNDROOT
|
|
32
|
+
].freeze
|
|
33
|
+
|
|
34
|
+
# Outdated directive names and their synonyms
|
|
35
|
+
SYNONYMS = {
|
|
36
|
+
'PSEUDOROOT' => 'NEEDAFFIX',
|
|
37
|
+
'COMPOUNDLAST' => 'COMPOUNDEND'
|
|
38
|
+
}.freeze
|
|
39
|
+
|
|
40
|
+
attr_reader :path, :encoding, :flag_format
|
|
41
|
+
|
|
42
|
+
# Create a new AFF reader.
|
|
43
|
+
#
|
|
44
|
+
# @param path [String] Path to the .aff file
|
|
45
|
+
# @param encoding [String] File encoding (default: 'UTF-8');
|
|
46
|
+
# overridden by the file's SET directive when present
|
|
47
|
+
def initialize(path, encoding: 'UTF-8')
|
|
48
|
+
@path = path
|
|
49
|
+
@encoding = detect_encoding(path) || encoding
|
|
50
|
+
@flag_format = 'short'
|
|
51
|
+
@flag_synonyms = {}
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Read the aff file and return the aff data structure.
|
|
55
|
+
#
|
|
56
|
+
# @param source [FileReader, nil] Optional file reader to use instead of creating a new one
|
|
57
|
+
# @return [Hash] The aff data structure
|
|
58
|
+
def read(source = nil)
|
|
59
|
+
owned = source.nil?
|
|
60
|
+
reader = source || FileReader.new(@path, @encoding)
|
|
61
|
+
|
|
62
|
+
begin
|
|
63
|
+
data = {
|
|
64
|
+
'SFX' => {},
|
|
65
|
+
'PFX' => {},
|
|
66
|
+
'FLAG' => 'short'
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
reader.each do |_line_no, line|
|
|
70
|
+
dir_value = read_directive(reader, line)
|
|
71
|
+
next unless dir_value
|
|
72
|
+
|
|
73
|
+
directive, value = dir_value
|
|
74
|
+
|
|
75
|
+
# Update flag format when FLAG directive is encountered (BEFORE using it)
|
|
76
|
+
if directive == 'FLAG'
|
|
77
|
+
@flag_format = value
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Re-parse FLAG directive value now that @flag_format is updated
|
|
81
|
+
if directive == 'FLAG' && value.is_a?(String)
|
|
82
|
+
# No re-parsing needed for FLAG, just update the format
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# SFX/PFX have multiple entries
|
|
86
|
+
if %w[SFX PFX].include?(directive)
|
|
87
|
+
data[directive][value.first.flag] = value
|
|
88
|
+
else
|
|
89
|
+
data[directive] = value
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Update flag synonyms when AF directive is encountered (AFTER storing it)
|
|
93
|
+
if directive == 'AF'
|
|
94
|
+
@flag_synonyms = value
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Note: We don't reset_encoding during iteration because it closes
|
|
98
|
+
# the file and breaks the iteration. The FileReader is initialized
|
|
99
|
+
# with UTF-8 encoding which handles most cases.
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
data
|
|
103
|
+
ensure
|
|
104
|
+
# Close the reader only when we created it. If the caller passed
|
|
105
|
+
# one in, they own its lifecycle. Without this, the underlying
|
|
106
|
+
# File handle leaks (on Windows it also blocks tempfile cleanup).
|
|
107
|
+
reader.close if owned
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
private
|
|
112
|
+
|
|
113
|
+
# Read a directive from a line.
|
|
114
|
+
#
|
|
115
|
+
# @param reader [FileReader] The file reader
|
|
116
|
+
# @param line [String] The line to parse
|
|
117
|
+
# @return [Array, nil] [directive, value] or nil
|
|
118
|
+
def read_directive(reader, line)
|
|
119
|
+
parts = line.split(/\s+/)
|
|
120
|
+
return nil if parts.empty?
|
|
121
|
+
|
|
122
|
+
name = parts[0]
|
|
123
|
+
|
|
124
|
+
# Check if it looks like a directive (all caps)
|
|
125
|
+
return nil unless /^[A-Z]+$/.match?(name)
|
|
126
|
+
|
|
127
|
+
# Handle synonyms
|
|
128
|
+
name = SYNONYMS[name] || name
|
|
129
|
+
|
|
130
|
+
value = read_value(reader, name, parts[1..])
|
|
131
|
+
|
|
132
|
+
return nil if value.nil?
|
|
133
|
+
|
|
134
|
+
[name, value]
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Read the value for a directive.
|
|
138
|
+
#
|
|
139
|
+
# @param reader [FileReader] The file reader
|
|
140
|
+
# @param directive [String] The directive name
|
|
141
|
+
# @param values [Array<String>] Values from the line
|
|
142
|
+
# @return [Object] The parsed value
|
|
143
|
+
def read_value(reader, directive, values)
|
|
144
|
+
value = values.first
|
|
145
|
+
|
|
146
|
+
# String directives
|
|
147
|
+
if STRING_DIRECTIVES.include?(directive)
|
|
148
|
+
return value
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Integer directives
|
|
152
|
+
if INTEGER_DIRECTIVES.include?(directive)
|
|
153
|
+
return value&.to_i
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Flag directives
|
|
157
|
+
if FLAG_DIRECTIVES.include?(directive)
|
|
158
|
+
return parse_flag(value)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Boolean directives
|
|
162
|
+
if BOOLEAN_DIRECTIVES.include?(directive)
|
|
163
|
+
return true
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# IGNORE directive
|
|
167
|
+
if directive == 'IGNORE'
|
|
168
|
+
return Ignore.new(value || '')
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# BREAK directive
|
|
172
|
+
if directive == 'BREAK'
|
|
173
|
+
count = value&.to_i || 0
|
|
174
|
+
return read_array(reader, count).map { |parts| BreakPattern.new(parts.first) }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# COMPOUNDRULE directive
|
|
178
|
+
if directive == 'COMPOUNDRULE'
|
|
179
|
+
count = value&.to_i || 0
|
|
180
|
+
return read_array(reader, count).map { |parts| CompoundRule.new(parts.first) }
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# ICONV/OCONV directives
|
|
184
|
+
if %w[ICONV OCONV].include?(directive)
|
|
185
|
+
count = value&.to_i || 0
|
|
186
|
+
pairs = read_array(reader, count).map { |parts| [parts[0], parts[1] || ''] }
|
|
187
|
+
return ConvTable.new(pairs)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# REP directive
|
|
191
|
+
if directive == 'REP'
|
|
192
|
+
count = value&.to_i || 0
|
|
193
|
+
return read_array(reader, count).map { |parts| RepPattern.new(parts[0], parts[1] || '') }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# MAP directive
|
|
197
|
+
if directive == 'MAP'
|
|
198
|
+
count = value&.to_i || 0
|
|
199
|
+
return read_array(reader, count).map do |parts|
|
|
200
|
+
chars = parts.first || ''
|
|
201
|
+
# Parse MAP format: "aàâä" or "ß(ss)" - split by parentheses or individual chars
|
|
202
|
+
# Parenthesized groups like "(ss)" should be kept as a single string "ss"
|
|
203
|
+
chars.scan(/(\([^()]+\)|[^()])/).flatten.map do |group|
|
|
204
|
+
# Remove parentheses from parenthesized groups, keep as single string
|
|
205
|
+
# For single characters, keep as is
|
|
206
|
+
if group.start_with?('(') && group.end_with?(')')
|
|
207
|
+
group[1..-2] # Remove parentheses, keep content as single string
|
|
208
|
+
else
|
|
209
|
+
group # Keep single character as is
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# SFX/PFX directives
|
|
216
|
+
if %w[SFX PFX].include?(directive)
|
|
217
|
+
flag = values[0]
|
|
218
|
+
crossproduct = values[1]
|
|
219
|
+
count = values[2]&.to_i || 0
|
|
220
|
+
type = directive == 'PFX' ? :prefix : :suffix
|
|
221
|
+
|
|
222
|
+
affixes = read_array(reader, count).map do |parts|
|
|
223
|
+
# Format: FLAG strip add condition [morph_data]
|
|
224
|
+
# After read_array (which skips directive), parts[0] is FLAG again
|
|
225
|
+
# So we skip parts[0] and use: parts[1]=strip, parts[2]=add, parts[3]=condition
|
|
226
|
+
strip = parts[1] == '0' ? '' : (parts[1] || '')
|
|
227
|
+
add = parts[2] || ''
|
|
228
|
+
condition = parts[3] || '.'
|
|
229
|
+
|
|
230
|
+
# Handle flags in add field: "able/CD" -> add="able", flags=["C", "D"]
|
|
231
|
+
if add.include?('/')
|
|
232
|
+
add_str, _, flags_str = add.rpartition('/')
|
|
233
|
+
else
|
|
234
|
+
add_str = add
|
|
235
|
+
flags_str = ''
|
|
236
|
+
end
|
|
237
|
+
flags = flags_str.empty? ? Set.new : parse_flags(flags_str).to_set
|
|
238
|
+
|
|
239
|
+
Affix.new(
|
|
240
|
+
type:,
|
|
241
|
+
flag:,
|
|
242
|
+
crossproduct: crossproduct == 'Y',
|
|
243
|
+
strip:,
|
|
244
|
+
add: add_str == '0' ? '' : add_str,
|
|
245
|
+
condition:,
|
|
246
|
+
flags:
|
|
247
|
+
)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
return affixes
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# CHECKCOMPOUNDPATTERN directive
|
|
254
|
+
if directive == 'CHECKCOMPOUNDPATTERN'
|
|
255
|
+
count = value&.to_i || 0
|
|
256
|
+
return read_array(reader, count).map do |parts|
|
|
257
|
+
CompoundPattern.new(parts[0], parts[1] || '', parts[2])
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# AF directive (flag synonyms)
|
|
262
|
+
if directive == 'AF'
|
|
263
|
+
count = value&.to_i || 0
|
|
264
|
+
result = {}
|
|
265
|
+
read_array(reader, count).each_with_index do |parts, i|
|
|
266
|
+
# AF entries are decoded with the dictionary's FLAG format
|
|
267
|
+
# (spylls: Context.parse_flags on the entry value). With
|
|
268
|
+
# `FLAG num`, "AF 214,216" defines the two flags 214 and 216;
|
|
269
|
+
# with `FLAG long`, "AF g?1G" defines the two flags g? and 1G.
|
|
270
|
+
result[(i + 1).to_s] = parse_flags(parts.first).to_set
|
|
271
|
+
end
|
|
272
|
+
return result
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# AM directive
|
|
276
|
+
if directive == 'AM'
|
|
277
|
+
count = value&.to_i || 0
|
|
278
|
+
result = {}
|
|
279
|
+
read_array(reader, count).each_with_index do |parts, i|
|
|
280
|
+
result[(i + 1).to_s] = parts.to_set
|
|
281
|
+
end
|
|
282
|
+
return result
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# COMPOUNDSYLLABLE directive
|
|
286
|
+
if directive == 'COMPOUNDSYLLABLE'
|
|
287
|
+
return [value&.to_i, values[1]]
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# PHONE directive
|
|
291
|
+
if directive == 'PHONE'
|
|
292
|
+
count = value&.to_i || 0
|
|
293
|
+
table = read_array(reader, count).map { |parts| [parts[0], parts[1] || '_'] }
|
|
294
|
+
return PhonetTable.new(table)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Unknown directive - return nil
|
|
298
|
+
nil
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Read an array of values from the reader.
|
|
302
|
+
#
|
|
303
|
+
# @param reader [FileReader] The file reader
|
|
304
|
+
# @param count [Integer] Number of lines to read
|
|
305
|
+
# @return [Array<Array<String>>] Array of parsed lines
|
|
306
|
+
def read_array(reader, count)
|
|
307
|
+
result = []
|
|
308
|
+
count.times do
|
|
309
|
+
_, line = reader.next
|
|
310
|
+
parts = line.split(/\s+/)
|
|
311
|
+
# Skip the directive name at the beginning
|
|
312
|
+
result << parts[1..] if parts.length > 1
|
|
313
|
+
end
|
|
314
|
+
result
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Parse a single flag.
|
|
318
|
+
#
|
|
319
|
+
# @param string [String] Flag string
|
|
320
|
+
# @return [String] Parsed flag
|
|
321
|
+
def parse_flag(string)
|
|
322
|
+
parse_flags(string).first
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Parse multiple flags.
|
|
326
|
+
#
|
|
327
|
+
# @param string [String] Flag string
|
|
328
|
+
# @return [Array<String>] Parsed flags
|
|
329
|
+
def parse_flags(string)
|
|
330
|
+
return [] if string.nil? || string.empty?
|
|
331
|
+
|
|
332
|
+
# Check flag synonyms (only if the key exists in @flag_synonyms)
|
|
333
|
+
if @flag_synonyms&.key?(string)
|
|
334
|
+
return @flag_synonyms[string].to_a
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
case @flag_format
|
|
338
|
+
when 'short'
|
|
339
|
+
string.chars
|
|
340
|
+
when 'long'
|
|
341
|
+
string.scan(/../)
|
|
342
|
+
when 'num'
|
|
343
|
+
string.scan(/\d+/)
|
|
344
|
+
when 'UTF-8'
|
|
345
|
+
string.chars
|
|
346
|
+
else
|
|
347
|
+
raise ArgumentError, "Unknown flag format: #{@flag_format}"
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Detect the file's encoding.
|
|
352
|
+
#
|
|
353
|
+
# Strategy (matches Hunspell's actual behavior):
|
|
354
|
+
# 1. If the file has a `SET` directive, use that (normalized to Ruby form).
|
|
355
|
+
# 2. Else if the file's bytes are valid UTF-8, treat it as UTF-8.
|
|
356
|
+
# 3. Otherwise default to ISO-8859-1 (Latin-1), which is what Hunspell
|
|
357
|
+
# itself does for non-UTF-8 locales when `SET` is missing.
|
|
358
|
+
#
|
|
359
|
+
# Pre-scans the file in binary mode so we can reopen with the correct
|
|
360
|
+
# encoding before the FileReader consumes it.
|
|
361
|
+
#
|
|
362
|
+
# @param path [String] Path to the .aff file
|
|
363
|
+
# @return [String, nil] Encoding name (e.g., "ISO-8859-1", "UTF-8") or nil
|
|
364
|
+
def detect_encoding(path)
|
|
365
|
+
return nil if path.nil? || path.empty?
|
|
366
|
+
return nil unless File.file?(path)
|
|
367
|
+
|
|
368
|
+
snippet = File.binread(path)
|
|
369
|
+
set_match = snippet.match(/^SET\s+(\S+)/)
|
|
370
|
+
return normalize_encoding_name(set_match[1]) if set_match
|
|
371
|
+
|
|
372
|
+
return "UTF-8" if snippet.force_encoding("UTF-8").valid_encoding?
|
|
373
|
+
|
|
374
|
+
"ISO-8859-1"
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# Normalize Hunspell encoding names to Ruby encoding names.
|
|
378
|
+
#
|
|
379
|
+
# @param name [String] Hunspell encoding identifier
|
|
380
|
+
# @return [String] Ruby encoding name
|
|
381
|
+
def normalize_encoding_name(name)
|
|
382
|
+
return name if name.upcase == "UTF-8"
|
|
383
|
+
|
|
384
|
+
normalized = name.upcase.delete("-")
|
|
385
|
+
if normalized.start_with?("ISO8859")
|
|
386
|
+
"ISO-8859-#{normalized.sub('ISO8859', '')}"
|
|
387
|
+
else
|
|
388
|
+
name
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
end
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Readers
|
|
5
|
+
# Base class for checking affix conditions.
|
|
6
|
+
#
|
|
7
|
+
# Hunspell affix rules specify conditions that the stem must match
|
|
8
|
+
# before an affix can be applied. Different scripts may have different
|
|
9
|
+
# interpretations of these conditions.
|
|
10
|
+
#
|
|
11
|
+
# Conditions are anchored differently for prefixes vs suffixes:
|
|
12
|
+
# - Suffix condition `y` matches stems that END with `y` (e.g. `[^y]$`)
|
|
13
|
+
# - Prefix condition `ij` matches stems that START with `ij`
|
|
14
|
+
# (e.g. `^ij`)
|
|
15
|
+
#
|
|
16
|
+
# @example Latin script condition checking
|
|
17
|
+
# checker = LatinScriptConditionChecker.compile('[^y]', type: :suffix)
|
|
18
|
+
# checker.matches?('try') # => true (doesn't end with 'y')
|
|
19
|
+
# checker.matches?('fly') # => false (ends with 'y')
|
|
20
|
+
#
|
|
21
|
+
# @abstract Subclasses must implement the matches? method
|
|
22
|
+
class ConditionChecker
|
|
23
|
+
# Compile a condition string into a checker.
|
|
24
|
+
#
|
|
25
|
+
# @param condition [String] The condition string from .aff file
|
|
26
|
+
# @param script [Symbol] The script type (:latin, :arabic, :hebrew, etc.)
|
|
27
|
+
# @param type [Symbol] :suffix (anchor at end) or :prefix (anchor at start)
|
|
28
|
+
# @return [ConditionChecker] A checker instance
|
|
29
|
+
def self.compile(condition, script: :latin, type: :suffix)
|
|
30
|
+
case script
|
|
31
|
+
when :latin
|
|
32
|
+
LatinScriptConditionChecker.compile(condition, type: type)
|
|
33
|
+
else
|
|
34
|
+
# For other scripts, create a passthrough checker
|
|
35
|
+
# (condition is not applied)
|
|
36
|
+
PassthroughConditionChecker.new
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Check if the given stem matches this condition.
|
|
41
|
+
#
|
|
42
|
+
# @param stem [String] The stem to check
|
|
43
|
+
# @return [Boolean] True if the stem matches
|
|
44
|
+
def matches?(stem)
|
|
45
|
+
raise NotImplementedError, "#{self.class} must implement #matches?"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Passthrough condition checker that always returns true.
|
|
50
|
+
#
|
|
51
|
+
# Used for scripts where Hunspell conditions don't apply or aren't supported.
|
|
52
|
+
class PassthroughConditionChecker < ConditionChecker
|
|
53
|
+
def matches?(_stem)
|
|
54
|
+
true
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Condition checker for Latin-script dictionaries.
|
|
59
|
+
#
|
|
60
|
+
# Hunspell conditions are regex-like patterns. Spylls compiles them
|
|
61
|
+
# verbatim (with `-` escaped so it doesn't become a range inside an
|
|
62
|
+
# unintended context) into a regex anchored at the start (for prefixes)
|
|
63
|
+
# or end (for suffixes), and uses `re.search` to test.
|
|
64
|
+
#
|
|
65
|
+
# Examples (suffix unless noted):
|
|
66
|
+
# - '.' matches any stem (regex: /.$/ finds end-of-string)
|
|
67
|
+
# - 'y' matches stems ending with 'y' (regex: /y$/)
|
|
68
|
+
# - '[^y]' matches stems NOT ending with 'y' (regex: /[^y]$/)
|
|
69
|
+
# - '[aeiou]y' matches stems ending with vowel + 'y' (regex: /[aeiou]y$/)
|
|
70
|
+
# - '.[^aeiou]y' matches stems whose last 3 chars are any-non-vowel-y
|
|
71
|
+
# - 'wr.' (prefix) matches stems starting with 'wr' + any char
|
|
72
|
+
#
|
|
73
|
+
# This is NOT suitable for RTL scripts or CJK languages.
|
|
74
|
+
class LatinScriptConditionChecker < ConditionChecker
|
|
75
|
+
attr_reader :condition, :type, :anchor, :pattern
|
|
76
|
+
|
|
77
|
+
# Compile a condition string.
|
|
78
|
+
#
|
|
79
|
+
# @param condition [String] The condition string (e.g., '[^y]', '[abc]', 'y', '.')
|
|
80
|
+
# @param type [Symbol] :suffix (end-anchor) or :prefix (start-anchor)
|
|
81
|
+
# @return [LatinScriptConditionChecker] A checker instance
|
|
82
|
+
def self.compile(condition, type: :suffix)
|
|
83
|
+
anchor = type == :prefix ? :start : :ending
|
|
84
|
+
new(condition: condition, type: type, anchor: anchor)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def initialize(condition:, type:, anchor:)
|
|
88
|
+
@condition = condition
|
|
89
|
+
@type = type
|
|
90
|
+
@anchor = anchor
|
|
91
|
+
@pattern = compile_pattern
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Check if the stem matches the condition.
|
|
95
|
+
#
|
|
96
|
+
# @param stem [String] The stem to check
|
|
97
|
+
# @return [Boolean] True if the stem matches
|
|
98
|
+
def matches?(stem)
|
|
99
|
+
return true if @condition.nil? || @condition.empty? || @condition == '.'
|
|
100
|
+
|
|
101
|
+
@pattern.match?(stem)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
# Compile the condition into a Ruby Regexp.
|
|
107
|
+
#
|
|
108
|
+
# Mirrors Spylls: prefix → `^<condition>`, suffix → `<condition>$`,
|
|
109
|
+
# with `-` escaped so it can appear literally inside the pattern.
|
|
110
|
+
# The check uses `match?` (which searches), so the anchors do the
|
|
111
|
+
# positional work.
|
|
112
|
+
#
|
|
113
|
+
# Returns a match-all regex for nil/empty conditions so the
|
|
114
|
+
# constructor doesn't crash on those inputs (the #matches? method
|
|
115
|
+
# already short-circuits on nil/empty/`.`).
|
|
116
|
+
def compile_pattern
|
|
117
|
+
return // if @condition.nil? || @condition.empty?
|
|
118
|
+
|
|
119
|
+
escaped = @condition.gsub('-', '\\-')
|
|
120
|
+
anchored = @anchor == :start ? "^#{escaped}" : "#{escaped}$"
|
|
121
|
+
Regexp.new(anchored)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|