kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Dictionary
|
|
5
|
+
# Base class for all dictionary backends.
|
|
6
|
+
#
|
|
7
|
+
# This abstract class defines the interface that all dictionary
|
|
8
|
+
# implementations must follow.
|
|
9
|
+
#
|
|
10
|
+
# @note Subclasses must implement the abstract methods: {#lookup},
|
|
11
|
+
# {#suggest}, {#add_word}, and {#remove_word}.
|
|
12
|
+
#
|
|
13
|
+
# @example Implementing a custom dictionary
|
|
14
|
+
# class MyDictionary < Base
|
|
15
|
+
# def initialize(path, language_code:, locale: nil)
|
|
16
|
+
# super(language_code, locale: locale)
|
|
17
|
+
# @words = load_words(path)
|
|
18
|
+
# end
|
|
19
|
+
#
|
|
20
|
+
# def lookup(word)
|
|
21
|
+
# @words.include?(word.downcase)
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# # ... implement other abstract methods
|
|
25
|
+
# end
|
|
26
|
+
class Base
|
|
27
|
+
# @return [String] The language code (e.g., "en-US", "en-GB")
|
|
28
|
+
attr_reader :language_code
|
|
29
|
+
|
|
30
|
+
# @return [String, nil] The locale (e.g., "en", "en_US")
|
|
31
|
+
attr_reader :locale
|
|
32
|
+
|
|
33
|
+
# @return [String, nil] On-disk path for file-backed dictionaries.
|
|
34
|
+
# Nil for in-memory backends (e.g. Custom). Subclasses that load
|
|
35
|
+
# from a file override this with their own +attr_reader :path+.
|
|
36
|
+
def path
|
|
37
|
+
nil
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @return [Hash] Additional metadata
|
|
41
|
+
attr_reader :metadata
|
|
42
|
+
|
|
43
|
+
# Create a new dictionary.
|
|
44
|
+
#
|
|
45
|
+
# @param language_code [String] The language code (e.g., "en-US")
|
|
46
|
+
# @param locale [String, nil] The locale (optional)
|
|
47
|
+
# @param metadata [Hash] Additional metadata (optional)
|
|
48
|
+
def initialize(language_code, locale: nil, metadata: {})
|
|
49
|
+
raise ArgumentError, "Language code cannot be empty" if language_code.nil? || language_code.empty?
|
|
50
|
+
|
|
51
|
+
@language_code = language_code.dup.freeze
|
|
52
|
+
@locale = locale&.dup&.freeze
|
|
53
|
+
@metadata = metadata.dup.freeze
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Check if a word exists in the dictionary.
|
|
57
|
+
#
|
|
58
|
+
# @abstract Subclasses must implement this method.
|
|
59
|
+
# @param word [String] The word to look up
|
|
60
|
+
# @return [Boolean] True if the word exists
|
|
61
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
62
|
+
def lookup(word)
|
|
63
|
+
raise NotImplementedError, "#{self.class} must implement #lookup"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Check if a word exists in the dictionary (alias for lookup).
|
|
67
|
+
#
|
|
68
|
+
# Dispatches dynamically to whatever +lookup+ the concrete subclass
|
|
69
|
+
# defines. (Ruby's +alias+ keyword would bind to Base#lookup at
|
|
70
|
+
# definition time and bypass subclass overrides — these one-line
|
|
71
|
+
# method definitions are what makes the alias actually work.)
|
|
72
|
+
#
|
|
73
|
+
# @param word [String] The word to look up
|
|
74
|
+
# @return [Boolean] True if the word exists
|
|
75
|
+
def lookup?(word)
|
|
76
|
+
lookup(word)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def has_word?(word)
|
|
80
|
+
lookup(word)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def include?(word)
|
|
84
|
+
lookup(word)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def contains?(word)
|
|
88
|
+
lookup(word)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Generate spelling suggestions for a word.
|
|
92
|
+
#
|
|
93
|
+
# @abstract Subclasses must implement this method.
|
|
94
|
+
# @param word [String] The misspelled word
|
|
95
|
+
# @param max_suggestions [Integer] Maximum number of suggestions
|
|
96
|
+
# @return [Array<String>] List of suggested words
|
|
97
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
98
|
+
def suggest(word, max_suggestions: 10)
|
|
99
|
+
raise NotImplementedError, "#{self.class} must implement #suggest"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Add a word to the dictionary.
|
|
103
|
+
#
|
|
104
|
+
# @abstract Subclasses must implement this method.
|
|
105
|
+
# @param word [String] The word to add
|
|
106
|
+
# @param flags [Array<String>] Morphological flags (optional)
|
|
107
|
+
# @return [Boolean] True if the word was added
|
|
108
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
109
|
+
def add_word(word, flags: [])
|
|
110
|
+
raise NotImplementedError, "#{self.class} must implement #add_word"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def <<(word, flags: [])
|
|
114
|
+
add_word(word, flags: flags)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Remove a word from the dictionary.
|
|
118
|
+
#
|
|
119
|
+
# @abstract Subclasses must implement this method.
|
|
120
|
+
# @param word [String] The word to remove
|
|
121
|
+
# @return [Boolean] True if the word was removed
|
|
122
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
123
|
+
def remove_word(word)
|
|
124
|
+
raise NotImplementedError, "#{self.class} must implement #remove_word"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Get all words in the dictionary.
|
|
128
|
+
#
|
|
129
|
+
# @abstract Subclasses must implement this method.
|
|
130
|
+
# @return [Array<String>] All words
|
|
131
|
+
# @raise [NotImplementedError] Subclass must implement
|
|
132
|
+
def words
|
|
133
|
+
raise NotImplementedError, "#{self.class} must implement #words"
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def all_words
|
|
137
|
+
words
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# The dictionary's TRY string: characters to try when generating
|
|
141
|
+
# single-edit candidates (substitutions, insertions), from the
|
|
142
|
+
# Hunspell TRY directive. Backends without a TRY concept answer
|
|
143
|
+
# nil — callers must treat nil as "no substitution/insertion
|
|
144
|
+
# alphabet" and fall back to the edit operations that need no
|
|
145
|
+
# alphabet (transposition, deletion).
|
|
146
|
+
#
|
|
147
|
+
# @return [String, nil] TRY characters, or nil when unavailable
|
|
148
|
+
def try_string
|
|
149
|
+
nil
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# The sweep invariants for this dictionary's word list — char
|
|
153
|
+
# lengths, Soundex codes and length buckets — built once at the
|
|
154
|
+
# first suggestion sweep and shared by every later sweep (see
|
|
155
|
+
# {Suggestions::SweepIndex}).
|
|
156
|
+
#
|
|
157
|
+
# The index derives only from {#words}. Backends that mutate the
|
|
158
|
+
# word list (their {#add_word} / {#remove_word} implementations)
|
|
159
|
+
# call {#reset_sweep_index} so the memo cannot go stale; backends
|
|
160
|
+
# that cannot mutate after load (e.g. CSpell's frozen trie) never
|
|
161
|
+
# need to.
|
|
162
|
+
#
|
|
163
|
+
# @return [Suggestions::SweepIndex] The memoized sweep index
|
|
164
|
+
def sweep_index
|
|
165
|
+
@sweep_index ||= Suggestions::SweepIndex.build(words)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Drop the memoized {#sweep_index} so the next sweep rebuilds it
|
|
169
|
+
# from the current word list. Mutating backends call this after
|
|
170
|
+
# every successful word-list change.
|
|
171
|
+
#
|
|
172
|
+
# @return [self] Self for chaining
|
|
173
|
+
def reset_sweep_index
|
|
174
|
+
@sweep_index = nil
|
|
175
|
+
self
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Return words whose length is in [min_length, max_length].
|
|
179
|
+
#
|
|
180
|
+
# Uses the memoized {#sweep_index} length buckets — O(buckets in
|
|
181
|
+
# range) instead of an O(n) scan — and restores word-list order,
|
|
182
|
+
# exactly the set and order the historical whole-list filter
|
|
183
|
+
# produced. Subclasses that maintain their own length index
|
|
184
|
+
# (e.g. PlainText) keep their override.
|
|
185
|
+
#
|
|
186
|
+
# @param min_length [Integer] Minimum length (inclusive)
|
|
187
|
+
# @param max_length [Integer] Maximum length (inclusive)
|
|
188
|
+
# @return [Array<String>] Words in the length range
|
|
189
|
+
def find_by_length_range(min_length:, max_length:)
|
|
190
|
+
sweep_index.words_in_length_range(min_length, max_length)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Get the number of words in the dictionary.
|
|
194
|
+
#
|
|
195
|
+
# @return [Integer] Word count
|
|
196
|
+
def size
|
|
197
|
+
words.length
|
|
198
|
+
end
|
|
199
|
+
alias count size
|
|
200
|
+
alias length size
|
|
201
|
+
|
|
202
|
+
# Check if the dictionary is empty.
|
|
203
|
+
#
|
|
204
|
+
# @return [Boolean] True if empty
|
|
205
|
+
def empty?
|
|
206
|
+
size.zero?
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Iterate over all words.
|
|
210
|
+
#
|
|
211
|
+
# @yield [word] Each word
|
|
212
|
+
# @return [Enumerator] Enumerator if no block given
|
|
213
|
+
def each_word(&)
|
|
214
|
+
return enum_for(:each_word) unless block_given?
|
|
215
|
+
|
|
216
|
+
words.each(&)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Get words starting with a prefix.
|
|
220
|
+
#
|
|
221
|
+
# @param prefix [String] The prefix
|
|
222
|
+
# @return [Array<String>] Words with the prefix
|
|
223
|
+
def words_with_prefix(prefix)
|
|
224
|
+
words.select { |w| w.start_with?(prefix) }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Get words matching a pattern.
|
|
228
|
+
#
|
|
229
|
+
# @param pattern [Regexp] The pattern
|
|
230
|
+
# @return [Array<String>] Matching words
|
|
231
|
+
def words_matching(pattern)
|
|
232
|
+
words.select { |w| w.match?(pattern) }
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Convert to string.
|
|
236
|
+
#
|
|
237
|
+
# @return [String] String representation
|
|
238
|
+
def to_s
|
|
239
|
+
"#{self.class.name}(language: #{@language_code}, size: #{size})"
|
|
240
|
+
end
|
|
241
|
+
alias inspect to_s
|
|
242
|
+
|
|
243
|
+
# Dictionary type identifier.
|
|
244
|
+
#
|
|
245
|
+
# @return [Symbol] The dictionary type
|
|
246
|
+
def type
|
|
247
|
+
self.class.name.split("::").last.gsub(/(.)([A-Z])/, '\1_\2').downcase.to_sym
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Register this dictionary type.
|
|
251
|
+
#
|
|
252
|
+
# @param type_key [Symbol] The type key to register as
|
|
253
|
+
#
|
|
254
|
+
# @example Registering a custom dictionary type
|
|
255
|
+
# MyDictionary.register_type(:my_custom)
|
|
256
|
+
def self.register_type(type_key)
|
|
257
|
+
Kotoshu::Dictionary.register_type(type_key, self)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Class-level registry for dictionary types.
|
|
261
|
+
#
|
|
262
|
+
# @return [Hash] Registry of type keys to classes
|
|
263
|
+
def self.registry
|
|
264
|
+
@registry ||= {}
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
# Load a dictionary by type.
|
|
268
|
+
#
|
|
269
|
+
# @param type [Symbol] The dictionary type
|
|
270
|
+
# @param args [Array] Arguments to pass to constructor
|
|
271
|
+
# @return [Base] The loaded dictionary
|
|
272
|
+
# @raise [ConfigurationError] If type is not registered
|
|
273
|
+
def self.load(type, *args)
|
|
274
|
+
klass = registry[type]
|
|
275
|
+
raise ConfigurationError, "Unknown dictionary type: #{type}" unless klass
|
|
276
|
+
|
|
277
|
+
klass.new(*args)
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# Module-level registry for dictionary types.
|
|
282
|
+
#
|
|
283
|
+
# @return [Hash] Registry of type keys to classes
|
|
284
|
+
def self.registry
|
|
285
|
+
@registry ||= {}
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Reset the type registry. Used by specs that register throw-away
|
|
289
|
+
# dictionary classes; not part of the runtime contract.
|
|
290
|
+
#
|
|
291
|
+
# @return [void]
|
|
292
|
+
def self.reset_registry!
|
|
293
|
+
@registry = {}
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Register a dictionary type.
|
|
297
|
+
#
|
|
298
|
+
# @param type [Symbol] The type key
|
|
299
|
+
# @param klass [Class] The dictionary class
|
|
300
|
+
#
|
|
301
|
+
# @example Registering a custom dictionary type
|
|
302
|
+
# Dictionary.register_type(:my_custom, MyDictionary)
|
|
303
|
+
def self.register_type(type, klass)
|
|
304
|
+
@registry ||= {}
|
|
305
|
+
@registry[type] = klass
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Load a dictionary by type.
|
|
309
|
+
#
|
|
310
|
+
# @param type [Symbol] The dictionary type
|
|
311
|
+
# @param args [Array] Arguments to pass to constructor
|
|
312
|
+
# @return [Base] The loaded dictionary
|
|
313
|
+
#
|
|
314
|
+
# @example Loading a dictionary
|
|
315
|
+
# dict = Dictionary.load(:unix_words, "/usr/share/dict/words",
|
|
316
|
+
# language_code: "en-US")
|
|
317
|
+
def self.load(type, *args)
|
|
318
|
+
klass = registry[type]
|
|
319
|
+
raise ConfigurationError, "Unknown dictionary type: #{type}" unless klass
|
|
320
|
+
|
|
321
|
+
klass.new(*args)
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
end
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Dictionary
|
|
5
|
+
# CSpell dictionary backend.
|
|
6
|
+
#
|
|
7
|
+
# This dictionary reads CSpell-formatted dictionary files (plain text .txt
|
|
8
|
+
# or compressed .trie files). CSpell is the spell checker used by VS Code.
|
|
9
|
+
#
|
|
10
|
+
# File format:
|
|
11
|
+
# - .txt: Plain text with one word per line, # comments supported
|
|
12
|
+
# - .trie: Compressed trie format (DAFSA - Deterministic Acyclic Finite State Automaton)
|
|
13
|
+
#
|
|
14
|
+
# @example Creating from a text file
|
|
15
|
+
# dict = CSpell.new("words.txt", language_code: "en-US")
|
|
16
|
+
# dict.lookup?("hello") # => true
|
|
17
|
+
#
|
|
18
|
+
# @example Creating from a trie file
|
|
19
|
+
# dict = CSpell.new("words.trie", language_code: "en")
|
|
20
|
+
class CSpell < Base
|
|
21
|
+
# @return [String] The path to the dictionary file
|
|
22
|
+
attr_reader :path
|
|
23
|
+
|
|
24
|
+
# @return [Boolean] Whether lookups are case-sensitive
|
|
25
|
+
attr_reader :case_sensitive
|
|
26
|
+
|
|
27
|
+
# @return [Core::Trie::Trie] The trie data structure
|
|
28
|
+
attr_reader :trie
|
|
29
|
+
|
|
30
|
+
# Create a new CSpell dictionary.
|
|
31
|
+
#
|
|
32
|
+
# .trie). Nil when constructing from in-memory +words:+.
|
|
33
|
+
# @param language_code [String] The language code
|
|
34
|
+
# @param locale [String, nil] The locale (optional)
|
|
35
|
+
# @param case_sensitive [Boolean] Whether lookups are case-sensitive
|
|
36
|
+
# @param metadata [Hash] Additional metadata (optional)
|
|
37
|
+
# @param words [Array<String>, nil] Initial word list (bypasses
|
|
38
|
+
# file loading; used by .from_words).
|
|
39
|
+
def initialize(path = nil, language_code:, locale: nil, case_sensitive: false,
|
|
40
|
+
metadata: {}, words: nil)
|
|
41
|
+
super(language_code, locale: locale, metadata: metadata)
|
|
42
|
+
|
|
43
|
+
@path = path ? File.expand_path(path) : nil
|
|
44
|
+
@case_sensitive = case_sensitive
|
|
45
|
+
|
|
46
|
+
if words
|
|
47
|
+
normalized = words.map { |w| case_sensitive ? w : w.downcase }.uniq
|
|
48
|
+
@trie = Core::Trie::Builder.from_array(normalized)
|
|
49
|
+
else
|
|
50
|
+
raise DictionaryNotFoundError, @path unless File.exist?(@path)
|
|
51
|
+
|
|
52
|
+
@trie = if @path.end_with?(".trie")
|
|
53
|
+
load_trie_file(@path)
|
|
54
|
+
else
|
|
55
|
+
load_text_file(@path)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Register this dictionary type
|
|
60
|
+
self.class.register_type(:cspell) unless Dictionary.registry.key?(:cspell)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Check if a word exists in the dictionary.
|
|
64
|
+
#
|
|
65
|
+
# @param word [String] The word to look up
|
|
66
|
+
# @return [Boolean] True if the word exists
|
|
67
|
+
def lookup(word)
|
|
68
|
+
return false if word.nil? || word.empty?
|
|
69
|
+
|
|
70
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
71
|
+
@trie.lookup(lookup_word)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Check if the dictionary has words with a prefix.
|
|
75
|
+
#
|
|
76
|
+
# @param prefix [String] The prefix
|
|
77
|
+
# @return [Boolean] True if words exist with the prefix
|
|
78
|
+
def has_prefix?(prefix)
|
|
79
|
+
return false if prefix.nil? || prefix.empty?
|
|
80
|
+
|
|
81
|
+
lookup_prefix = @case_sensitive ? prefix : prefix.downcase
|
|
82
|
+
@trie.has_prefix?(lookup_prefix)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Generate spelling suggestions.
|
|
86
|
+
#
|
|
87
|
+
# Uses trie walk to find similar words.
|
|
88
|
+
#
|
|
89
|
+
# @param word [String] The misspelled word
|
|
90
|
+
# @param max_suggestions [Integer] Maximum suggestions
|
|
91
|
+
# @return [Array<String>] List of suggested words
|
|
92
|
+
def suggest(word, max_suggestions: 10)
|
|
93
|
+
return [] if word.nil? || word.empty?
|
|
94
|
+
|
|
95
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
96
|
+
|
|
97
|
+
# First try prefix-based suggestions
|
|
98
|
+
prefix_suggestions = @trie.suggestions(lookup_word, max_results: max_suggestions)
|
|
99
|
+
|
|
100
|
+
# If we have enough prefix suggestions, return them
|
|
101
|
+
return prefix_suggestions if prefix_suggestions.length >= max_suggestions
|
|
102
|
+
|
|
103
|
+
# Otherwise, use edit distance for more suggestions
|
|
104
|
+
all_words = @trie.all_words
|
|
105
|
+
candidates = all_words.select do |w|
|
|
106
|
+
w.length >= lookup_word.length - 2 &&
|
|
107
|
+
w.length <= lookup_word.length + 2
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Calculate edit distances
|
|
111
|
+
results = candidates.map do |dict_word|
|
|
112
|
+
dist = edit_distance(lookup_word, dict_word)
|
|
113
|
+
[dict_word, dist]
|
|
114
|
+
end.select { |_, dist| dist.positive? && dist <= 2 }
|
|
115
|
+
.sort_by { |_, dist| dist }
|
|
116
|
+
.first(max_suggestions - prefix_suggestions.length)
|
|
117
|
+
.map(&:first)
|
|
118
|
+
|
|
119
|
+
# Combine both sets
|
|
120
|
+
(prefix_suggestions + results).uniq.first(max_suggestions)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Add a word to the dictionary.
|
|
124
|
+
#
|
|
125
|
+
# @param word [String] The word to add
|
|
126
|
+
# @param flags [Array<String>] Flags (ignored for CSpell)
|
|
127
|
+
# @return [Boolean] Always false — CSpell dictionaries are loaded
|
|
128
|
+
# into a frozen Trie (see Core::Trie::Builder#build) and cannot
|
|
129
|
+
# be mutated after load. The Trie itself rejects inserts with
|
|
130
|
+
# +FrozenError+; this method short-circuits so callers see the
|
|
131
|
+
# same "no" answer they would from {#remove_word}.
|
|
132
|
+
def add_word(word, flags: [])
|
|
133
|
+
false
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Remove a word from the dictionary.
|
|
137
|
+
#
|
|
138
|
+
# @param _word [String] The word to remove (unused — the trie
|
|
139
|
+
# does not support removal)
|
|
140
|
+
# @return [Boolean] Always false; trie dictionaries are immutable
|
|
141
|
+
# @note CSpell dictionaries are typically immutable after loading
|
|
142
|
+
def remove_word(_word)
|
|
143
|
+
# Trie doesn't support removal easily
|
|
144
|
+
# Would need to rebuild the trie
|
|
145
|
+
false
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Get all words in the dictionary.
|
|
149
|
+
#
|
|
150
|
+
# @return [Array<String>] All words
|
|
151
|
+
def words
|
|
152
|
+
@trie.all_words
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Get words with a prefix.
|
|
156
|
+
#
|
|
157
|
+
# @param prefix [String] The prefix
|
|
158
|
+
# @return [Array<String>] Words with the prefix
|
|
159
|
+
def words_with_prefix(prefix)
|
|
160
|
+
return [] if prefix.nil? || prefix.empty?
|
|
161
|
+
|
|
162
|
+
lookup_prefix = @case_sensitive ? prefix : prefix.downcase
|
|
163
|
+
@trie.words_with_prefix(lookup_prefix)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Create a dictionary from an array of words.
|
|
167
|
+
#
|
|
168
|
+
# @param words [Array<String>] The words
|
|
169
|
+
# @param language_code [String] The language code
|
|
170
|
+
# @param locale [String, nil] The locale (optional)
|
|
171
|
+
# @param case_sensitive [Boolean] Whether lookups are case-sensitive
|
|
172
|
+
# @return [CSpell] New dictionary
|
|
173
|
+
#
|
|
174
|
+
# @example
|
|
175
|
+
# dict = CSpell.from_words(%w[hello world test], language_code: "en")
|
|
176
|
+
def self.from_words(words, language_code:, locale: nil, case_sensitive: false)
|
|
177
|
+
new(words: words, language_code: language_code,
|
|
178
|
+
locale: locale, case_sensitive: case_sensitive)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
private
|
|
182
|
+
|
|
183
|
+
# Load a text dictionary file.
|
|
184
|
+
#
|
|
185
|
+
# @param path [String] The file path
|
|
186
|
+
# @return [Core::Trie::Trie] The loaded trie
|
|
187
|
+
def load_text_file(path)
|
|
188
|
+
words = File.foreach(path, chomp: true)
|
|
189
|
+
.reject { |line| line.empty? || line.strip.empty? || line.strip.start_with?("#") }
|
|
190
|
+
.map(&:strip)
|
|
191
|
+
.map { |word| @case_sensitive ? word : word.downcase }
|
|
192
|
+
.uniq
|
|
193
|
+
|
|
194
|
+
Core::Trie::Builder.from_array(words)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Load a compressed trie file.
|
|
198
|
+
#
|
|
199
|
+
# @param path [String] The file path
|
|
200
|
+
# @return [Core::Trie::Trie] The loaded trie
|
|
201
|
+
#
|
|
202
|
+
# @note For now, this falls back to treating the file as text.
|
|
203
|
+
# Full .trie format support would require implementing DAFSA decompression.
|
|
204
|
+
def load_trie_file(path)
|
|
205
|
+
# For now, treat as text file
|
|
206
|
+
# Full implementation would parse the CSpell .trie format
|
|
207
|
+
# which uses DAFSA (Deterministic Acyclic Finite State Automaton) compression
|
|
208
|
+
load_text_file(path)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Calculate Levenshtein edit distance.
|
|
212
|
+
#
|
|
213
|
+
# @param str1 [String] First string
|
|
214
|
+
# @param str2 [String] Second string
|
|
215
|
+
# @return [Integer] Edit distance
|
|
216
|
+
def edit_distance(str1, str2)
|
|
217
|
+
return str2.length if str1.empty?
|
|
218
|
+
return str1.length if str2.empty?
|
|
219
|
+
|
|
220
|
+
# Use smaller string for inner loop
|
|
221
|
+
str1, str2 = str2, str1 if str1.length > str2.length
|
|
222
|
+
|
|
223
|
+
previous = (0..str1.length).to_a
|
|
224
|
+
|
|
225
|
+
str2.each_char.with_index do |char2, j|
|
|
226
|
+
current = [j + 1]
|
|
227
|
+
|
|
228
|
+
str1.each_char.with_index do |char1, i|
|
|
229
|
+
insert_cost = current[i] + 1
|
|
230
|
+
delete_cost = previous[i + 1] + 1
|
|
231
|
+
substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
|
|
232
|
+
|
|
233
|
+
current << [insert_cost, delete_cost, substitute_cost].min
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
previous = current
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
previous.last
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|