kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'zip'
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Dictionary
|
|
7
|
+
# Spylls-style facade over Hunspell-format .aff/.dic dictionary
|
|
8
|
+
# files. Distinct from {Kotoshu::Dictionary::Hunspell} (which is a
|
|
9
|
+
# backend subclass of {Dictionary::Base}) and from
|
|
10
|
+
# {Kotoshu::Spellchecker} (the high-level facade): this class is the
|
|
11
|
+
# low-level entry point for callers that want direct access to the
|
|
12
|
+
# underlying Lookuper / Suggester without the surrounding
|
|
13
|
+
# configuration / personal-dictionary / suggestion-generator
|
|
14
|
+
# machinery.
|
|
15
|
+
#
|
|
16
|
+
# Was previously declared as `class Kotoshu::Dictionary`, which
|
|
17
|
+
# collided with the {Dictionary} module that holds the backend
|
|
18
|
+
# namespace — Ruby raised TypeError on autoload. Renamed to
|
|
19
|
+
# {Dictionary::Unified} so it loads cleanly under the existing
|
|
20
|
+
# `autoload :Unified, "kotoshu/dictionary/unified"` entry.
|
|
21
|
+
#
|
|
22
|
+
# @example Loading from files
|
|
23
|
+
# dictionary = Dictionary::Unified.from_files('/path/to/dictionary/en_US')
|
|
24
|
+
# dictionary.lookup('spells') # => true
|
|
25
|
+
#
|
|
26
|
+
# @example Loading from zip archive
|
|
27
|
+
# dictionary = Dictionary::Unified.from_zip('/path/to/dictionary/en_US.odt')
|
|
28
|
+
#
|
|
29
|
+
# @example Loading from system
|
|
30
|
+
# dictionary = Dictionary::Unified.from_system('en_US')
|
|
31
|
+
#
|
|
32
|
+
# @example Getting suggestions
|
|
33
|
+
# dictionary.suggest('spylls') # => ["spells", "spills", ...]
|
|
34
|
+
#
|
|
35
|
+
# @example Accessing algorithms for experimentation
|
|
36
|
+
# dictionary.lookuper.good_forms('building') do |form|
|
|
37
|
+
# puts form
|
|
38
|
+
# end
|
|
39
|
+
class Unified
|
|
40
|
+
# System paths to search for Hunspell dictionaries
|
|
41
|
+
PATHES = [
|
|
42
|
+
'/usr/share/hunspell',
|
|
43
|
+
'/usr/share/myspell',
|
|
44
|
+
'/usr/share/myspell/dicts',
|
|
45
|
+
'/Library/Spelling',
|
|
46
|
+
'/opt/openoffice.org/basis3.0/share/dict/ooo',
|
|
47
|
+
'/usr/lib/openoffice.org/basis3.0/share/dict/ooo',
|
|
48
|
+
'/opt/openoffice.org2.4/share/dict/ooo',
|
|
49
|
+
'/usr/lib/openoffice.org2.4/share/dict/ooo',
|
|
50
|
+
'/opt/openoffice.org2.3/share/dict/ooo',
|
|
51
|
+
'/usr/lib/openoffice.org2.3/share/dict/ooo',
|
|
52
|
+
'/opt/openoffice.org2.2/share/dict/ooo',
|
|
53
|
+
'/usr/lib/openoffice.org2.2/share/dict/ooo',
|
|
54
|
+
'/opt/openoffice.org2.1/share/dict/ooo',
|
|
55
|
+
'/usr/lib/openoffice.org2.1/share/dict/ooo',
|
|
56
|
+
'/opt/openoffice.org2.0/share/dict/ooo',
|
|
57
|
+
'/usr/lib/openoffice.org2.0/share/dict/ooo'
|
|
58
|
+
].freeze
|
|
59
|
+
|
|
60
|
+
# Distributed dictionaries for testing
|
|
61
|
+
DISTRIBUTED = {
|
|
62
|
+
'en_US' => 'en',
|
|
63
|
+
'ru' => 'ru',
|
|
64
|
+
'sv_SE' => 'sv'
|
|
65
|
+
}.freeze
|
|
66
|
+
|
|
67
|
+
# @return [Hash] Aff data structure
|
|
68
|
+
attr_reader :aff
|
|
69
|
+
|
|
70
|
+
# @return [Array<Readers::Word>] Dic data structure
|
|
71
|
+
attr_reader :dic_words
|
|
72
|
+
|
|
73
|
+
# @return [Algorithms::Lookup::Lookuper] Lookuper instance for experimentation
|
|
74
|
+
attr_reader :lookuper
|
|
75
|
+
|
|
76
|
+
# @return [Algorithms::Suggest::Suggester] Suggester instance for experimentation
|
|
77
|
+
attr_reader :suggester
|
|
78
|
+
|
|
79
|
+
# Create a Dictionary from aff and dic data.
|
|
80
|
+
#
|
|
81
|
+
# @param aff [Hash] Aff data structure
|
|
82
|
+
# @param dic_words [Array<Readers::Word>] Dictionary word entries
|
|
83
|
+
def initialize(aff, dic_words)
|
|
84
|
+
@aff = aff
|
|
85
|
+
@dic_words = dic_words
|
|
86
|
+
|
|
87
|
+
# Build the Lookuper via LookupBuilder — it owns the aff →
|
|
88
|
+
# structure transformation (casing, suffixes/prefixes indexes,
|
|
89
|
+
# REP patterns, etc.). The Suggester needs that processed aff
|
|
90
|
+
# structure too, so we read it back off the Lookuper rather
|
|
91
|
+
# than passing the raw AffReader hash.
|
|
92
|
+
@lookuper = Readers::LookupBuilder.from_data(aff, dic_words).build
|
|
93
|
+
@suggester = Algorithms::Suggest::Suggester.new(
|
|
94
|
+
@lookuper.aff, @lookuper.dic, @lookuper
|
|
95
|
+
)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Load dictionary from file path.
|
|
99
|
+
#
|
|
100
|
+
# The path should be the base name without extension, e.g., 'en_US'
|
|
101
|
+
# for files 'en_US.aff' and 'en_US.dic'.
|
|
102
|
+
#
|
|
103
|
+
# @param path [String] Base path to dictionary files (without extension)
|
|
104
|
+
# @return [Dictionary] The loaded dictionary
|
|
105
|
+
#
|
|
106
|
+
# @example
|
|
107
|
+
# Dictionary.from_files('en_US')
|
|
108
|
+
def self.from_files(path)
|
|
109
|
+
# Check if it's a distributed dictionary
|
|
110
|
+
if DISTRIBUTED.key?(path) && !File.exist?("#{path}.aff")
|
|
111
|
+
distributed_path = File.join(File.dirname(__FILE__), '../../data', DISTRIBUTED[path], path)
|
|
112
|
+
if File.exist?("#{distributed_path}.aff")
|
|
113
|
+
path = distributed_path
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
aff_path = "#{path}.aff"
|
|
118
|
+
dic_path = "#{path}.dic"
|
|
119
|
+
|
|
120
|
+
raise ArgumentError, "Dictionary file not found: #{aff_path}" unless File.exist?(aff_path)
|
|
121
|
+
raise ArgumentError, "Dictionary file not found: #{dic_path}" unless File.exist?(dic_path)
|
|
122
|
+
|
|
123
|
+
# Read aff file
|
|
124
|
+
aff_reader = Readers::AffReader.new(aff_path)
|
|
125
|
+
aff_data = aff_reader.read
|
|
126
|
+
|
|
127
|
+
# Read dic file
|
|
128
|
+
dic_reader = Readers::DicReader.new(dic_path,
|
|
129
|
+
flag_format: aff_data['FLAG'] || 'short',
|
|
130
|
+
flag_synonyms: aff_data['AF'] || {})
|
|
131
|
+
dic_words = dic_reader.read
|
|
132
|
+
|
|
133
|
+
new(aff_data, dic_words)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Load dictionary from zip archive.
|
|
137
|
+
#
|
|
138
|
+
# Supports OpenOffice/LibreOffice dictionary extensions (.odt, .oxt)
|
|
139
|
+
# and Firefox/Thunderbird dictionary extensions (.xpi).
|
|
140
|
+
#
|
|
141
|
+
# @param zip_path [String] Path to zip archive
|
|
142
|
+
# @return [Dictionary] The loaded dictionary
|
|
143
|
+
#
|
|
144
|
+
# @example
|
|
145
|
+
# Dictionary.from_zip('en_US.odt')
|
|
146
|
+
def self.from_zip(zip_path)
|
|
147
|
+
Zip::File.open(zip_path) do |zipfile|
|
|
148
|
+
# Find .aff and .dic files
|
|
149
|
+
aff_entry = nil
|
|
150
|
+
dic_entry = nil
|
|
151
|
+
|
|
152
|
+
zipfile.each do |entry|
|
|
153
|
+
if entry.name.end_with?('.aff')
|
|
154
|
+
raise ArgumentError, "Multiple .aff files found in zip" if aff_entry
|
|
155
|
+
|
|
156
|
+
aff_entry = entry
|
|
157
|
+
elsif entry.name.end_with?('.dic')
|
|
158
|
+
raise ArgumentError, "Multiple .dic files found in zip" if dic_entry
|
|
159
|
+
|
|
160
|
+
dic_entry = entry
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
raise ArgumentError, "No .aff file found in zip" unless aff_entry
|
|
165
|
+
raise ArgumentError, "No .dic file found in zip" unless dic_entry
|
|
166
|
+
|
|
167
|
+
# Read aff file
|
|
168
|
+
aff_reader = Readers::ZipReader.new(zipfile, aff_entry.name)
|
|
169
|
+
aff_reader.to_a
|
|
170
|
+
# Parse the raw data into proper aff structure
|
|
171
|
+
Readers::AffReader.new(zip_path) # Temporary for context
|
|
172
|
+
aff_data = Readers::AffReader.new(aff_entry.name).read
|
|
173
|
+
|
|
174
|
+
# Read dic file
|
|
175
|
+
dic_reader = Readers::DicReader.new(dic_entry.name,
|
|
176
|
+
flag_format: aff_data['FLAG'] || 'short',
|
|
177
|
+
flag_synonyms: aff_data['AF'] || {})
|
|
178
|
+
dic_words = dic_reader.read
|
|
179
|
+
|
|
180
|
+
new(aff_data, dic_words)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Load dictionary from system paths.
|
|
185
|
+
#
|
|
186
|
+
# Searches standard system locations for Hunspell dictionaries.
|
|
187
|
+
#
|
|
188
|
+
# @param name [String] Dictionary name (e.g., 'en_US', 'ru_RU')
|
|
189
|
+
# @return [Dictionary] The loaded dictionary
|
|
190
|
+
# @raise [ArgumentError] If dictionary not found in system paths
|
|
191
|
+
#
|
|
192
|
+
# @example
|
|
193
|
+
# Dictionary.from_system('en_US')
|
|
194
|
+
def self.from_system(name)
|
|
195
|
+
PATHES.each do |folder|
|
|
196
|
+
aff_path = File.join(folder, "#{name}.aff")
|
|
197
|
+
if File.exist?(aff_path)
|
|
198
|
+
base_path = aff_path.sub(/\.aff$/, '')
|
|
199
|
+
return from_files(base_path)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
raise ArgumentError, "#{name}.aff not found in system paths: #{PATHES.inspect}"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Check if a word is correct.
|
|
207
|
+
#
|
|
208
|
+
# @param word [String] Word to check
|
|
209
|
+
# @return [Boolean] True if the word exists in the dictionary
|
|
210
|
+
#
|
|
211
|
+
# @example
|
|
212
|
+
# dictionary.lookup('spells') # => true
|
|
213
|
+
# dictionary.lookup('spylls') # => false
|
|
214
|
+
def lookup(word)
|
|
215
|
+
@lookuper.call(word)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Generate suggestions for a misspelled word.
|
|
219
|
+
#
|
|
220
|
+
# Returns suggestions in order of probability/similarity,
|
|
221
|
+
# with best suggestions first.
|
|
222
|
+
#
|
|
223
|
+
# @param word [String] The misspelled word
|
|
224
|
+
# @yield [String] Each suggestion
|
|
225
|
+
# @return [Enumerator] If no block given
|
|
226
|
+
#
|
|
227
|
+
# @example
|
|
228
|
+
# dictionary.suggest('spylls') # => ["spells", "spills", ...]
|
|
229
|
+
def suggest(word, &)
|
|
230
|
+
return enum_for(:suggest, word) unless block_given?
|
|
231
|
+
|
|
232
|
+
@suggester.suggestions(word, &)
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
end
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Dictionary
|
|
5
|
+
# Unix system dictionary backend.
|
|
6
|
+
#
|
|
7
|
+
# This dictionary reads from Unix-style system dictionary files,
|
|
8
|
+
# typically located at `/usr/share/dict/words` or symlinks to
|
|
9
|
+
# dictionaries like `web2` (Webster's Second International).
|
|
10
|
+
#
|
|
11
|
+
# @example Using system dictionary
|
|
12
|
+
# dict = UnixWords.new("/usr/share/dict/words", language_code: "en-US")
|
|
13
|
+
# dict.lookup?("hello") # => true
|
|
14
|
+
# dict.suggest("helo") # => ["hello", "help", "held", ...]
|
|
15
|
+
#
|
|
16
|
+
# @example Auto-detecting system dictionary
|
|
17
|
+
# dict = UnixWords.detect(language_code: "en-US")
|
|
18
|
+
class UnixWords < Base
|
|
19
|
+
# Standard system paths to check for dictionaries.
|
|
20
|
+
SYSTEM_PATHS = [
|
|
21
|
+
"/usr/share/dict/words",
|
|
22
|
+
"/usr/share/dict/web2",
|
|
23
|
+
"/usr/share/dict/american-english",
|
|
24
|
+
"/usr/share/dict/british-english",
|
|
25
|
+
"/usr/dict/words",
|
|
26
|
+
"/System/Library/Assets/com_apple_MobileAsset_DictionaryServices_dictionaryOS/Dictionary/words" # macOS
|
|
27
|
+
].freeze
|
|
28
|
+
|
|
29
|
+
# @return [String] The path to the dictionary file
|
|
30
|
+
attr_reader :path
|
|
31
|
+
|
|
32
|
+
# @return [Boolean] Whether lookups are case-sensitive
|
|
33
|
+
attr_reader :case_sensitive
|
|
34
|
+
|
|
35
|
+
# Create a new UnixWords dictionary.
|
|
36
|
+
#
|
|
37
|
+
# @param path [String] Path to the dictionary file
|
|
38
|
+
# @param language_code [String] The language code
|
|
39
|
+
# @param locale [String, nil] The locale (optional)
|
|
40
|
+
# @param case_sensitive [Boolean] Whether lookups are case-sensitive
|
|
41
|
+
# @param metadata [Hash] Additional metadata (optional)
|
|
42
|
+
def initialize(path, language_code:, locale: nil, case_sensitive: false, metadata: {})
|
|
43
|
+
super(language_code, locale: locale, metadata: metadata)
|
|
44
|
+
|
|
45
|
+
@path = File.expand_path(path)
|
|
46
|
+
@case_sensitive = case_sensitive
|
|
47
|
+
@words = load_words(@path)
|
|
48
|
+
@word_set = build_word_set
|
|
49
|
+
|
|
50
|
+
# Register this dictionary type
|
|
51
|
+
self.class.register_type(:unix_words) unless Dictionary.registry.key?(:unix_words)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Check if a word exists in the dictionary.
|
|
55
|
+
#
|
|
56
|
+
# @param word [String] The word to look up
|
|
57
|
+
# @return [Boolean] True if the word exists
|
|
58
|
+
def lookup(word)
|
|
59
|
+
return false if word.nil? || word.empty?
|
|
60
|
+
|
|
61
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
62
|
+
@word_set.key?(lookup_word)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Generate spelling suggestions.
|
|
66
|
+
#
|
|
67
|
+
# Uses edit distance to find similar words in the dictionary.
|
|
68
|
+
#
|
|
69
|
+
# @param word [String] The misspelled word
|
|
70
|
+
# @param max_suggestions [Integer] Maximum suggestions
|
|
71
|
+
# @return [Array<String>] List of suggested words
|
|
72
|
+
def suggest(word, max_suggestions: 10)
|
|
73
|
+
return [] if word.nil? || word.empty?
|
|
74
|
+
|
|
75
|
+
# For now, use simple prefix matching and edit distance
|
|
76
|
+
# This will be improved with the suggestion algorithms
|
|
77
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
78
|
+
|
|
79
|
+
# Find words with same prefix
|
|
80
|
+
prefix_len = [lookup_word.length - 1, 3].max
|
|
81
|
+
prefix = lookup_word[0...prefix_len]
|
|
82
|
+
candidates = @words.select { |w| w.start_with?(prefix) }
|
|
83
|
+
|
|
84
|
+
# Calculate edit distances
|
|
85
|
+
candidates.map do |dict_word|
|
|
86
|
+
dist = edit_distance(lookup_word, dict_word)
|
|
87
|
+
[dict_word, dist]
|
|
88
|
+
end.select { |_, dist| dist.positive? && dist <= 2 } # Only close matches
|
|
89
|
+
.sort_by { |_, dist| dist }
|
|
90
|
+
.first(max_suggestions)
|
|
91
|
+
.map(&:first)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Add a word to the dictionary.
|
|
95
|
+
#
|
|
96
|
+
# @param word [String] The word to add
|
|
97
|
+
# @param flags [Array<String>] Flags (ignored for UnixWords)
|
|
98
|
+
# @return [Boolean] True if added
|
|
99
|
+
def add_word(word, flags: [])
|
|
100
|
+
return false if word.nil? || word.empty?
|
|
101
|
+
|
|
102
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
103
|
+
return false if @word_set.key?(lookup_word)
|
|
104
|
+
|
|
105
|
+
@words << lookup_word
|
|
106
|
+
@word_set[lookup_word] = true
|
|
107
|
+
reset_sweep_index
|
|
108
|
+
|
|
109
|
+
true
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Remove a word from the dictionary.
|
|
113
|
+
#
|
|
114
|
+
# @param word [String] The word to remove
|
|
115
|
+
# @return [Boolean] True if removed
|
|
116
|
+
def remove_word(word)
|
|
117
|
+
return false if word.nil? || word.empty?
|
|
118
|
+
|
|
119
|
+
lookup_word = @case_sensitive ? word : word.downcase
|
|
120
|
+
return false unless @word_set.key?(lookup_word)
|
|
121
|
+
|
|
122
|
+
@word_set.delete(lookup_word)
|
|
123
|
+
@words.delete(lookup_word)
|
|
124
|
+
reset_sweep_index
|
|
125
|
+
|
|
126
|
+
true
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Get all words in the dictionary.
|
|
130
|
+
#
|
|
131
|
+
# @return [Array<String>] All words
|
|
132
|
+
def words
|
|
133
|
+
@words.dup
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Detect system dictionary path.
|
|
137
|
+
#
|
|
138
|
+
# Checks standard system paths for an existing dictionary file.
|
|
139
|
+
#
|
|
140
|
+
# @return [String, nil] The detected path or nil
|
|
141
|
+
#
|
|
142
|
+
# @example
|
|
143
|
+
# UnixWords.detect_system_dictionary # => "/usr/share/dict/words"
|
|
144
|
+
def self.detect_system_dictionary
|
|
145
|
+
SYSTEM_PATHS.find { |p| File.exist?(p) }
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Create a dictionary by auto-detecting system dictionary.
|
|
149
|
+
#
|
|
150
|
+
# @param language_code [String] The language code
|
|
151
|
+
# @param locale [String, nil] The locale (optional)
|
|
152
|
+
# @param case_sensitive [Boolean] Whether lookups are case-sensitive
|
|
153
|
+
# @return [UnixWords, nil] The dictionary or nil if not found
|
|
154
|
+
#
|
|
155
|
+
# @example
|
|
156
|
+
# dict = UnixWords.detect(language_code: "en-US")
|
|
157
|
+
def self.detect(language_code:, locale: nil, case_sensitive: false)
|
|
158
|
+
path = detect_system_dictionary
|
|
159
|
+
return nil unless path
|
|
160
|
+
|
|
161
|
+
new(path, language_code: language_code, locale: locale,
|
|
162
|
+
case_sensitive: case_sensitive)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
private
|
|
166
|
+
|
|
167
|
+
# Load words from dictionary file.
|
|
168
|
+
#
|
|
169
|
+
# @param path [String] The file path
|
|
170
|
+
# @return [Array<String>] List of words
|
|
171
|
+
def load_words(path)
|
|
172
|
+
raise DictionaryNotFoundError, path unless File.exist?(path)
|
|
173
|
+
|
|
174
|
+
File.foreach(path, chomp: true)
|
|
175
|
+
.reject { |line| line.empty? || line.start_with?("#") }
|
|
176
|
+
.map { |word| @case_sensitive ? word : word.downcase }
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Build a hash set for O(1) lookups.
|
|
180
|
+
#
|
|
181
|
+
# @return [Hash] Word membership hash (word => true)
|
|
182
|
+
def build_word_set
|
|
183
|
+
@words.to_h { |word| [word, true] }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Calculate Levenshtein edit distance.
|
|
187
|
+
#
|
|
188
|
+
# @param str1 [String] First string
|
|
189
|
+
# @param str2 [String] Second string
|
|
190
|
+
# @return [Integer] Edit distance
|
|
191
|
+
def edit_distance(str1, str2)
|
|
192
|
+
return str2.length if str1.empty?
|
|
193
|
+
return str1.length if str2.empty?
|
|
194
|
+
|
|
195
|
+
# Use smaller string for inner loop
|
|
196
|
+
str1, str2 = str2, str1 if str1.length > str2.length
|
|
197
|
+
|
|
198
|
+
previous = (0..str1.length).to_a
|
|
199
|
+
|
|
200
|
+
str2.each_char.with_index do |char2, j|
|
|
201
|
+
current = [j + 1]
|
|
202
|
+
|
|
203
|
+
str1.each_char.with_index do |char1, i|
|
|
204
|
+
insert_cost = current[i] + 1
|
|
205
|
+
delete_cost = previous[i + 1] + 1
|
|
206
|
+
substitute_cost = previous[i] + (char1 == char2 ? 0 : 1)
|
|
207
|
+
|
|
208
|
+
current << [insert_cost, delete_cost, substitute_cost].min
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
previous = current
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
previous.last
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Dictionary backends for different formats (Hunspell, CSpell, plain text, etc.).
|
|
5
|
+
module Dictionary
|
|
6
|
+
autoload :Base, "kotoshu/dictionary/base"
|
|
7
|
+
autoload :CSpell, "kotoshu/dictionary/cspell"
|
|
8
|
+
autoload :Custom, "kotoshu/dictionary/custom"
|
|
9
|
+
autoload :Hunspell, "kotoshu/dictionary/hunspell"
|
|
10
|
+
autoload :PlainText, "kotoshu/dictionary/plain_text"
|
|
11
|
+
autoload :Repository, "kotoshu/dictionary/repository"
|
|
12
|
+
autoload :Unified, "kotoshu/dictionary/unified"
|
|
13
|
+
autoload :UnixWords, "kotoshu/dictionary/unix_words"
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
# Document abstraction for structure-aware spell and grammar
|
|
6
|
+
# checking. Pairs the flattened text a checker scans with the
|
|
7
|
+
# source positions of each {TextNode}, so errors can be reported
|
|
8
|
+
# against the original markup-bearing source.
|
|
9
|
+
#
|
|
10
|
+
# Subclasses (or plugins) populate +text_nodes+ with one entry per
|
|
11
|
+
# contiguous text run, each carrying the {SourceRange} it occupies
|
|
12
|
+
# in the source. The base class provides the bidirectional
|
|
13
|
+
# offset-mapping logic — it walks the nodes to recover a source
|
|
14
|
+
# range from a flattened offset.
|
|
15
|
+
#
|
|
16
|
+
# Kotoshu ships only {PlainTextDocument}; format-specific parsers
|
|
17
|
+
# (Markdown, AsciiDoc, etc.) live in plugins.
|
|
18
|
+
class Document
|
|
19
|
+
attr_reader :text_nodes, :source, :format, :language_code, :suppressions
|
|
20
|
+
|
|
21
|
+
# @param text_nodes [Array<TextNode>] one entry per contiguous
|
|
22
|
+
# text run, in source order
|
|
23
|
+
# @param source [String, nil] the original markup-bearing source
|
|
24
|
+
# (kept around so consumers can slice it for context display)
|
|
25
|
+
# @param format [Symbol] e.g. :plain, :markdown, :asciidoc
|
|
26
|
+
# @param language_code [String, nil] ISO 639-1 code when known
|
|
27
|
+
# @param suppressions [Array<Suppressions::Suppression>, nil] inline
|
|
28
|
+
# ignore directives; nil means scan +source+ with +format+ comment
|
|
29
|
+
# syntax (plan 82)
|
|
30
|
+
def initialize(text_nodes:, source: nil, format: :plain, language_code: nil, suppressions: nil)
|
|
31
|
+
raise ArgumentError, "text_nodes cannot be empty" if text_nodes.nil? || text_nodes.empty?
|
|
32
|
+
|
|
33
|
+
@text_nodes = text_nodes.freeze
|
|
34
|
+
@source = source
|
|
35
|
+
@format = format
|
|
36
|
+
@language_code = language_code
|
|
37
|
+
@suppressions = (suppressions || Suppressions.scan(source, format: format)).freeze
|
|
38
|
+
freeze
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The flattened text — concatenation of every node's +text+.
|
|
42
|
+
# This is what the checker actually scans.
|
|
43
|
+
#
|
|
44
|
+
# @return [String]
|
|
45
|
+
def flattened_text
|
|
46
|
+
@text_nodes.map(&:text).join
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Total character length of the flattened text.
|
|
50
|
+
def flattened_length
|
|
51
|
+
@text_nodes.sum(&:length)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# True when +offset+ falls within the flattened text.
|
|
55
|
+
def flattened_offset?(offset)
|
|
56
|
+
offset >= 0 && offset < flattened_length
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Recover the {SourceRange} of the {TextNode} that contains the
|
|
60
|
+
# given flattened offset. Returns nil for out-of-range offsets.
|
|
61
|
+
#
|
|
62
|
+
# Note: this returns the *node's* source range, not a one-char
|
|
63
|
+
# slice. For a tighter range, use {#source_range_for} with a
|
|
64
|
+
# flattened sub-range.
|
|
65
|
+
#
|
|
66
|
+
# @param flattened_offset [Integer] 0-based offset in flattened text
|
|
67
|
+
# @return [SourceRange, nil]
|
|
68
|
+
def source_range_at(flattened_offset)
|
|
69
|
+
return nil unless flattened_offset?(flattened_offset)
|
|
70
|
+
|
|
71
|
+
node = node_at(flattened_offset)
|
|
72
|
+
node&.source_range
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Recover the {SourceRange} spanning a flattened sub-range.
|
|
76
|
+
# The start of the result is the start of the node containing
|
|
77
|
+
# +flattened_start+; the end is the end of the node containing
|
|
78
|
+
# +flattened_end - 1+. Useful for errors that span multiple
|
|
79
|
+
# text nodes (e.g. a grammar error like "an **friend**" that
|
|
80
|
+
# crosses a markup boundary).
|
|
81
|
+
#
|
|
82
|
+
# @param flattened_start [Integer] inclusive
|
|
83
|
+
# @param flattened_end [Integer] exclusive; pass nil for "to end"
|
|
84
|
+
# @return [SourceRange, nil]
|
|
85
|
+
def source_range_for(flattened_start, flattened_end = nil)
|
|
86
|
+
return nil unless flattened_offset?(flattened_start)
|
|
87
|
+
|
|
88
|
+
end_offset = flattened_end ? flattened_end - 1 : flattened_length - 1
|
|
89
|
+
end_offset = [end_offset, flattened_length - 1].min
|
|
90
|
+
return nil unless flattened_offset?(end_offset)
|
|
91
|
+
|
|
92
|
+
start_node = node_at(flattened_start)
|
|
93
|
+
end_node = node_at(end_offset)
|
|
94
|
+
return nil unless start_node && end_node
|
|
95
|
+
|
|
96
|
+
start_node.source_range.union(end_node.source_range)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Iterate text nodes. Returns an Enumerator when no block given.
|
|
100
|
+
def each_node(&block)
|
|
101
|
+
return enum_for(:each_node) unless block
|
|
102
|
+
|
|
103
|
+
@text_nodes.each(&block)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# True when +word+ (any word when nil) reported on 1-based +line+
|
|
107
|
+
# is suppressed by an inline ignore directive (plan 82).
|
|
108
|
+
def suppressed?(line, word: nil)
|
|
109
|
+
@suppressions.any? { |suppression| suppression.applies_to?(line, word: word) }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
# Find the TextNode whose flattened range contains +offset+.
|
|
115
|
+
def node_at(offset)
|
|
116
|
+
@text_nodes.find { |node| node.contains_flattened?(offset) }
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
# Built-in {Document} parser for plain text (no markup).
|
|
6
|
+
#
|
|
7
|
+
# The only parser kotoshu ships. Format-specific parsers (Markdown,
|
|
8
|
+
# AsciiDoc, etc.) live in plugins — see {Documents.register}.
|
|
9
|
+
#
|
|
10
|
+
# PlainTextDocument is intentionally simple: one {TextNode}
|
|
11
|
+
# covering the entire source. The flattened text equals the source,
|
|
12
|
+
# so source-to-flattened mapping is the identity.
|
|
13
|
+
class PlainTextDocument < Document
|
|
14
|
+
# Build a document from a string.
|
|
15
|
+
#
|
|
16
|
+
# @param text [String]
|
|
17
|
+
# @param language_code [String, nil]
|
|
18
|
+
# @return [PlainTextDocument]
|
|
19
|
+
# +suppressions+ defaults to scanning the text for bare
|
|
20
|
+
# directives (plan 82); pass [] to attach none.
|
|
21
|
+
def self.from_string(text, language_code: nil, suppressions: nil)
|
|
22
|
+
text ||= ""
|
|
23
|
+
range = source_range_for_string(text, 0)
|
|
24
|
+
new(
|
|
25
|
+
text_nodes: [TextNode.new(text: text, source_range: range,
|
|
26
|
+
flattened_offset: 0, format: :plain, metadata: {})],
|
|
27
|
+
source: text,
|
|
28
|
+
format: :plain,
|
|
29
|
+
language_code: language_code,
|
|
30
|
+
suppressions: suppressions
|
|
31
|
+
)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Build a document from a file.
|
|
35
|
+
#
|
|
36
|
+
# @param path [String]
|
|
37
|
+
# @param language_code [String, nil]
|
|
38
|
+
# @return [PlainTextDocument]
|
|
39
|
+
def self.from_file(path, language_code: nil)
|
|
40
|
+
from_string(File.read(path), language_code: language_code)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
class << self
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
# Compute the SourceRange for +text+ starting at +base_offset+
|
|
47
|
+
# in the source. Walks the string once to find the final line
|
|
48
|
+
# and column.
|
|
49
|
+
def source_range_for_string(text, base_offset)
|
|
50
|
+
start_pos = SourcePosition.new(offset: base_offset, line: 1, column: 1)
|
|
51
|
+
|
|
52
|
+
line = 1
|
|
53
|
+
column = 1
|
|
54
|
+
text.each_char do |char|
|
|
55
|
+
if char == "\n"
|
|
56
|
+
line += 1
|
|
57
|
+
column = 1
|
|
58
|
+
else
|
|
59
|
+
column += 1
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
end_pos = SourcePosition.new(
|
|
64
|
+
offset: base_offset + text.length,
|
|
65
|
+
line: line,
|
|
66
|
+
column: column
|
|
67
|
+
)
|
|
68
|
+
SourceRange.new(start_pos: start_pos, end_pos: end_pos)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|