kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Single source of truth for where each remote resource lives.
|
|
5
|
+
#
|
|
6
|
+
# Every URL the cache layer fetches is built here. Caches do not
|
|
7
|
+
# construct URL strings inline. Per-repo pins honor that
|
|
8
|
+
# `kotoshu/dictionaries` ships on `v1` while the other repos are on
|
|
9
|
+
# `main`, which previously caused silent 404s on first-use.
|
|
10
|
+
#
|
|
11
|
+
# @example
|
|
12
|
+
# registry = Kotoshu::SourceRegistry.new
|
|
13
|
+
# registry.url_for(:spelling, lang: "en", ext: "aff")
|
|
14
|
+
# # => "https://raw.githubusercontent.com/kotoshu/dictionaries/v1/en/spelling/index.aff"
|
|
15
|
+
class SourceRegistry
|
|
16
|
+
Source = Struct.new(:repo, :default_pin, :template, :lfs, keyword_init: true)
|
|
17
|
+
|
|
18
|
+
DEFAULT_BASE_URL = "https://raw.githubusercontent.com/kotoshu"
|
|
19
|
+
|
|
20
|
+
# Host serving Git-LFS object content. raw.githubusercontent.com serves
|
|
21
|
+
# the 134-byte pointer stub for LFS-tracked files, so LFS sources
|
|
22
|
+
# (*.onnx in models-fasttext-onnx) must be fetched from here.
|
|
23
|
+
MEDIA_BASE_URL = "https://media.githubusercontent.com/media/kotoshu"
|
|
24
|
+
|
|
25
|
+
# @return [Hash<Symbol, Source>]
|
|
26
|
+
SOURCES = {
|
|
27
|
+
spelling: Source.new(repo: "dictionaries", default_pin: "v1",
|
|
28
|
+
template: "dictionaries/%<pin>s/%<lang>s/spelling/index.%<ext>s"),
|
|
29
|
+
# Flat layout: only en ships under the spelling/ sublayout; the
|
|
30
|
+
# staged languages sit at {lang}/index.*. The cache layer tries
|
|
31
|
+
# spelling first and falls back to this.
|
|
32
|
+
spelling_flat: Source.new(repo: "dictionaries", default_pin: "v1",
|
|
33
|
+
template: "dictionaries/%<pin>s/%<lang>s/index.%<ext>s"),
|
|
34
|
+
grammar: Source.new(repo: "dictionaries", default_pin: "v1",
|
|
35
|
+
template: "dictionaries/%<pin>s/%<lang>s/grammar/rules.yaml"),
|
|
36
|
+
dict_manifest: Source.new(repo: "dictionaries", default_pin: "v1",
|
|
37
|
+
template: "dictionaries/%<pin>s/manifest.json"),
|
|
38
|
+
frequency: Source.new(repo: "frequency-list-kelly", default_pin: "main",
|
|
39
|
+
template: "frequency-list-kelly/%<pin>s/data/%<lang>s.json"),
|
|
40
|
+
freq_manifest: Source.new(repo: "frequency-list-kelly", default_pin: "main",
|
|
41
|
+
template: "frequency-list-kelly/%<pin>s/manifest.json"),
|
|
42
|
+
model: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
|
|
43
|
+
template: "models-fasttext-onnx/%<pin>s/models/%<lang>s/fasttext.%<lang>s.onnx",
|
|
44
|
+
lfs: true),
|
|
45
|
+
model_vocab: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
|
|
46
|
+
template: "models-fasttext-onnx/%<pin>s/models/%<lang>s/fasttext.%<lang>s.vocab.json"),
|
|
47
|
+
model_manifest: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
|
|
48
|
+
template: "models-fasttext-onnx/%<pin>s/manifest.json"),
|
|
49
|
+
model_registry: Source.new(repo: "models-fasttext-onnx", default_pin: "main",
|
|
50
|
+
template: "models-fasttext-onnx/%<pin>s/registry.json")
|
|
51
|
+
}.freeze
|
|
52
|
+
|
|
53
|
+
# @param base_url [String] GitHub raw root, no trailing slash.
|
|
54
|
+
# @param pins [Hash<String, String>] Optional per-repo pin overrides
|
|
55
|
+
# keyed by repo name (e.g. `{ "dictionaries" => "v2" }`).
|
|
56
|
+
def initialize(base_url: DEFAULT_BASE_URL, pins: {})
|
|
57
|
+
@base_url = base_url.to_s.chomp("/")
|
|
58
|
+
@pins = pins.transform_keys(&:to_s).freeze
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @return [String] Configured GitHub raw root (no trailing slash).
|
|
62
|
+
attr_reader :base_url
|
|
63
|
+
|
|
64
|
+
# @param source_key [Symbol] One of `SOURCES.keys`.
|
|
65
|
+
# @param lang [String, nil] Language code, interpolated into template.
|
|
66
|
+
# @param ext [String, nil] File extension, interpolated into template.
|
|
67
|
+
# @return [String] Fully-qualified URL.
|
|
68
|
+
def url_for(source_key, lang: nil, ext: nil)
|
|
69
|
+
source = SOURCES.fetch(source_key) do
|
|
70
|
+
raise ArgumentError, "unknown source: #{source_key.inspect}"
|
|
71
|
+
end
|
|
72
|
+
path = source.template % { pin: pin_for(source), lang: lang, ext: ext }
|
|
73
|
+
"#{base_for(source)}/#{path}"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @param source_key [Symbol]
|
|
77
|
+
# @return [String] Resolved pin (override or default).
|
|
78
|
+
def pin_for_source(source_key)
|
|
79
|
+
source = SOURCES.fetch(source_key)
|
|
80
|
+
pin_for(source)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @param source_key [Symbol]
|
|
84
|
+
# @return [String] Repo name (e.g. "dictionaries").
|
|
85
|
+
def repo_for(source_key)
|
|
86
|
+
SOURCES.fetch(source_key).repo
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
|
|
91
|
+
def pin_for(source)
|
|
92
|
+
@pins.fetch(source.repo, source.default_pin)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Base URL for a source. LFS-tracked content is only served by the
|
|
96
|
+
# media host — but only when we are on the default GitHub host; a
|
|
97
|
+
# custom mirror base is left alone (the mirror resolves LFS itself).
|
|
98
|
+
def base_for(source)
|
|
99
|
+
return MEDIA_BASE_URL if source.lfs && @base_url == DEFAULT_BASE_URL
|
|
100
|
+
|
|
101
|
+
@base_url
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
class Spellchecker
|
|
5
|
+
# Parallel file checker for concurrent spellchecking.
|
|
6
|
+
#
|
|
7
|
+
# Uses a thread pool to check multiple files simultaneously,
|
|
8
|
+
# providing significant speedup on multi-core systems.
|
|
9
|
+
#
|
|
10
|
+
# @example Check files in parallel
|
|
11
|
+
# checker = ParallelChecker.new(spellchecker: spellchecker, worker_count: 4)
|
|
12
|
+
# results = checker.check_files_parallel(["file1.txt", "file2.txt"])
|
|
13
|
+
class ParallelChecker
|
|
14
|
+
# Default number of worker threads
|
|
15
|
+
DEFAULT_WORKER_COUNT = 4
|
|
16
|
+
|
|
17
|
+
# @return [Spellchecker] The underlying spellchecker
|
|
18
|
+
attr_reader :spellchecker
|
|
19
|
+
|
|
20
|
+
# @return [Integer] Number of worker threads
|
|
21
|
+
attr_reader :worker_count
|
|
22
|
+
|
|
23
|
+
# Create a new parallel checker.
|
|
24
|
+
#
|
|
25
|
+
# @param spellchecker [Spellchecker] The spellchecker to use
|
|
26
|
+
# @param worker_count [Integer] Number of worker threads (default: 4)
|
|
27
|
+
def initialize(spellchecker:, worker_count: DEFAULT_WORKER_COUNT)
|
|
28
|
+
@spellchecker = spellchecker
|
|
29
|
+
@worker_count = worker_count
|
|
30
|
+
@queue = Queue.new
|
|
31
|
+
@results = []
|
|
32
|
+
@mutex = Mutex.new
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Check multiple files in parallel.
|
|
36
|
+
#
|
|
37
|
+
# @param file_paths [Array<String>] Paths to files to check
|
|
38
|
+
# @return [Array<Core::Models::Result::DocumentResult>] Results for each file
|
|
39
|
+
def check_files_parallel(file_paths)
|
|
40
|
+
return [] if file_paths.empty?
|
|
41
|
+
|
|
42
|
+
# Add all files to the queue
|
|
43
|
+
file_paths.each { |path| @queue << path }
|
|
44
|
+
|
|
45
|
+
# Add poison pills to signal workers to stop
|
|
46
|
+
@worker_count.times { @queue << :done }
|
|
47
|
+
|
|
48
|
+
# Create and start workers
|
|
49
|
+
workers = @worker_count.times.map { create_worker }
|
|
50
|
+
|
|
51
|
+
# Wait for all workers to complete
|
|
52
|
+
workers.each(&:join)
|
|
53
|
+
|
|
54
|
+
# Clear queue for reuse
|
|
55
|
+
@queue.clear while @queue.empty? == false
|
|
56
|
+
|
|
57
|
+
@results
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Check a single file (convenience method).
|
|
61
|
+
#
|
|
62
|
+
# @param file_path [String] Path to file
|
|
63
|
+
# @return [Core::Models::Result::DocumentResult] Check result
|
|
64
|
+
def check_file(file_path)
|
|
65
|
+
@spellchecker.check_file(file_path)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# Create a worker thread.
|
|
71
|
+
#
|
|
72
|
+
# @return [Thread] Worker thread
|
|
73
|
+
def create_worker
|
|
74
|
+
Thread.new do
|
|
75
|
+
while (path = @queue.pop) != :done
|
|
76
|
+
begin
|
|
77
|
+
result = @spellchecker.check_file(path)
|
|
78
|
+
@mutex.synchronize do
|
|
79
|
+
@results << result
|
|
80
|
+
end
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
# Log error but continue processing other files
|
|
83
|
+
warn "Error checking file #{path}: #{e.message}"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# Main spellchecker class.
|
|
7
|
+
#
|
|
8
|
+
# This is the primary facade for spell checking operations,
|
|
9
|
+
# providing methods to check words, text, and files.
|
|
10
|
+
#
|
|
11
|
+
# @example Creating a spellchecker with a dictionary
|
|
12
|
+
# dict = Kotoshu::Dictionary::UnixWords.new("/usr/share/dict/words", language_code: "en-US")
|
|
13
|
+
# spellchecker = Spellchecker.new(dictionary: dict)
|
|
14
|
+
# spellchecker.correct?("hello") # => true
|
|
15
|
+
#
|
|
16
|
+
# @example Using configuration
|
|
17
|
+
# spellchecker = Spellchecker.new(
|
|
18
|
+
# dictionary_path: "/usr/share/dict/words",
|
|
19
|
+
# language: "en-US"
|
|
20
|
+
# )
|
|
21
|
+
class Spellchecker
|
|
22
|
+
# @return [Suggestions::Generator] The suggestion generator
|
|
23
|
+
attr_reader :generator
|
|
24
|
+
|
|
25
|
+
# @return [Configuration] The configuration
|
|
26
|
+
attr_reader :config
|
|
27
|
+
|
|
28
|
+
# @return [ResourceBundle, nil] The resource bundle if provided
|
|
29
|
+
attr_reader :resource_bundle
|
|
30
|
+
|
|
31
|
+
# @return [NativeBackend, nil] The native engine, when backend
|
|
32
|
+
# selection activated one (see Configuration#backend /
|
|
33
|
+
# KOTOSHU_BACKEND). Nil means every call runs the pure-Ruby engine.
|
|
34
|
+
attr_reader :native_backend
|
|
35
|
+
|
|
36
|
+
# Create a new spellchecker.
|
|
37
|
+
#
|
|
38
|
+
# @param dictionary [Dictionary::Base, nil] The dictionary (optional)
|
|
39
|
+
# @param config [Configuration, Hash] Configuration or settings
|
|
40
|
+
# @param resource_bundle [ResourceBundle, nil] Pre-resolved resource bundle
|
|
41
|
+
# @param kwargs [Hash] Additional configuration options
|
|
42
|
+
#
|
|
43
|
+
# @example With dictionary
|
|
44
|
+
# spellchecker = Spellchecker.new(dictionary: dict)
|
|
45
|
+
#
|
|
46
|
+
# @example With resource bundle (0.2+)
|
|
47
|
+
# bundle = Kotoshu::ResourceManager.resolve(language: "en")
|
|
48
|
+
# spellchecker = Spellchecker.new(resource_bundle: bundle)
|
|
49
|
+
# spellchecker.correct?("hello") # => true
|
|
50
|
+
#
|
|
51
|
+
# @example With configuration hash
|
|
52
|
+
# spellchecker = Spellchecker.new(
|
|
53
|
+
# dictionary_path: "/usr/share/dict/words",
|
|
54
|
+
# language: "en-US"
|
|
55
|
+
# )
|
|
56
|
+
#
|
|
57
|
+
# @example With Configuration object
|
|
58
|
+
# config = Configuration.new(dictionary_path: "words.txt")
|
|
59
|
+
# spellchecker = Spellchecker.new(config: config)
|
|
60
|
+
def initialize(dictionary: nil, config: nil, resource_bundle: nil, **kwargs)
|
|
61
|
+
@resource_bundle = resource_bundle
|
|
62
|
+
|
|
63
|
+
if resource_bundle
|
|
64
|
+
dictionary ||= resource_bundle.dictionary
|
|
65
|
+
kwargs[:language] = resource_bundle.language unless kwargs.key?(:language)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
if config.is_a?(Configuration)
|
|
69
|
+
@config = config
|
|
70
|
+
else
|
|
71
|
+
settings = kwargs.dup
|
|
72
|
+
settings[:dictionary_path] = dictionary.path if dictionary
|
|
73
|
+
@config = Configuration.new(settings)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
@config.dictionary = dictionary if dictionary
|
|
77
|
+
|
|
78
|
+
dict = @config.dictionary
|
|
79
|
+
max_suggestions = @config.max_suggestions
|
|
80
|
+
|
|
81
|
+
@generator = Suggestions::Generator.new(
|
|
82
|
+
dict,
|
|
83
|
+
max_suggestions: max_suggestions,
|
|
84
|
+
algorithms: @config.suggestion_algorithms
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Backend selection (plan 66): when active, correct?/suggest run on
|
|
88
|
+
# the native engine; everything else keeps using @generator. The
|
|
89
|
+
# generator is always built — the native backend is an accelerator,
|
|
90
|
+
# never a replacement for the Ruby engine's non-hot-path surface.
|
|
91
|
+
@native_backend = NativeBackend.resolve(
|
|
92
|
+
dictionary: dict,
|
|
93
|
+
backend: @config.backend,
|
|
94
|
+
max_suggestions: max_suggestions
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Word extraction follows the configured language's script
|
|
98
|
+
# (plan 91): its tokenizer decides which letters form words,
|
|
99
|
+
# so Greek/Cyrillic words become checkable for el/uk while
|
|
100
|
+
# wrong-script words stay invisible. Nil keeps the fallback:
|
|
101
|
+
# ASCII letters and the apostrophe, the pre-plan-91 set (so
|
|
102
|
+
# unknown languages behave as before).
|
|
103
|
+
@language_tokenizer = resolve_language_tokenizer
|
|
104
|
+
@word_char_regex = @language_tokenizer&.spellcheck_word_regex || /[a-zA-Z']/
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Check if a word is spelled correctly.
|
|
108
|
+
#
|
|
109
|
+
# @param word [String] The word to check
|
|
110
|
+
# @return [Boolean] True if the word is correct
|
|
111
|
+
#
|
|
112
|
+
# @example
|
|
113
|
+
# spellchecker.correct?("hello") # => true
|
|
114
|
+
# spellchecker.correct?("helo") # => false
|
|
115
|
+
def correct?(word)
|
|
116
|
+
return false if word.nil? || word.empty?
|
|
117
|
+
|
|
118
|
+
return @native_backend.correct?(word) if @native_backend
|
|
119
|
+
|
|
120
|
+
@generator.correct?(word)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Check if a word is misspelled.
|
|
124
|
+
#
|
|
125
|
+
# @param word [String] The word to check
|
|
126
|
+
# @return [Boolean] True if the word is misspelled
|
|
127
|
+
def incorrect?(word)
|
|
128
|
+
!correct?(word)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Get spelling suggestions for a word.
|
|
132
|
+
#
|
|
133
|
+
# @param word [String] The misspelled word
|
|
134
|
+
# @param max_suggestions [Integer] Maximum suggestions (optional)
|
|
135
|
+
# @return [Suggestions::SuggestionSet] Generated suggestions
|
|
136
|
+
#
|
|
137
|
+
# @example
|
|
138
|
+
# suggestions = spellchecker.suggest("helo")
|
|
139
|
+
# suggestions.to_words # => ["hello", "help", "held", ...]
|
|
140
|
+
def suggest(word, max_suggestions: nil)
|
|
141
|
+
return Suggestions::SuggestionSet.empty if word.nil? || word.empty?
|
|
142
|
+
|
|
143
|
+
return @native_backend.suggest(word, max_suggestions: max_suggestions) if @native_backend
|
|
144
|
+
|
|
145
|
+
@generator.generate(word, max_suggestions: max_suggestions)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Check a word and return a result object.
|
|
149
|
+
#
|
|
150
|
+
# @param word [String] The word to check
|
|
151
|
+
# @return [Models::Result::WordResult] The check result
|
|
152
|
+
#
|
|
153
|
+
# @example
|
|
154
|
+
# result = spellchecker.check_word("hello")
|
|
155
|
+
# result.correct? # => true
|
|
156
|
+
#
|
|
157
|
+
# @example With misspelled word
|
|
158
|
+
# result = spellchecker.check_word("helo")
|
|
159
|
+
# result.correct? # => false
|
|
160
|
+
# result.suggestions # => SuggestionSet with suggestions
|
|
161
|
+
def check_word(word)
|
|
162
|
+
if word.nil? || word.empty?
|
|
163
|
+
return Models::Result::WordResult.new(word: "", correct: false,
|
|
164
|
+
suggestions: Suggestions::SuggestionSet.empty)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
if correct?(word)
|
|
168
|
+
Models::Result::WordResult.correct(word)
|
|
169
|
+
else
|
|
170
|
+
suggestions = suggest(word)
|
|
171
|
+
Models::Result::WordResult.incorrect(word, suggestions: suggestions)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Check text for spelling errors.
|
|
176
|
+
#
|
|
177
|
+
# Words present in the user's personal dictionary
|
|
178
|
+
# (~/.config/kotoshu/personal.dic, KOTOSHU_PERSONAL_DIC override)
|
|
179
|
+
# never surface as errors — the same semantics the LSP applies to
|
|
180
|
+
# its diagnostics (plan 105). They are dropped at result assembly
|
|
181
|
+
# without suppression metadata, still count toward +word_count+,
|
|
182
|
+
# and remain spellcheckable through {#suggest}. Opt out with
|
|
183
|
+
# {Configuration#personal_dictionary} (KOTOSHU_PERSONAL_DICTIONARY,
|
|
184
|
+
# `kotoshu check --no-personal`).
|
|
185
|
+
#
|
|
186
|
+
# @param text [String] The text to check
|
|
187
|
+
# @param suggestions [Boolean] Generate suggestions for errors
|
|
188
|
+
# (default true; false yields errors with empty suggestion
|
|
189
|
+
# sets — positions and counts are unaffected)
|
|
190
|
+
# @param suggestions_filter [#call, nil] Optional word predicate;
|
|
191
|
+
# when it answers false for a misspelled word, that occurrence
|
|
192
|
+
# skips suggestion generation. The CI baseline uses this to
|
|
193
|
+
# skip the sweep for occurrences it is about to absorb
|
|
194
|
+
# (plan 116) while new errors keep their suggestions.
|
|
195
|
+
# @return [Models::Result::DocumentResult] The check result
|
|
196
|
+
#
|
|
197
|
+
# @example
|
|
198
|
+
# result = spellchecker.check("Hello wrold")
|
|
199
|
+
# result.success? # => false
|
|
200
|
+
# result.errors.map(&:word) # => ["wrold"]
|
|
201
|
+
def check(text, suggestions: true, suggestions_filter: nil)
|
|
202
|
+
return Models::Result::DocumentResult.success if text.nil? || text.empty?
|
|
203
|
+
|
|
204
|
+
# Inline ignore directives (plan 82): suppressed errors move into
|
|
205
|
+
# suppressed_errors instead of failing the check. The :auto profile
|
|
206
|
+
# recognizes bare, Markdown HTML-comment, and AsciiDoc // directives
|
|
207
|
+
# because the facade sees raw text without a format parser.
|
|
208
|
+
words = tokenize(text)
|
|
209
|
+
suppressions = Documents::Suppressions.scan(text, format: :auto)
|
|
210
|
+
personal = personal_words
|
|
211
|
+
errors = []
|
|
212
|
+
suppressed_errors = []
|
|
213
|
+
words.each do |word, pos|
|
|
214
|
+
next if personal.include?(word.downcase)
|
|
215
|
+
|
|
216
|
+
correct = correct?(word)
|
|
217
|
+
next if correct
|
|
218
|
+
|
|
219
|
+
want_suggestions =
|
|
220
|
+
suggestions && (!suggestions_filter || suggestions_filter.call(word))
|
|
221
|
+
line = Documents::SourcePosition.line_for_offset(text, pos)
|
|
222
|
+
suppressed = suppressions.any? { |s| s.applies_to?(line, word: word) }
|
|
223
|
+
entry = Models::Result::WordResult.new(
|
|
224
|
+
word: word,
|
|
225
|
+
correct: false,
|
|
226
|
+
suggestions: (suggest(word) if want_suggestions) || Suggestions::SuggestionSet.empty,
|
|
227
|
+
position: pos,
|
|
228
|
+
suppressed: suppressed,
|
|
229
|
+
suppressed_by: (Models::Result::WordResult::SUPPRESSED_BY_INLINE if suppressed)
|
|
230
|
+
)
|
|
231
|
+
(suppressed ? suppressed_errors : errors) << entry
|
|
232
|
+
end
|
|
233
|
+
Models::Result::DocumentResult.new(
|
|
234
|
+
file: nil,
|
|
235
|
+
errors: errors,
|
|
236
|
+
suppressed_errors: suppressed_errors,
|
|
237
|
+
word_count: words.size
|
|
238
|
+
)
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Check a file for spelling errors.
|
|
242
|
+
#
|
|
243
|
+
# @param path [String] The file path
|
|
244
|
+
# @return [Models::Result::DocumentResult] The check result
|
|
245
|
+
#
|
|
246
|
+
# @example
|
|
247
|
+
# result = spellchecker.check_file("README.md")
|
|
248
|
+
# result.to_s # => "File 'README.md': 3 spelling error(s) found"
|
|
249
|
+
def check_file(path)
|
|
250
|
+
raise DictionaryNotFoundError, path unless File.exist?(path)
|
|
251
|
+
|
|
252
|
+
text = File.read(path, encoding: @config.encoding)
|
|
253
|
+
result = check(text)
|
|
254
|
+
|
|
255
|
+
# Create a new result with the file path
|
|
256
|
+
Models::Result::DocumentResult.new(
|
|
257
|
+
file: path,
|
|
258
|
+
errors: result.errors,
|
|
259
|
+
word_count: result.word_count
|
|
260
|
+
)
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# Check a directory for spelling errors.
|
|
264
|
+
#
|
|
265
|
+
# @param path [String] The directory path
|
|
266
|
+
# @param pattern [String] File pattern to match (default: "*.txt")
|
|
267
|
+
# @return [Array<Models::Result::DocumentResult>] Results for each file
|
|
268
|
+
#
|
|
269
|
+
# @example
|
|
270
|
+
# results = spellchecker.check_directory("docs/")
|
|
271
|
+
# results.select(&:failed?).map(&:file)
|
|
272
|
+
def check_directory(path, pattern: "*.txt")
|
|
273
|
+
raise DictionaryNotFoundError, path unless File.exist?(path) && File.directory?(path)
|
|
274
|
+
|
|
275
|
+
files = Dir.glob(File.join(path, pattern))
|
|
276
|
+
files.map { |file| check_file(file) }
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Tokenize text into words.
|
|
280
|
+
#
|
|
281
|
+
# @param text [String] The text to tokenize
|
|
282
|
+
# @return [Array<Array>] Array of [word, position] pairs
|
|
283
|
+
#
|
|
284
|
+
# @example
|
|
285
|
+
# spellchecker.tokenize("Hello world!")
|
|
286
|
+
# # => [["Hello", 0], ["world", 6]]
|
|
287
|
+
def tokenize(text)
|
|
288
|
+
return [] if text.nil? || text.empty?
|
|
289
|
+
|
|
290
|
+
words = []
|
|
291
|
+
position = 0
|
|
292
|
+
word_buffer = String.new
|
|
293
|
+
word_start = 0
|
|
294
|
+
|
|
295
|
+
text.each_char.with_index do |char, i|
|
|
296
|
+
if word_char?(char)
|
|
297
|
+
word_buffer << char
|
|
298
|
+
word_start = i if word_buffer.length == 1
|
|
299
|
+
position = i
|
|
300
|
+
elsif !word_buffer.empty?
|
|
301
|
+
words << [word_buffer.dup.freeze, word_start]
|
|
302
|
+
word_buffer.clear
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# Don't forget the last word
|
|
307
|
+
words << [word_buffer.dup.freeze, word_start] unless word_buffer.empty?
|
|
308
|
+
|
|
309
|
+
words
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Get the dictionary being used.
|
|
313
|
+
#
|
|
314
|
+
# @return [Dictionary::Base] The dictionary
|
|
315
|
+
def dictionary
|
|
316
|
+
@generator.dictionary
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# Reload the dictionary.
|
|
320
|
+
#
|
|
321
|
+
# @return [self] Self for chaining
|
|
322
|
+
def reload_dictionary
|
|
323
|
+
@config.reset_dictionary
|
|
324
|
+
|
|
325
|
+
dict = @config.dictionary
|
|
326
|
+
@generator = Suggestions::Generator.new(
|
|
327
|
+
dict,
|
|
328
|
+
max_suggestions: @config.max_suggestions,
|
|
329
|
+
algorithms: @config.suggestion_algorithms
|
|
330
|
+
)
|
|
331
|
+
@native_backend = NativeBackend.resolve(
|
|
332
|
+
dictionary: dict,
|
|
333
|
+
backend: @config.backend,
|
|
334
|
+
max_suggestions: @config.max_suggestions
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
self
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
private
|
|
341
|
+
|
|
342
|
+
# The personal dictionary as a downcased Set for the check path
|
|
343
|
+
# (plan 105), mirroring the LSP's filter. Loaded once per process:
|
|
344
|
+
# the first {#check} call snapshots the words, and later edits to
|
|
345
|
+
# personal.dic take effect in the next process (the LSP reloads on
|
|
346
|
+
# mtime change; this path deliberately does not — one-shot CLI runs
|
|
347
|
+
# always read fresh, and long-lived embedders rebuild the
|
|
348
|
+
# spellchecker via {Kotoshu.reset_spellchecker}). An empty set when
|
|
349
|
+
# disabled through {Configuration#personal_dictionary}.
|
|
350
|
+
#
|
|
351
|
+
# @return [Set<String>]
|
|
352
|
+
def personal_words
|
|
353
|
+
return @personal_words if instance_variable_defined?(:@personal_words)
|
|
354
|
+
|
|
355
|
+
@personal_words =
|
|
356
|
+
if @config.personal_dictionary
|
|
357
|
+
PersonalDictionary.words.map(&:downcase).to_set
|
|
358
|
+
else
|
|
359
|
+
Set.new
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Resolve the tokenizer of the language being checked for word
|
|
364
|
+
# extraction (plan 91).
|
|
365
|
+
#
|
|
366
|
+
# Only Language::Tokenizer::Base tokenizers participate: their
|
|
367
|
+
# script subclasses (Greek, Cyrillic, Latin) declare which letters
|
|
368
|
+
# form words. Anything else — components tokenizers such as
|
|
369
|
+
# English's whitespace tokenizer, or an unknown language — returns
|
|
370
|
+
# nil and the ASCII fallback applies.
|
|
371
|
+
#
|
|
372
|
+
# Module-less staged languages get the script fallback (plan 107):
|
|
373
|
+
# Language::Script maps the code to its writing script and hands
|
|
374
|
+
# back the script-appropriate tokenizer, so basic-tier languages
|
|
375
|
+
# check with real word extraction while their module is pending.
|
|
376
|
+
#
|
|
377
|
+
# A resource bundle pins the language: spellchecker_for passes the
|
|
378
|
+
# shared global Configuration next to the resolved bundle, and the
|
|
379
|
+
# config may still carry the global default language instead of
|
|
380
|
+
# the one actually being checked.
|
|
381
|
+
#
|
|
382
|
+
# @return [Language::Tokenizer::Base, nil] The language tokenizer
|
|
383
|
+
def resolve_language_tokenizer
|
|
384
|
+
code = @resource_bundle&.language || @config.language
|
|
385
|
+
return nil if code.nil? || code.empty?
|
|
386
|
+
|
|
387
|
+
language_class = Language::Registry.get(code)
|
|
388
|
+
return Language::Script.tokenizer_for(code) if language_class.nil?
|
|
389
|
+
|
|
390
|
+
tokenizer = language_class.new.tokenizer
|
|
391
|
+
tokenizer.is_a?(Language::Tokenizer::Base) ? tokenizer : nil
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
# Check if a character is part of a word.
|
|
395
|
+
#
|
|
396
|
+
# Uses the language tokenizer's script-aware set when one is
|
|
397
|
+
# configured; otherwise the historical ASCII letters plus
|
|
398
|
+
# apostrophe.
|
|
399
|
+
#
|
|
400
|
+
# @param char [String] The character
|
|
401
|
+
# @return [Boolean] True if it's a word character
|
|
402
|
+
def word_char?(char)
|
|
403
|
+
@word_char_regex.match?(char)
|
|
404
|
+
end
|
|
405
|
+
end
|
|
406
|
+
end
|