kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "kotoshu"
|
|
4
|
+
require "rake"
|
|
5
|
+
require "kotoshu/tasks/check_task"
|
|
6
|
+
|
|
7
|
+
# Wiring for `require "kotoshu/tasks"` (plan 89): installs the
|
|
8
|
+
# default `rake kotoshu:check` over the repository text files using
|
|
9
|
+
# the plan-88 file selection. Configure or add more tasks with
|
|
10
|
+
# Kotoshu::Tasks::CheckTask.new.
|
|
11
|
+
Kotoshu::Tasks::CheckTask.new
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Framework integrations for validation layers (plan 89).
|
|
5
|
+
#
|
|
6
|
+
# Soft dependency: ActiveModel is NOT in kotoshu.gemspec. The
|
|
7
|
+
# validator file soft-requires it exactly like Models::OnnxModel
|
|
8
|
+
# soft-requires onnxruntime (see CLAUDE.md); when it is absent the
|
|
9
|
+
# class still loads but raises a caller-friendly error on use.
|
|
10
|
+
module Validators
|
|
11
|
+
# True when ActiveModel was requireable at load time.
|
|
12
|
+
ACTIVE_MODEL_LOADED = begin
|
|
13
|
+
require "active_model"
|
|
14
|
+
true
|
|
15
|
+
rescue LoadError
|
|
16
|
+
false
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Raised when the validator is used without ActiveModel.
|
|
20
|
+
class ActiveModelUnavailable < Kotoshu::Error
|
|
21
|
+
def initialize
|
|
22
|
+
super("Kotoshu spelling validation needs ActiveModel. " \
|
|
23
|
+
"Add gem 'activemodel' (or the full Rails stack) to your Gemfile.")
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
if ACTIVE_MODEL_LOADED
|
|
28
|
+
# ActiveModel EachValidator that fails validation on
|
|
29
|
+
# misspelled content.
|
|
30
|
+
#
|
|
31
|
+
# One error is added per misspelling, with the top suggestion
|
|
32
|
+
# in the message.
|
|
33
|
+
#
|
|
34
|
+
# @example Wired for the validates short syntax
|
|
35
|
+
# # Rails initializer or the model file:
|
|
36
|
+
# SpellingValidator = Kotoshu::Validators::SpellingValidator
|
|
37
|
+
#
|
|
38
|
+
# class Post < ApplicationRecord
|
|
39
|
+
# validates :body, spelling: true
|
|
40
|
+
# validates :summary, spelling: { language: :en,
|
|
41
|
+
# personal_words: %w[kotoshu] }
|
|
42
|
+
# end
|
|
43
|
+
#
|
|
44
|
+
# @example Without the short syntax
|
|
45
|
+
# validates_with Kotoshu::Validators::SpellingValidator,
|
|
46
|
+
# attributes: [:body]
|
|
47
|
+
#
|
|
48
|
+
# Options:
|
|
49
|
+
# - language: language code; defaults to auto-detection.
|
|
50
|
+
# - personal_words: array of words accepted in addition to the
|
|
51
|
+
# dictionary.
|
|
52
|
+
class SpellingValidator < ActiveModel::EachValidator
|
|
53
|
+
def validate_each(record, attribute, value)
|
|
54
|
+
return if value.nil? || value.empty?
|
|
55
|
+
|
|
56
|
+
result = Kotoshu.check(value, language: language)
|
|
57
|
+
errors = result.errors.reject { |error| personal_words.include?(error.word.downcase) }
|
|
58
|
+
errors.each do |error|
|
|
59
|
+
record.errors.add(attribute, :spelling, **error_attributes(error))
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
# The language to check with: the option value, or
|
|
66
|
+
# auto-detection when unset.
|
|
67
|
+
#
|
|
68
|
+
# @return [String, Symbol, nil]
|
|
69
|
+
def language
|
|
70
|
+
options[:language]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Words accepted on top of the dictionary, downcased.
|
|
74
|
+
#
|
|
75
|
+
# @return [Array<String>]
|
|
76
|
+
def personal_words
|
|
77
|
+
@personal_words ||= Array(options[:personal_words]).map(&:downcase)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Build the error details for one misspelling.
|
|
81
|
+
#
|
|
82
|
+
# @param error [Models::Result::WordResult] the misspelling
|
|
83
|
+
# @return [Hash] message and metadata
|
|
84
|
+
def error_attributes(error)
|
|
85
|
+
suggestion = error.top_suggestions(1).first
|
|
86
|
+
message = "'#{error.word}' is misspelled"
|
|
87
|
+
message += " - did you mean '#{suggestion}'?" if suggestion
|
|
88
|
+
{ message: message, word: error.word, suggestions: error.top_suggestions(3) }
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
else
|
|
92
|
+
# Placeholder raising a friendly error when ActiveModel is
|
|
93
|
+
# missing; keeps the constant resolvable for tooling.
|
|
94
|
+
class SpellingValidator
|
|
95
|
+
def initialize(*)
|
|
96
|
+
raise ActiveModelUnavailable
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Validation-layer integrations (plan 89). Soft dependency on
|
|
5
|
+
# ActiveModel — see Validators::SpellingValidator.
|
|
6
|
+
module Validators
|
|
7
|
+
autoload :SpellingValidator, "kotoshu/validators/spelling_validator"
|
|
8
|
+
end
|
|
9
|
+
end
|
data/lib/kotoshu.rb
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Kotoshu is a semantic spell checker for Ruby.
|
|
4
|
+
#
|
|
5
|
+
# Public API entry point. All internal namespaces are autoloaded on first
|
|
6
|
+
# reference; loading this file is cheap and triggers no heavy work.
|
|
7
|
+
#
|
|
8
|
+
# Exception: version.rb is required because autoload only works for module
|
|
9
|
+
# constants, and Bundler/gemspec also reads Kotoshu::VERSION from it.
|
|
10
|
+
require_relative "kotoshu/version"
|
|
11
|
+
|
|
12
|
+
module Kotoshu
|
|
13
|
+
# ---- Namespaces (autoloaded; each namespace file declares its children) ----
|
|
14
|
+
autoload :Algorithms, "kotoshu/algorithms"
|
|
15
|
+
autoload :Analyzers, "kotoshu/analyzers"
|
|
16
|
+
autoload :Baseline, "kotoshu/baseline"
|
|
17
|
+
autoload :Cache, "kotoshu/cache"
|
|
18
|
+
autoload :Cli, "kotoshu/cli"
|
|
19
|
+
autoload :Components, "kotoshu/components"
|
|
20
|
+
autoload :Configuration, "kotoshu/configuration"
|
|
21
|
+
autoload :ConformanceExporter, "kotoshu/conformance_exporter"
|
|
22
|
+
autoload :ConformanceRunner, "kotoshu/conformance_runner"
|
|
23
|
+
autoload :Core, "kotoshu/core"
|
|
24
|
+
autoload :Data, "kotoshu/data"
|
|
25
|
+
autoload :DataStructures, "kotoshu/data_structures"
|
|
26
|
+
autoload :Defaults, "kotoshu/defaults"
|
|
27
|
+
autoload :Dictionaries, "kotoshu/dictionaries"
|
|
28
|
+
autoload :Dictionary, "kotoshu/dictionary"
|
|
29
|
+
autoload :Documents, "kotoshu/documents"
|
|
30
|
+
autoload :Embeddings, "kotoshu/embeddings"
|
|
31
|
+
autoload :FluentChecker, "kotoshu/fluent_checker"
|
|
32
|
+
autoload :Grammar, "kotoshu/grammar"
|
|
33
|
+
autoload :Integrity, "kotoshu/integrity"
|
|
34
|
+
autoload :Keyboard, "kotoshu/keyboard"
|
|
35
|
+
autoload :Language, "kotoshu/language"
|
|
36
|
+
autoload :Languages, "kotoshu/languages"
|
|
37
|
+
autoload :Models, "kotoshu/models"
|
|
38
|
+
autoload :Paths, "kotoshu/paths"
|
|
39
|
+
autoload :PersonalDictionary, "kotoshu/personal_dictionary"
|
|
40
|
+
autoload :Plugins, "kotoshu/plugins"
|
|
41
|
+
autoload :ProjectConfig, "kotoshu/project_config"
|
|
42
|
+
autoload :MultiLanguageChecker, "kotoshu/multi_language_checker"
|
|
43
|
+
# Native engine (Rust extension loader + adapter; plan 66). Required via
|
|
44
|
+
# autoload so a missing extension can never break `require "kotoshu"`.
|
|
45
|
+
autoload :Native, "kotoshu/native"
|
|
46
|
+
autoload :NativeBackend, "kotoshu/native_backend"
|
|
47
|
+
autoload :Readers, "kotoshu/readers"
|
|
48
|
+
autoload :ResourceBundle, "kotoshu/resource_bundle"
|
|
49
|
+
autoload :ResourceManager, "kotoshu/resource_manager"
|
|
50
|
+
autoload :Results, "kotoshu/results"
|
|
51
|
+
autoload :SourceRegistry, "kotoshu/source_registry"
|
|
52
|
+
autoload :Spellchecker, "kotoshu/spellchecker"
|
|
53
|
+
autoload :StringMetrics, "kotoshu/string_metrics"
|
|
54
|
+
autoload :Suggestions, "kotoshu/suggestions"
|
|
55
|
+
# Framework integrations (plan 89): soft dependencies, autoloaded
|
|
56
|
+
# only on reference. The rake task is require-only instead:
|
|
57
|
+
# `require "kotoshu/tasks"` (it defines a task on load, so an
|
|
58
|
+
# autoload would surprise).
|
|
59
|
+
autoload :Validators, "kotoshu/validators"
|
|
60
|
+
autoload :Rspec, "kotoshu/rspec"
|
|
61
|
+
autoload :Jekyll, "kotoshu/jekyll"
|
|
62
|
+
|
|
63
|
+
# ---- Top-level error classes (all defined in core/exceptions.rb) ----
|
|
64
|
+
autoload :Error, "kotoshu/core/exceptions"
|
|
65
|
+
autoload :DictionaryNotFoundError, "kotoshu/core/exceptions"
|
|
66
|
+
autoload :InvalidDictionaryFormatError, "kotoshu/core/exceptions"
|
|
67
|
+
autoload :ConfigurationError, "kotoshu/core/exceptions"
|
|
68
|
+
autoload :SpellcheckError, "kotoshu/core/exceptions"
|
|
69
|
+
autoload :AffixRuleError, "kotoshu/core/exceptions"
|
|
70
|
+
autoload :ResourceNotCachedError, "kotoshu/core/exceptions"
|
|
71
|
+
autoload :ResourceNotSetupError, "kotoshu/core/exceptions"
|
|
72
|
+
autoload :ResourceResolutionError, "kotoshu/core/exceptions"
|
|
73
|
+
autoload :IntegrityError, "kotoshu/core/exceptions"
|
|
74
|
+
autoload :SuikaUnavailable, "kotoshu/core/exceptions"
|
|
75
|
+
|
|
76
|
+
# ---- Lazily-loaded singletons (not in their own namespace file) ----
|
|
77
|
+
autoload :Debug, "kotoshu/debug_mode"
|
|
78
|
+
autoload :DebugLogger, "kotoshu/debug_logger"
|
|
79
|
+
autoload :Metrics, "kotoshu/metrics_module"
|
|
80
|
+
autoload :MetricsCollector, "kotoshu/metrics_collector"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
module Kotoshu
|
|
84
|
+
# Global configuration instance.
|
|
85
|
+
#
|
|
86
|
+
# @return [Configuration] The global configuration
|
|
87
|
+
#
|
|
88
|
+
# @example
|
|
89
|
+
# Kotoshu.configure do |config|
|
|
90
|
+
# config.dictionary_path = "/usr/share/dict/words"
|
|
91
|
+
# config.language = "en-US"
|
|
92
|
+
# end
|
|
93
|
+
def self.configure
|
|
94
|
+
yield configuration if block_given?
|
|
95
|
+
configuration
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Get the global configuration.
|
|
99
|
+
#
|
|
100
|
+
# @return [Configuration] The global configuration
|
|
101
|
+
#
|
|
102
|
+
# @example
|
|
103
|
+
# config = Kotoshu.configuration
|
|
104
|
+
def self.configuration
|
|
105
|
+
Configuration.instance
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Replace the global configuration. Useful for tests and for
|
|
109
|
+
# embedding kotoshu in a larger application that wants to share a
|
|
110
|
+
# Configuration across multiple Spellcheckers without each one
|
|
111
|
+
# independently touching the singleton.
|
|
112
|
+
#
|
|
113
|
+
# The argument must be a Configuration instance (type-checked so a
|
|
114
|
+
# typo doesn't silently flip the global to nil and break every
|
|
115
|
+
# downstream lookup).
|
|
116
|
+
#
|
|
117
|
+
# @param config [Configuration]
|
|
118
|
+
# @return [Configuration] the same instance, for chaining
|
|
119
|
+
#
|
|
120
|
+
# @example Inject a test-only configuration
|
|
121
|
+
# Kotoshu.configuration = Configuration.new.tap { |c| c.cache_ttl = 1 }
|
|
122
|
+
def self.configuration=(config)
|
|
123
|
+
raise ArgumentError, "configuration must be a Configuration instance" unless config.is_a?(Configuration)
|
|
124
|
+
|
|
125
|
+
Configuration.instance = config
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Default spellchecker (singleton). Uses the configured default language.
|
|
129
|
+
# Cache-only — raises ResourceNotSetupError if the default language hasn't
|
|
130
|
+
# been set up via Kotoshu.setup.
|
|
131
|
+
#
|
|
132
|
+
# @return [Spellchecker] The default spellchecker
|
|
133
|
+
# @raise [ResourceNotSetupError] if no language is set up
|
|
134
|
+
def self.spellchecker
|
|
135
|
+
return @spellchecker if @spellchecker
|
|
136
|
+
|
|
137
|
+
lang = configuration.default_language
|
|
138
|
+
raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil? || lang.to_s.empty?
|
|
139
|
+
|
|
140
|
+
@spellchecker = spellchecker_for(lang)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Get a spellchecker for a specific language (cache-only, raises on miss).
|
|
144
|
+
#
|
|
145
|
+
# @param language [String, Symbol] Language code (e.g., "en", "de", "fr")
|
|
146
|
+
# @return [Spellchecker] Spellchecker using a ResourceManager-resolved bundle
|
|
147
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
148
|
+
#
|
|
149
|
+
# @example
|
|
150
|
+
# Kotoshu.setup(:de)
|
|
151
|
+
# Kotoshu.spellchecker_for("de").correct?("Hallo") # => true
|
|
152
|
+
def self.spellchecker_for(language)
|
|
153
|
+
key = language.to_s
|
|
154
|
+
@spellcheckers ||= {}
|
|
155
|
+
@spellcheckers[key] ||= begin
|
|
156
|
+
bundle = ResourceManager.resolve(language: language)
|
|
157
|
+
Spellchecker.new(resource_bundle: bundle, config: configuration)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Resolve language resources from the cache (no download).
|
|
162
|
+
#
|
|
163
|
+
# @param language [String, Symbol, nil] Language code; if nil, uses default
|
|
164
|
+
# @param want [Array<Symbol>] Resource types (default: [:spelling])
|
|
165
|
+
# @param tier [Symbol, nil] Model tier to resolve (:full default,
|
|
166
|
+
# :fluency, :mini, or :any for "the single cached tier")
|
|
167
|
+
# @return [ResourceBundle] Resolved bundle
|
|
168
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
169
|
+
#
|
|
170
|
+
# @example
|
|
171
|
+
# Kotoshu.setup(:en)
|
|
172
|
+
# bundle = Kotoshu.resolve(language: "en")
|
|
173
|
+
# bundle.dictionary # => #<Dictionary::Hunspell ...>
|
|
174
|
+
def self.resolve(language: nil, want: nil, tier: nil)
|
|
175
|
+
lang = language || configuration.default_language
|
|
176
|
+
raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil?
|
|
177
|
+
|
|
178
|
+
want_param = want || ResourceManager::DEFAULT_WANT
|
|
179
|
+
ResourceManager.resolve(language: lang, want: want_param, tier: tier)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# ---- Stage 1: Setup ----
|
|
183
|
+
|
|
184
|
+
# Set up resources for one or more languages (download or register local files).
|
|
185
|
+
# Idempotent: re-running with the same args is a no-op unless `force: true`.
|
|
186
|
+
#
|
|
187
|
+
# @param languages [String, Symbol, Array<String, Symbol>] One or more language codes
|
|
188
|
+
# @param want [Array<Symbol>] Resource types to fetch (default: [:spelling])
|
|
189
|
+
# @param opts [Hash] Additional keyword options, all forwarded to
|
|
190
|
+
# ResourceManager.setup
|
|
191
|
+
# @option opts [Boolean] :force Re-fetch even if already cached
|
|
192
|
+
# @option opts [Boolean] :strict Re-raise on optional-resource failure
|
|
193
|
+
# @option opts [Symbol, String] :tier Model tier when want includes :model
|
|
194
|
+
# (:full default, :fluency, :mini)
|
|
195
|
+
# @option opts [String, nil] :aff Path to local .aff file (single-language only)
|
|
196
|
+
# @option opts [String, nil] :dic Path to local .dic file (single-language only)
|
|
197
|
+
# @option opts [String, nil] :from Directory containing local .aff/.dic
|
|
198
|
+
# (single-language only)
|
|
199
|
+
# @option opts [String, nil] :frequency Path to local frequency.json
|
|
200
|
+
# (single-language only)
|
|
201
|
+
# @return [SetupResult, Array<SetupResult>] Result or results (array if multiple languages)
|
|
202
|
+
#
|
|
203
|
+
# @example Download from kotoshu/dictionaries
|
|
204
|
+
# Kotoshu.setup(:en) # spelling only
|
|
205
|
+
# Kotoshu.setup(:en, want: %i[spelling frequency]) # spelling + frequency
|
|
206
|
+
# Kotoshu.setup(:en, want: %i[model], tier: :mini) # mini-tier model
|
|
207
|
+
# Kotoshu.setup(:en, :de, :fr) # multiple languages
|
|
208
|
+
#
|
|
209
|
+
# @example Register local files (user already has hunspell dicts)
|
|
210
|
+
# Kotoshu.setup(:en, aff: "/usr/share/hunspell/en_US.aff",
|
|
211
|
+
# dic: "/usr/share/hunspell/en_US.dic")
|
|
212
|
+
#
|
|
213
|
+
# @example Register local files from a directory
|
|
214
|
+
# Kotoshu.setup(:en, from: "/usr/share/hunspell/") # looks for en.aff, en.dic
|
|
215
|
+
def self.setup(*languages, want: nil, **opts)
|
|
216
|
+
raise ArgumentError, "Kotoshu.setup requires at least one language" if languages.empty?
|
|
217
|
+
|
|
218
|
+
want_param = want || ResourceManager::DEFAULT_WANT
|
|
219
|
+
if languages.size == 1
|
|
220
|
+
ResourceManager.setup(languages.first, want: want_param, **opts)
|
|
221
|
+
else
|
|
222
|
+
languages.map { |lang| ResourceManager.setup(lang, want: want_param, **opts) }
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Check if a language (or a specific resource for that language) is set up.
|
|
227
|
+
#
|
|
228
|
+
# @param language [String, Symbol] Language code
|
|
229
|
+
# @param resource [Symbol, nil] :spelling, :frequency, :model, or nil for any
|
|
230
|
+
# @param tier [Symbol, nil] Model tier to check (:full, :fluency, :mini,
|
|
231
|
+
# or :any for "any tier cached"); only meaningful with resource: :model
|
|
232
|
+
# @return [Boolean] True if the resource is cached and available
|
|
233
|
+
#
|
|
234
|
+
# @example
|
|
235
|
+
# Kotoshu.setup(:en)
|
|
236
|
+
# Kotoshu.setup?(:en) # => true
|
|
237
|
+
# Kotoshu.setup?(:en, :spelling) # => true
|
|
238
|
+
# Kotoshu.setup?(:en, :frequency) # => false (not set up)
|
|
239
|
+
# Kotoshu.setup?(:en, resource: :model, tier: :mini) # => false
|
|
240
|
+
def self.setup?(language, resource = nil, tier: nil, **options)
|
|
241
|
+
# The resource may be given positionally (legacy) or as a keyword.
|
|
242
|
+
resource = options.fetch(:resource, resource)
|
|
243
|
+
ResourceManager.setup?(language, resource: resource, tier: tier)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# List languages that have been set up.
|
|
247
|
+
#
|
|
248
|
+
# @return [Array<String>] Sorted array of language codes with cached spelling
|
|
249
|
+
#
|
|
250
|
+
# @example
|
|
251
|
+
# Kotoshu.languages_setup # => ["de", "en", "fr"]
|
|
252
|
+
def self.languages_setup
|
|
253
|
+
ResourceManager.languages_setup
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Reset the spellchecker cache. The next call to `spellchecker` or
|
|
257
|
+
# `spellchecker_for` re-resolves from the current configuration.
|
|
258
|
+
#
|
|
259
|
+
# Does NOT eagerly reload — clearing the cache is enough. This makes
|
|
260
|
+
# the method safe to call between tests even when no language is set
|
|
261
|
+
# up yet (the next call will raise ResourceNotSetupError per the
|
|
262
|
+
# strict two-stage contract).
|
|
263
|
+
def self.reset_spellchecker
|
|
264
|
+
@spellchecker = nil
|
|
265
|
+
@spellcheckers = nil
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Check if a word is spelled correctly.
|
|
270
|
+
# Hot path — cache-only, raises if language not set up.
|
|
271
|
+
#
|
|
272
|
+
# @param word [String] The word to check
|
|
273
|
+
# @param language [String, Symbol, nil] Language code; if nil, uses configured default
|
|
274
|
+
# @return [Boolean] True if the word is correct
|
|
275
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
276
|
+
#
|
|
277
|
+
# @example
|
|
278
|
+
# Kotoshu.setup(:en)
|
|
279
|
+
# Kotoshu.correct?("hello") # => true
|
|
280
|
+
# Kotoshu.correct?("Hallo", language: "de") # requires Kotoshu.setup(:de) first
|
|
281
|
+
def self.correct?(word, language: nil)
|
|
282
|
+
checker = language ? spellchecker_for(language) : spellchecker
|
|
283
|
+
checker.correct?(word)
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Check if a word is misspelled. Hot path.
|
|
287
|
+
#
|
|
288
|
+
# @param word [String] The word to check
|
|
289
|
+
# @param language [String, Symbol, nil] Language code
|
|
290
|
+
# @return [Boolean] True if the word is misspelled
|
|
291
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
292
|
+
def self.misspelled?(word, language: nil)
|
|
293
|
+
!correct?(word, language: language)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Get spelling suggestions for a word. Hot path.
|
|
297
|
+
#
|
|
298
|
+
# @param word [String] The misspelled word
|
|
299
|
+
# @param language [String, Symbol, nil] Language code
|
|
300
|
+
# @param options [Hash] Options (max_suggestions, etc.)
|
|
301
|
+
# @return [Suggestions::SuggestionSet] Generated suggestions
|
|
302
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
303
|
+
#
|
|
304
|
+
# @example
|
|
305
|
+
# Kotoshu.setup(:en)
|
|
306
|
+
# suggestions = Kotoshu.suggest("helo")
|
|
307
|
+
# suggestions.to_words # => ["hello", "help", "held", ...]
|
|
308
|
+
def self.suggest(word, language: nil, **options)
|
|
309
|
+
checker = language ? spellchecker_for(language) : spellchecker
|
|
310
|
+
checker.suggest(word, **options)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Check text for spelling errors. Hot path.
|
|
314
|
+
#
|
|
315
|
+
# @param text [String] The text to check
|
|
316
|
+
# @param language [String, Symbol, nil] Language code; if nil, uses configured default
|
|
317
|
+
# @param _options [Hash] Accepted and ignored — kept so call sites can
|
|
318
|
+
# pass extra options without breaking
|
|
319
|
+
# @return [Models::Result::DocumentResult] The check result
|
|
320
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
321
|
+
#
|
|
322
|
+
# @example
|
|
323
|
+
# Kotoshu.setup(:en)
|
|
324
|
+
# result = Kotoshu.check("Hello wrold")
|
|
325
|
+
# result.errors.map(&:word) # => ["wrold"]
|
|
326
|
+
def self.check(text, language: nil, suggestions_filter: nil, **_options)
|
|
327
|
+
checker = language ? spellchecker_for(language) : spellchecker
|
|
328
|
+
checker.check(text, suggestions_filter: suggestions_filter)
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
# Check a file for spelling errors. Hot path.
|
|
332
|
+
#
|
|
333
|
+
# @param path [String] The file path
|
|
334
|
+
# @param language [String, Symbol, nil] Language code
|
|
335
|
+
# @param _options [Hash] Accepted and ignored — kept so call sites can
|
|
336
|
+
# pass extra options without breaking
|
|
337
|
+
# @return [Models::Result::DocumentResult] The check result
|
|
338
|
+
# @raise [ResourceNotSetupError] if the language hasn't been set up
|
|
339
|
+
#
|
|
340
|
+
# @example
|
|
341
|
+
# Kotoshu.setup(:en)
|
|
342
|
+
# result = Kotoshu.check_file("README.md")
|
|
343
|
+
# result.success? # => false
|
|
344
|
+
def self.check_file(path, language: nil, **_options)
|
|
345
|
+
checker = language ? spellchecker_for(language) : spellchecker
|
|
346
|
+
checker.check_file(path)
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Check multiple files for spelling errors.
|
|
350
|
+
#
|
|
351
|
+
# @param paths [Array<String>] The file paths
|
|
352
|
+
# @param options [Hash] Options
|
|
353
|
+
# @return [Array<Models::Result::DocumentResult>] Results for each file
|
|
354
|
+
#
|
|
355
|
+
# @example
|
|
356
|
+
# results = Kotoshu.check_files(%w[README.md CHANGELOG.md])
|
|
357
|
+
# results.select(&:failed?)
|
|
358
|
+
def self.check_files(paths, **options)
|
|
359
|
+
paths.map { |path| check_file(path, **options) }
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# Convenience method for creating an indexed dictionary.
|
|
363
|
+
#
|
|
364
|
+
# @param source [Array<String>, Hash, nil] Words or file path
|
|
365
|
+
# @return [Core::IndexedDictionary] New dictionary
|
|
366
|
+
def self.dictionary(source = nil)
|
|
367
|
+
case source
|
|
368
|
+
when Array
|
|
369
|
+
Core::IndexedDictionary.new(source)
|
|
370
|
+
when String
|
|
371
|
+
Core::IndexedDictionary.from_file(source)
|
|
372
|
+
when nil, Hash
|
|
373
|
+
Core::IndexedDictionary.new
|
|
374
|
+
else
|
|
375
|
+
raise ArgumentError, "Invalid dictionary source: #{source.inspect}"
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# Convenience method for creating a trie.
|
|
380
|
+
#
|
|
381
|
+
# @param source [Array<String>, String, nil] Words or file path
|
|
382
|
+
# @return [Core::Trie::Trie] New trie
|
|
383
|
+
def self.trie(source = nil)
|
|
384
|
+
case source
|
|
385
|
+
when Array
|
|
386
|
+
Core::Trie::Builder.from_array(source)
|
|
387
|
+
when String
|
|
388
|
+
if File.exist?(source)
|
|
389
|
+
Core::Trie::Builder.from_file(source)
|
|
390
|
+
else
|
|
391
|
+
Core::Trie::Builder.from_string(source)
|
|
392
|
+
end
|
|
393
|
+
when nil
|
|
394
|
+
Core::Trie::Trie.new
|
|
395
|
+
else
|
|
396
|
+
raise ArgumentError, "Invalid trie source: #{source.inspect}"
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Convenience method for creating a suggestion pipeline.
|
|
401
|
+
#
|
|
402
|
+
# @param strategies [Array] Optional strategies to add
|
|
403
|
+
# @return [Suggestions::Strategies::CompositeStrategy] New pipeline
|
|
404
|
+
def self.suggestion_pipeline(*strategies)
|
|
405
|
+
pipeline = Suggestions::Strategies::CompositeStrategy.new(name: :default)
|
|
406
|
+
strategies.each { |s| pipeline.add(s) }
|
|
407
|
+
pipeline
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
# Register a custom dictionary type.
|
|
411
|
+
#
|
|
412
|
+
# @param type [Symbol] The type key
|
|
413
|
+
# @param klass [Class] The dictionary class
|
|
414
|
+
#
|
|
415
|
+
# @example
|
|
416
|
+
# Kotoshu.register_dictionary_type(:my_custom, MyDictionary)
|
|
417
|
+
def self.register_dictionary_type(type, klass)
|
|
418
|
+
Dictionary.register_type(type, klass)
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
# Register a custom suggestion algorithm.
|
|
422
|
+
#
|
|
423
|
+
# @param name [Symbol] The algorithm name
|
|
424
|
+
# @param klass [Class] The algorithm class
|
|
425
|
+
#
|
|
426
|
+
# @example
|
|
427
|
+
# Kotoshu.register_suggestion_algorithm(:my_custom, MyStrategy)
|
|
428
|
+
def self.register_suggestion_algorithm(name, klass)
|
|
429
|
+
Suggestions::Strategies::BaseStrategy.register_type(name, klass)
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
# Access the language module.
|
|
433
|
+
#
|
|
434
|
+
# @return [Module] The Language module
|
|
435
|
+
#
|
|
436
|
+
# @example
|
|
437
|
+
# Kotoshu::Language.detect("Hello world") # => "en"
|
|
438
|
+
def self.language
|
|
439
|
+
Language
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
# Detect language of text.
|
|
443
|
+
#
|
|
444
|
+
# Native LID when available (plan 106): the Rust extension scoring
|
|
445
|
+
# the `kotoshu://models/lid/lid-176` artifact pair over 176
|
|
446
|
+
# languages, after a one-time {setup_lid}. Falls back to the
|
|
447
|
+
# {Language::Detector} heuristic when the extension is absent,
|
|
448
|
+
# KOTOSHU_BACKEND=ruby, or the model is not set up — so pure-Ruby
|
|
449
|
+
# installs keep the previous behavior.
|
|
450
|
+
#
|
|
451
|
+
# @param text [String] Text to analyze
|
|
452
|
+
# @return [Language::Detection] +code+ (String, or nil when the
|
|
453
|
+
# heuristic is uncertain) and +score+ (Float in [0, 1]);
|
|
454
|
+
# interpolation yields the code
|
|
455
|
+
#
|
|
456
|
+
# @example
|
|
457
|
+
# Kotoshu.setup_lid
|
|
458
|
+
# Kotoshu.detect_language("Bonjour le monde").code # => "fr"
|
|
459
|
+
# Kotoshu.detect_language("こんにちは").score # => 0.99
|
|
460
|
+
def self.detect_language(text)
|
|
461
|
+
Language::LidDetector.detect(text)
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# Detect language with confidence score.
|
|
465
|
+
#
|
|
466
|
+
# Same engine selection as {detect_language}, returned as the
|
|
467
|
+
# historical [code, confidence] pair.
|
|
468
|
+
#
|
|
469
|
+
# @param text [String] Text to analyze
|
|
470
|
+
# @return [Array<String, Float>] Language code and confidence
|
|
471
|
+
#
|
|
472
|
+
# @example
|
|
473
|
+
# lang, conf = Kotoshu.detect_language_with_confidence("Hello world")
|
|
474
|
+
# lang # => "en"
|
|
475
|
+
# conf # => 0.85
|
|
476
|
+
def self.detect_language_with_confidence(text)
|
|
477
|
+
detection = Language::LidDetector.detect(text)
|
|
478
|
+
[detection.code, detection.score]
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
# Set up the language-identification model: downloads the
|
|
482
|
+
# lid.176.onnx + vocab.json pair from the models registry into the
|
|
483
|
+
# model cache (stage one of the two-stage model — {detect_language}
|
|
484
|
+
# never downloads implicitly). Idempotent; +force+ re-downloads.
|
|
485
|
+
#
|
|
486
|
+
# @param force [Boolean] Re-fetch even if already cached
|
|
487
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
488
|
+
# @raise [Kotoshu::Error] registry unreachable while offline with no
|
|
489
|
+
# cached registry (KOTOSHU_OFFLINE=1), or no registry entry
|
|
490
|
+
# @raise [Kotoshu::IntegrityError] downloaded bytes fail the
|
|
491
|
+
# registry checksum
|
|
492
|
+
#
|
|
493
|
+
# @example
|
|
494
|
+
# Kotoshu.setup_lid
|
|
495
|
+
# Kotoshu.setup_lid? # => true
|
|
496
|
+
def self.setup_lid(force: false)
|
|
497
|
+
Language::LidDetector.setup(config: configuration, force: force)
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
# Whether the language-identification model is cached (cache-only).
|
|
501
|
+
#
|
|
502
|
+
# @return [Boolean]
|
|
503
|
+
def self.setup_lid?
|
|
504
|
+
Language::LidDetector.setup?(config: configuration)
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
# Get language class by code.
|
|
508
|
+
#
|
|
509
|
+
# @param code [String] Language code (e.g., "en-US", "de-DE")
|
|
510
|
+
# @return [Class, nil] Language class or nil
|
|
511
|
+
#
|
|
512
|
+
# @example
|
|
513
|
+
# Kotoshu.get_language("en-US")
|
|
514
|
+
def self.get_language(code)
|
|
515
|
+
Language.get(code)
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Check if a language is registered.
|
|
519
|
+
#
|
|
520
|
+
# @param code [String] Language code
|
|
521
|
+
# @return [Boolean] True if registered
|
|
522
|
+
#
|
|
523
|
+
# @example
|
|
524
|
+
# Kotoshu.language_registered?("en-US") # => true or false
|
|
525
|
+
def self.language_registered?(code)
|
|
526
|
+
Language.registered?(code)
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# Get all supported language codes.
|
|
530
|
+
#
|
|
531
|
+
# @return [Array<String>] List of language codes
|
|
532
|
+
#
|
|
533
|
+
# @example
|
|
534
|
+
# Kotoshu.supported_languages # => ["de-DE", "en-US", "fr-FR", ...]
|
|
535
|
+
def self.supported_languages
|
|
536
|
+
Language.supported_codes
|
|
537
|
+
end
|
|
538
|
+
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rb_sys/extensiontask"
|
|
4
|
+
|
|
5
|
+
# Load gemspec directly if GEMSPEC constant is not defined
|
|
6
|
+
gemspec = defined?(GEMSPEC) ? GEMSPEC : Gem::Specification.load("kotoshu.gemspec")
|
|
7
|
+
|
|
8
|
+
RbSys::ExtensionTask.new("kotoshu_native", gemspec) do |ext|
|
|
9
|
+
ext.lib_dir = "lib/kotoshu"
|
|
10
|
+
end
|