kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Suggestions
|
|
7
|
+
# A single suggestion with associated metadata and behavior.
|
|
8
|
+
#
|
|
9
|
+
# Serialized via lutaml-model. Use +to_hash+ / +Suggestion.as_json(instance)+ /
|
|
10
|
+
# +Suggestion.from_hash(hash)+ / +Suggestion.from_json(string)+ for the
|
|
11
|
+
# wire forms — no hand-rolled +to_h+ / +as_json+ on the model.
|
|
12
|
+
class Suggestion < Lutaml::Model::Serializable
|
|
13
|
+
attribute :word, :string
|
|
14
|
+
attribute :distance, :integer, default: 0
|
|
15
|
+
attribute :confidence, :float, default: 1.0
|
|
16
|
+
attribute :source, :string, default: "unknown"
|
|
17
|
+
attribute :metadata, :hash, default: {}
|
|
18
|
+
|
|
19
|
+
# Support the legacy +**metadata+ kwarg catch-all so existing callers
|
|
20
|
+
# (e.g., +Suggestion.new(word:, distance:, source:, original_length: 5)+)
|
|
21
|
+
# continue to work; extra kwargs land in the +metadata+ attribute.
|
|
22
|
+
# Source is stored as a string for clean serialization; +from_source?+
|
|
23
|
+
# normalizes Symbol/String comparison so callers can pass either.
|
|
24
|
+
#
|
|
25
|
+
# +word+ defaults to nil purely to accommodate lutaml-model's
|
|
26
|
+
# deserialization pathway, which allocates a shell via
|
|
27
|
+
# +new(lutaml_register:)+ before applying attribute values through
|
|
28
|
+
# the deserialize pipeline. Direct callers must still pass +word:+
|
|
29
|
+
# — a Suggestion without a word is degenerate and not user-facing.
|
|
30
|
+
def initialize(word: nil, distance: 0, confidence: 1.0, source: "unknown", **metadata)
|
|
31
|
+
lutaml_register = metadata.delete(:lutaml_register)
|
|
32
|
+
kwargs = {
|
|
33
|
+
word: word,
|
|
34
|
+
distance: distance,
|
|
35
|
+
confidence: confidence,
|
|
36
|
+
source: source.to_s,
|
|
37
|
+
metadata: metadata
|
|
38
|
+
}
|
|
39
|
+
kwargs[:lutaml_register] = lutaml_register if lutaml_register
|
|
40
|
+
super(**kwargs)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Check if this is a high-confidence suggestion.
|
|
44
|
+
#
|
|
45
|
+
# @return [Boolean] True if confidence >= 0.8
|
|
46
|
+
def high_confidence?
|
|
47
|
+
confidence >= 0.8
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Check if this is a low-confidence suggestion.
|
|
51
|
+
#
|
|
52
|
+
# @return [Boolean] True if confidence < 0.5
|
|
53
|
+
def low_confidence?
|
|
54
|
+
confidence < 0.5
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Calculate combined score considering distance and confidence.
|
|
58
|
+
#
|
|
59
|
+
# @param distance_weight [Float] Weight for distance (default: 0.3)
|
|
60
|
+
# @param confidence_weight [Float] Weight for confidence (default: 0.7)
|
|
61
|
+
# @return [Float] Combined score (0.0 to 1.0, higher is better)
|
|
62
|
+
def combined_score(distance_weight: 0.3, confidence_weight: 0.7)
|
|
63
|
+
normalized_distance = [distance, 5].min / 5.0
|
|
64
|
+
distance_score = 1.0 - normalized_distance
|
|
65
|
+
|
|
66
|
+
(distance_score * distance_weight) + (confidence * confidence_weight)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Check if this suggestion is the same word as another.
|
|
70
|
+
#
|
|
71
|
+
# @param other [Suggestion, String] The other suggestion or word string
|
|
72
|
+
# @return [Boolean] True if words match (case-insensitive)
|
|
73
|
+
def same_word?(other)
|
|
74
|
+
other_word = other.is_a?(Suggestion) ? other.word : other.to_s
|
|
75
|
+
word.downcase == other_word.downcase
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Check if this suggestion comes from a specific source.
|
|
79
|
+
#
|
|
80
|
+
# Source is stored as a string; comparison normalizes Symbol/String.
|
|
81
|
+
#
|
|
82
|
+
# @param source [String, Symbol] The source to check
|
|
83
|
+
# @return [Boolean] True if this suggestion came from the source
|
|
84
|
+
def from_source?(source)
|
|
85
|
+
self.source == source.to_s
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Compare suggestions for sorting (higher combined score first).
|
|
89
|
+
#
|
|
90
|
+
# Ranking priority (following CSpell/Hunspell approach):
|
|
91
|
+
# 1. Combined score (higher is better)
|
|
92
|
+
# 2. Edit distance (lower is better)
|
|
93
|
+
# 3. Length similarity (prefer similar length to original word)
|
|
94
|
+
# 4. N-gram similarity (more shared n-grams is better)
|
|
95
|
+
# 5. Alphabetical (ONLY as final tiebreaker)
|
|
96
|
+
#
|
|
97
|
+
# @param other [Suggestion] The other suggestion
|
|
98
|
+
# @return [Integer, nil] -1, 0, or 1; nil if +other+ is not a Suggestion
|
|
99
|
+
def <=>(other)
|
|
100
|
+
return nil unless other.is_a?(Suggestion)
|
|
101
|
+
|
|
102
|
+
score_cmp = other.combined_score <=> combined_score
|
|
103
|
+
return score_cmp unless score_cmp.zero?
|
|
104
|
+
|
|
105
|
+
distance_cmp = distance <=> other.distance
|
|
106
|
+
return distance_cmp unless distance_cmp.zero?
|
|
107
|
+
|
|
108
|
+
orig_len = metadata[:original_length] || word.length
|
|
109
|
+
other_orig_len = other.metadata[:original_length] || other.word.length
|
|
110
|
+
|
|
111
|
+
my_len_diff = (word.length - orig_len).abs
|
|
112
|
+
other_len_diff = (other.word.length - other_orig_len).abs
|
|
113
|
+
|
|
114
|
+
len_cmp = my_len_diff <=> other_len_diff
|
|
115
|
+
return len_cmp unless len_cmp.zero?
|
|
116
|
+
|
|
117
|
+
my_ngram = metadata[:ngram_score] || 0
|
|
118
|
+
other_ngram = other.metadata[:ngram_score] || 0
|
|
119
|
+
|
|
120
|
+
ngram_cmp = other_ngram <=> my_ngram
|
|
121
|
+
return ngram_cmp unless ngram_cmp.zero?
|
|
122
|
+
|
|
123
|
+
word.downcase <=> other.word.downcase
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def ==(other)
|
|
127
|
+
return false unless other.is_a?(Suggestion)
|
|
128
|
+
|
|
129
|
+
word.downcase == other.word.downcase
|
|
130
|
+
end
|
|
131
|
+
alias eql? ==
|
|
132
|
+
|
|
133
|
+
def hash
|
|
134
|
+
word.downcase.hash
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def to_s
|
|
138
|
+
format("Suggestion(word: '%<word>s', distance: %<distance>d, confidence: %<confidence>.2f, source: %<source>s)",
|
|
139
|
+
word: word, distance: distance, confidence: confidence, source: source)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
alias inspect to_s
|
|
143
|
+
|
|
144
|
+
# Create a suggestion from a simple word (convenience method).
|
|
145
|
+
#
|
|
146
|
+
# @param word [String] The word
|
|
147
|
+
# @param source [String, Symbol] The source
|
|
148
|
+
# @return [Suggestion] New suggestion
|
|
149
|
+
def self.from_word(word, source: :unknown)
|
|
150
|
+
new(word: word, distance: 0, confidence: 1.0, source: source)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# A collection of suggestions with rich query methods.
|
|
6
|
+
# This is MORE OOP than Spylls which returns plain iterators of strings.
|
|
7
|
+
class SuggestionSet
|
|
8
|
+
include Enumerable
|
|
9
|
+
|
|
10
|
+
# @return [Integer] Number of duplicate suggestions removed by the
|
|
11
|
+
# last call to {#deduplicate!}. Zero until deduplicate! runs.
|
|
12
|
+
attr_reader :suggestions, :max_size, :duplicates_removed
|
|
13
|
+
|
|
14
|
+
# @param suggestions [Array<Suggestion>] Initial suggestions
|
|
15
|
+
# @param max_size [Integer] Maximum number of suggestions to keep
|
|
16
|
+
# @param ranked [Boolean] When true, the producer already ranked,
|
|
17
|
+
# deduplicated, and limited these suggestions and the set adopts
|
|
18
|
+
# their order verbatim (only truncation to +max_size+ applies).
|
|
19
|
+
# Used by producers whose ranking is conformance-frozen — the
|
|
20
|
+
# native engine (NativeBackend), whose tie order can depend on
|
|
21
|
+
# ranking signals (e.g. ngram scores) not carried on the Suggestion
|
|
22
|
+
# objects themselves; re-sorting here would permute those ties.
|
|
23
|
+
def initialize(suggestions = [], max_size: 10, ranked: false)
|
|
24
|
+
@suggestions = suggestions
|
|
25
|
+
@max_size = max_size
|
|
26
|
+
@duplicates_removed = 0
|
|
27
|
+
ranked ? limit! : sort_and_limit!
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Add a suggestion to the set.
|
|
31
|
+
#
|
|
32
|
+
# @param suggestion [Suggestion] The suggestion to add
|
|
33
|
+
# @return [SuggestionSet] Self for chaining
|
|
34
|
+
def add(suggestion)
|
|
35
|
+
@suggestions << suggestion
|
|
36
|
+
sort_and_limit!
|
|
37
|
+
self
|
|
38
|
+
end
|
|
39
|
+
alias << add
|
|
40
|
+
|
|
41
|
+
# Add multiple suggestions.
|
|
42
|
+
#
|
|
43
|
+
# @param new_suggestions [Array<Suggestion>] Suggestions to add
|
|
44
|
+
# @return [SuggestionSet] Self for chaining
|
|
45
|
+
def concat(new_suggestions)
|
|
46
|
+
@suggestions.concat(new_suggestions)
|
|
47
|
+
sort_and_limit!
|
|
48
|
+
self
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Merge another suggestion set into this one.
|
|
52
|
+
#
|
|
53
|
+
# @param other [SuggestionSet] The other set
|
|
54
|
+
# @return [SuggestionSet] Self for chaining
|
|
55
|
+
def merge!(other)
|
|
56
|
+
concat(other.suggestions)
|
|
57
|
+
self
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Get suggestions by source.
|
|
61
|
+
#
|
|
62
|
+
# @param source [String, Symbol] The source to filter by
|
|
63
|
+
# @return [SuggestionSet] New set with filtered suggestions
|
|
64
|
+
def from_source(source)
|
|
65
|
+
SuggestionSet.new(@suggestions.select { |s| s.from_source?(source) }, max_size: @max_size)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Get high-confidence suggestions.
|
|
69
|
+
#
|
|
70
|
+
# @return [SuggestionSet] New set with high-confidence suggestions
|
|
71
|
+
def high_confidence
|
|
72
|
+
SuggestionSet.new(@suggestions.select(&:high_confidence?), max_size: @max_size)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Get low-confidence suggestions.
|
|
76
|
+
#
|
|
77
|
+
# @return [SuggestionSet] New set with low-confidence suggestions
|
|
78
|
+
def low_confidence
|
|
79
|
+
SuggestionSet.new(@suggestions.select(&:low_confidence?), max_size: @max_size)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Get suggestions within a distance range.
|
|
83
|
+
#
|
|
84
|
+
# @param min_distance [Integer] Minimum distance
|
|
85
|
+
# @param max_distance [Integer] Maximum distance
|
|
86
|
+
# @return [SuggestionSet] New set with filtered suggestions
|
|
87
|
+
def within_distance(min_distance: 0, max_distance: 2)
|
|
88
|
+
filtered = @suggestions.select do |s|
|
|
89
|
+
s.distance >= min_distance && s.distance <= max_distance
|
|
90
|
+
end
|
|
91
|
+
SuggestionSet.new(filtered, max_size: @max_size)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Check if set contains a specific word.
|
|
95
|
+
#
|
|
96
|
+
# @param word [String] The word to check
|
|
97
|
+
# @return [Boolean] True if word is in suggestions
|
|
98
|
+
def include?(word)
|
|
99
|
+
@suggestions.any? { |s| s.same_word?(word) }
|
|
100
|
+
end
|
|
101
|
+
alias has_word? include?
|
|
102
|
+
|
|
103
|
+
# Find a suggestion by word.
|
|
104
|
+
#
|
|
105
|
+
# @param word [String] The word to find
|
|
106
|
+
# @return [Suggestion, nil] The suggestion or nil
|
|
107
|
+
def find_word(word)
|
|
108
|
+
@suggestions.find { |s| s.same_word?(word) }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Get the top N suggestions.
|
|
112
|
+
#
|
|
113
|
+
# @param n [Integer] Number of suggestions to get
|
|
114
|
+
# @return [Array<Suggestion>] Top N suggestions
|
|
115
|
+
def top(n)
|
|
116
|
+
@suggestions.first(n)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Get the first (best) suggestion.
|
|
120
|
+
#
|
|
121
|
+
# @return [Suggestion, nil] The best suggestion or nil
|
|
122
|
+
def first
|
|
123
|
+
@suggestions.first
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Get the last suggestion.
|
|
127
|
+
#
|
|
128
|
+
# @return [Suggestion, nil] The last suggestion or nil
|
|
129
|
+
def last
|
|
130
|
+
@suggestions.last
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Check if the set is empty.
|
|
134
|
+
#
|
|
135
|
+
# @return [Boolean] True if no suggestions
|
|
136
|
+
def empty?
|
|
137
|
+
@suggestions.empty?
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Get the number of suggestions.
|
|
141
|
+
#
|
|
142
|
+
# @return [Integer] Number of suggestions
|
|
143
|
+
def size
|
|
144
|
+
@suggestions.size
|
|
145
|
+
end
|
|
146
|
+
alias count size
|
|
147
|
+
alias length size
|
|
148
|
+
|
|
149
|
+
# Iterate over suggestions.
|
|
150
|
+
#
|
|
151
|
+
# @yield [suggestion] Each suggestion
|
|
152
|
+
# @return [Enumerator] Enumerator if no block given
|
|
153
|
+
def each(&)
|
|
154
|
+
return enum_for(:each) unless block_given?
|
|
155
|
+
|
|
156
|
+
@suggestions.each(&)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Get unique suggestions (by word, case-insensitive).
|
|
160
|
+
#
|
|
161
|
+
# @return [SuggestionSet] New set with unique suggestions
|
|
162
|
+
def unique
|
|
163
|
+
seen = {}
|
|
164
|
+
unique_suggestions = @suggestions.select do |s|
|
|
165
|
+
word = s.word.downcase
|
|
166
|
+
if seen[word]
|
|
167
|
+
false
|
|
168
|
+
else
|
|
169
|
+
seen[word] = true
|
|
170
|
+
true
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
SuggestionSet.new(unique_suggestions, max_size: @max_size)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Convert to array of words.
|
|
177
|
+
#
|
|
178
|
+
# @return [Array<String>] Array of suggestion words
|
|
179
|
+
def to_words
|
|
180
|
+
@suggestions.map(&:word)
|
|
181
|
+
end
|
|
182
|
+
alias words to_words
|
|
183
|
+
|
|
184
|
+
# Convert to an array of hashes for serialization.
|
|
185
|
+
#
|
|
186
|
+
# Use this when you need the Hash shape (e.g. JSON output). For the
|
|
187
|
+
# enumerable contract, {#to_a} / {#each} yield {Suggestion} objects.
|
|
188
|
+
#
|
|
189
|
+
# @return [Array<Hash>] Array of suggestion hashes
|
|
190
|
+
def to_hashes
|
|
191
|
+
@suggestions.map(&:to_hash)
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Convert to JSON-compatible array.
|
|
195
|
+
#
|
|
196
|
+
# @return [Array<Hash>] JSON-compatible array
|
|
197
|
+
def as_json(*)
|
|
198
|
+
to_hashes
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# String representation.
|
|
202
|
+
#
|
|
203
|
+
# @return [String] String representation
|
|
204
|
+
def to_s
|
|
205
|
+
"SuggestionSet(size: #{size}, max_size: #{@max_size})"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Inspect the suggestion set.
|
|
209
|
+
#
|
|
210
|
+
# @return [String] Inspection string
|
|
211
|
+
def inspect
|
|
212
|
+
if @suggestions.empty?
|
|
213
|
+
to_s
|
|
214
|
+
else
|
|
215
|
+
"#{self} [#{@suggestions.map(&:word).join(', ')}]"
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Create an empty suggestion set.
|
|
220
|
+
#
|
|
221
|
+
# @param max_size [Integer] Maximum size
|
|
222
|
+
# @return [SuggestionSet] Empty set
|
|
223
|
+
def self.empty(max_size: 10)
|
|
224
|
+
new([], max_size: max_size)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Create a suggestion set from an array of words.
|
|
228
|
+
#
|
|
229
|
+
# @param words [Array<String>] Array of words
|
|
230
|
+
# @param source [String, Symbol] The source
|
|
231
|
+
# @param max_size [Integer] Maximum size
|
|
232
|
+
# @return [SuggestionSet] New set
|
|
233
|
+
def self.from_words(words, source: :unknown, max_size: 10)
|
|
234
|
+
suggestions = words.map { |w| Suggestion.from_word(w, source: source) }
|
|
235
|
+
new(suggestions, max_size: max_size)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
private
|
|
239
|
+
|
|
240
|
+
# Sort suggestions by combined score.
|
|
241
|
+
def sort!
|
|
242
|
+
@suggestions.sort!
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Remove duplicate suggestions (same word, case-insensitive).
|
|
246
|
+
# Called explicitly so the dedup is visible as a named step,
|
|
247
|
+
# not buried inside sort_and_limit as a side effect (TODO 56 T5.1).
|
|
248
|
+
# Tracks the count of removed duplicates for diagnostics.
|
|
249
|
+
def deduplicate!
|
|
250
|
+
before = @suggestions.length
|
|
251
|
+
@suggestions.uniq! { |s| s.word.downcase }
|
|
252
|
+
@duplicates_removed = before - @suggestions.length
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Truncate to max_size.
|
|
256
|
+
def limit!
|
|
257
|
+
@suggestions = @suggestions.first(@max_size)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Sort, deduplicate, then limit. The canonical pipeline called
|
|
261
|
+
# after every mutation (add, concat, merge!). Each step is a
|
|
262
|
+
# named method so callers can reason about the order.
|
|
263
|
+
def sort_and_limit!
|
|
264
|
+
sort!
|
|
265
|
+
deduplicate!
|
|
266
|
+
limit!
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Suggestions
|
|
5
|
+
# Per-dictionary sweep invariants, built once at the first
|
|
6
|
+
# suggestion sweep and kept for the dictionary's lifetime.
|
|
7
|
+
#
|
|
8
|
+
# Every strategy used to walk the whole word list per sweep and
|
|
9
|
+
# compute per-word values before its gate — the length window, the
|
|
10
|
+
# Soundex code, the n-gram length bound — so a full dictionary paid
|
|
11
|
+
# for work on words a sweep was about to skip. Those per-word
|
|
12
|
+
# values never change with the query: they are computed here one
|
|
13
|
+
# time (char lengths, packed Soundex codes, length buckets), and
|
|
14
|
+
# the sweeps touch only the entries their gates admit.
|
|
15
|
+
#
|
|
16
|
+
# Soundex codes are the 4-character {Algorithms::Soundex} strings
|
|
17
|
+
# (frozen); two letter-less words both code to +""+ and compare
|
|
18
|
+
# equal, exactly as the live computation does. Lengths are char
|
|
19
|
+
# counts. +by_length+ groups word indices by length in original
|
|
20
|
+
# order so the edit-distance sweep can iterate only its ±2 window;
|
|
21
|
+
# {#indices_in_length_range} restores word-list order because ties
|
|
22
|
+
# in the strategies' ranking sorts are decided by input order.
|
|
23
|
+
#
|
|
24
|
+
# The index derives only from the word list it was built over.
|
|
25
|
+
# Dictionary backends memoize it (see
|
|
26
|
+
# {Dictionary::Base#sweep_index}) and invalidate the memo wherever
|
|
27
|
+
# they mutate the word list ({Dictionary::Base#add_word} /
|
|
28
|
+
# {Dictionary::Base#remove_word} implementations).
|
|
29
|
+
#
|
|
30
|
+
# This mirrors SweepIndex in kotoshu-rs
|
|
31
|
+
# (kotoshu/src/suggest/sweep_index.rs).
|
|
32
|
+
class SweepIndex
|
|
33
|
+
# @return [Array<String>] The words in list order (as captured at
|
|
34
|
+
# build time)
|
|
35
|
+
attr_reader :words
|
|
36
|
+
|
|
37
|
+
# Build an index over a word list.
|
|
38
|
+
#
|
|
39
|
+
# @param words [Array<String>] Dictionary words in list order
|
|
40
|
+
# @return [SweepIndex] The built index
|
|
41
|
+
def self.build(words)
|
|
42
|
+
lengths = Array.new(words.length)
|
|
43
|
+
soundex_codes = Array.new(words.length)
|
|
44
|
+
by_length = {}
|
|
45
|
+
words.each_with_index do |word, idx|
|
|
46
|
+
length = word.length
|
|
47
|
+
lengths[idx] = length
|
|
48
|
+
soundex_codes[idx] = Algorithms::Soundex.code(word).freeze
|
|
49
|
+
(by_length[length] ||= []) << idx
|
|
50
|
+
end
|
|
51
|
+
new(words, lengths, soundex_codes, by_length)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Create an index from prebuilt structures.
|
|
55
|
+
#
|
|
56
|
+
# @param words [Array<String>] The indexed words in list order
|
|
57
|
+
# @param lengths [Array<Integer>] Char length per word index
|
|
58
|
+
# @param soundex_codes [Array<String>] Soundex code per word index
|
|
59
|
+
# @param by_length [Hash<Integer, Array<Integer>>] Length => word
|
|
60
|
+
# indices in list order
|
|
61
|
+
def initialize(words, lengths, soundex_codes, by_length)
|
|
62
|
+
@words = words
|
|
63
|
+
@lengths = lengths
|
|
64
|
+
@soundex_codes = soundex_codes
|
|
65
|
+
@by_length = by_length
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Char length of the word at +idx+.
|
|
69
|
+
#
|
|
70
|
+
# @param idx [Integer] Word index
|
|
71
|
+
# @return [Integer] Char count
|
|
72
|
+
def length(idx)
|
|
73
|
+
@lengths[idx]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Soundex code of the word at +idx+ — equal codes compare equal,
|
|
77
|
+
# including two letter-less empty codes.
|
|
78
|
+
#
|
|
79
|
+
# @param idx [Integer] Word index
|
|
80
|
+
# @return [String] The frozen 4-char (or empty) Soundex code
|
|
81
|
+
def soundex(idx)
|
|
82
|
+
@soundex_codes[idx]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Word indices whose length is within +min..max+, every word in
|
|
86
|
+
# exactly one bucket, each bucket in original order.
|
|
87
|
+
#
|
|
88
|
+
# @param min [Integer] Minimum length (inclusive)
|
|
89
|
+
# @param max [Integer] Maximum length (inclusive)
|
|
90
|
+
# @return [Array<Integer>] Matching indices in word-list order
|
|
91
|
+
def indices_in_length_range(min, max)
|
|
92
|
+
hits = []
|
|
93
|
+
(min..max).each do |length|
|
|
94
|
+
bucket = @by_length[length]
|
|
95
|
+
hits.concat(bucket) if bucket
|
|
96
|
+
end
|
|
97
|
+
# Restore the word-list order the strategies' candidate arrays
|
|
98
|
+
# were built in before this index existed — ties in the ranking
|
|
99
|
+
# sort are decided by input order, so this must not drift.
|
|
100
|
+
hits.sort!
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Words whose length is within +min..max+, in word-list order —
|
|
104
|
+
# exactly the set and order {Dictionary::Base#find_by_length_range}
|
|
105
|
+
# historically returned.
|
|
106
|
+
#
|
|
107
|
+
# @param min [Integer] Minimum length (inclusive)
|
|
108
|
+
# @param max [Integer] Maximum length (inclusive)
|
|
109
|
+
# @return [Array<String>] Matching words in word-list order
|
|
110
|
+
def words_in_length_range(min, max)
|
|
111
|
+
indices_in_length_range(min, max).map { |idx| @words[idx] }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Yield every word with its memoized char length.
|
|
115
|
+
#
|
|
116
|
+
# @yield [String, Integer] Each word and its char count
|
|
117
|
+
# @return [void]
|
|
118
|
+
def each_with_length
|
|
119
|
+
@words.each_with_index do |word, idx|
|
|
120
|
+
yield word, @lengths[idx]
|
|
121
|
+
end
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Yield every word with its memoized Soundex code.
|
|
126
|
+
#
|
|
127
|
+
# @yield [String, String] Each word and its Soundex code
|
|
128
|
+
# @return [void]
|
|
129
|
+
def each_with_soundex
|
|
130
|
+
@words.each_with_index do |word, idx|
|
|
131
|
+
yield word, @soundex_codes[idx]
|
|
132
|
+
end
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Suggestion generation system: strategies, generator, and result types.
|
|
5
|
+
module Suggestions
|
|
6
|
+
autoload :Context, "kotoshu/suggestions/context"
|
|
7
|
+
autoload :FrequencyProvider, "kotoshu/suggestions/frequency_provider"
|
|
8
|
+
autoload :FrozenTiers, "kotoshu/data/frozen_kelly"
|
|
9
|
+
autoload :Generator, "kotoshu/suggestions/generator"
|
|
10
|
+
autoload :Pipeline, "kotoshu/suggestions/pipeline"
|
|
11
|
+
autoload :SemanticCascade, "kotoshu/suggestions/semantic_cascade"
|
|
12
|
+
autoload :Suggestion, "kotoshu/suggestions/suggestion"
|
|
13
|
+
autoload :SuggestionSet, "kotoshu/suggestions/suggestion_set"
|
|
14
|
+
autoload :SweepIndex, "kotoshu/suggestions/sweep_index"
|
|
15
|
+
|
|
16
|
+
# Strategies sub-namespace.
|
|
17
|
+
module Strategies
|
|
18
|
+
autoload :BaseStrategy, "kotoshu/suggestions/strategies/base_strategy"
|
|
19
|
+
autoload :CompositeStrategy, "kotoshu/suggestions/strategies/composite_strategy"
|
|
20
|
+
autoload :EditDistanceStrategy, "kotoshu/suggestions/strategies/edit_distance_strategy"
|
|
21
|
+
autoload :KeyboardProximityStrategy, "kotoshu/suggestions/strategies/keyboard_proximity_strategy"
|
|
22
|
+
autoload :NgramStrategy, "kotoshu/suggestions/strategies/ngram_strategy"
|
|
23
|
+
autoload :PhoneticStrategy, "kotoshu/suggestions/strategies/phonetic_strategy"
|
|
24
|
+
autoload :SemanticStrategy, "kotoshu/suggestions/strategies/semantic_strategy"
|
|
25
|
+
autoload :SymspellStrategy, "kotoshu/suggestions/strategies/symspell_strategy"
|
|
26
|
+
autoload :WordIndex, "kotoshu/suggestions/strategies/word_index"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Rake integration namespace (plan 89). Requiring
|
|
5
|
+
# `kotoshu/tasks` installs the default {CheckTask}; build custom
|
|
6
|
+
# tasks by constructing {CheckTask} yourself.
|
|
7
|
+
module Tasks
|
|
8
|
+
# Rake task checking a repository text files with the plan-88
|
|
9
|
+
# file selection (plan 89, item 3).
|
|
10
|
+
#
|
|
11
|
+
# @example Default task via require
|
|
12
|
+
# # Rakefile
|
|
13
|
+
# require "kotoshu/tasks"
|
|
14
|
+
# # provides rake kotoshu:check over the repo text files
|
|
15
|
+
#
|
|
16
|
+
# @example Configured
|
|
17
|
+
# Kotoshu::Tasks::CheckTask.new do |t|
|
|
18
|
+
# t.files = FileList["_posts/**/*.md"]
|
|
19
|
+
# t.language = "en"
|
|
20
|
+
# t.fail_on_error = true
|
|
21
|
+
# end
|
|
22
|
+
class CheckTask
|
|
23
|
+
include Rake::DSL
|
|
24
|
+
|
|
25
|
+
# @param name [Symbol] task name under the kotoshu namespace
|
|
26
|
+
# @yieldparam task [CheckTask] self for configuration
|
|
27
|
+
def initialize(name = :check, &block)
|
|
28
|
+
@name = name
|
|
29
|
+
@files = nil
|
|
30
|
+
@root = Dir.pwd
|
|
31
|
+
@language = nil
|
|
32
|
+
@fail_on_error = true
|
|
33
|
+
@baseline_path = nil
|
|
34
|
+
yield self if block
|
|
35
|
+
|
|
36
|
+
define
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
attr_accessor :name, :files, :root, :language, :fail_on_error, :baseline_path
|
|
40
|
+
|
|
41
|
+
# The files the task will check: the configured list, or the
|
|
42
|
+
# plan-88 selection over the root.
|
|
43
|
+
#
|
|
44
|
+
# @return [Array<String>]
|
|
45
|
+
def target_files
|
|
46
|
+
return Array(files) if files
|
|
47
|
+
|
|
48
|
+
Cli::DirectoryWalker.new.files(root)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# Register the kotoshu:<name> task.
|
|
54
|
+
def define
|
|
55
|
+
desc "Check spelling in the repository text files"
|
|
56
|
+
namespace :kotoshu do
|
|
57
|
+
task(name) { run }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Run the check and abort on errors when fail_on_error.
|
|
62
|
+
def run
|
|
63
|
+
runner = Cli::DirectoryCheck.new(
|
|
64
|
+
target_files,
|
|
65
|
+
check: ->(text, suggestions_filter: nil) { check_text(text, suggestions_filter: suggestions_filter) },
|
|
66
|
+
baseline_path: baseline_path
|
|
67
|
+
)
|
|
68
|
+
runner.run
|
|
69
|
+
report(runner)
|
|
70
|
+
if fail_on_error && runner.failed?
|
|
71
|
+
abort("kotoshu: #{runner.error_count} spelling error(s) in " \
|
|
72
|
+
"#{runner.file_count} file(s)")
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Check one text through the configured language.
|
|
77
|
+
#
|
|
78
|
+
# @param text [String] file contents
|
|
79
|
+
# @param suggestions_filter [#call, nil] baseline budget filter
|
|
80
|
+
# (plan 116) — see Baseline::Store#suggestions_filter_for
|
|
81
|
+
# @return [Models::Result::DocumentResult]
|
|
82
|
+
def check_text(text, suggestions_filter: nil)
|
|
83
|
+
if language
|
|
84
|
+
Kotoshu.spellchecker_for(language).check(text, suggestions_filter: suggestions_filter)
|
|
85
|
+
else
|
|
86
|
+
Kotoshu.check(text, suggestions_filter: suggestions_filter)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Print one line per file with errors plus a summary.
|
|
91
|
+
def report(runner)
|
|
92
|
+
runner.outcomes.each do |outcome|
|
|
93
|
+
next if outcome.result.success?
|
|
94
|
+
|
|
95
|
+
outcome.result.each_error do |error|
|
|
96
|
+
suggestions = error.has_suggestions? ? " -> #{error.top_suggestions(3).join(', ')}" : ""
|
|
97
|
+
puts "#{outcome.path}: #{error.word}#{suggestions}"
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
puts "#{runner.file_count} file(s) checked, #{runner.error_count} error(s)"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|