kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# ONNX embedding model implementation.
|
|
6
|
+
#
|
|
7
|
+
# Loads FastText models converted to ONNX format for faster inference.
|
|
8
|
+
# Uses ONNX Runtime for efficient embedding lookup.
|
|
9
|
+
#
|
|
10
|
+
# @example Loading from file
|
|
11
|
+
# model = OnnxModel.from_file('fasttext.en.onnx')
|
|
12
|
+
# embedding = model.embedding_for('hello')
|
|
13
|
+
#
|
|
14
|
+
# @example Loading from GitHub (via ModelCache)
|
|
15
|
+
# model = OnnxModel.from_github('en')
|
|
16
|
+
# neighbors = model.nearest_neighbors('hello', k: 10)
|
|
17
|
+
class OnnxModel < EmbeddingModel
|
|
18
|
+
# Soft-load onnxruntime. The gem is intentionally NOT a hard runtime
|
|
19
|
+
# dependency — it fails to build on some platforms and would block
|
|
20
|
+
# install for users who only want traditional spell-checking. Semantic
|
|
21
|
+
# features light up automatically when the gem is present.
|
|
22
|
+
#
|
|
23
|
+
# KOTOSHU_NO_ONNX=1 forces semantic analysis off even when the gem is
|
|
24
|
+
# installed (useful for benchmarks / CI determinism).
|
|
25
|
+
ONNX_LOADED = begin
|
|
26
|
+
if ENV["KOTOSHU_NO_ONNX"] == "1"
|
|
27
|
+
false
|
|
28
|
+
else
|
|
29
|
+
require "onnxruntime"
|
|
30
|
+
true
|
|
31
|
+
end
|
|
32
|
+
rescue LoadError
|
|
33
|
+
false
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Error raised when semantic features are requested but onnxruntime
|
|
37
|
+
# is unavailable. Caller-friendly message points at the fix.
|
|
38
|
+
class OnnxUnavailable < Kotoshu::Error
|
|
39
|
+
def initialize(detail = nil)
|
|
40
|
+
message = "onnxruntime gem not loaded"
|
|
41
|
+
message += " (#{detail})" if detail
|
|
42
|
+
message += ". Install with: gem install onnxruntime"
|
|
43
|
+
message += ". Or set KOTOSHU_NO_ONNX=1 to silence this in code paths that opt out."
|
|
44
|
+
super(message)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Default dimension for FastText models
|
|
49
|
+
DEFAULT_DIMENSION = 300
|
|
50
|
+
|
|
51
|
+
attr_reader :onnx_path, :vocabulary, :embedding_matrix
|
|
52
|
+
|
|
53
|
+
# Create a new ONNX model.
|
|
54
|
+
#
|
|
55
|
+
# @param language_code [String] ISO 639-1 language code
|
|
56
|
+
# @param dimension [Integer] Vector dimension
|
|
57
|
+
# @param onnx_path [String] Path to .onnx file
|
|
58
|
+
# @param vocabulary [Hash<String, Integer>] Word-to-index mapping
|
|
59
|
+
# @param embedding_matrix [Numo::SFloat] Pre-loaded embeddings (optional)
|
|
60
|
+
def initialize(language_code:, onnx_path:, vocabulary:, dimension: DEFAULT_DIMENSION, embedding_matrix: nil)
|
|
61
|
+
super(language_code: language_code, dimension: dimension)
|
|
62
|
+
@onnx_path = onnx_path
|
|
63
|
+
@vocabulary = vocabulary.freeze
|
|
64
|
+
@vocabulary_size = @vocabulary.size
|
|
65
|
+
|
|
66
|
+
# Pre-load embedding matrix if provided (for faster nearest neighbor search)
|
|
67
|
+
@embedding_matrix = embedding_matrix
|
|
68
|
+
|
|
69
|
+
# Lazy load session
|
|
70
|
+
@session = nil
|
|
71
|
+
@input_name = nil
|
|
72
|
+
@output_name = nil
|
|
73
|
+
@loaded = false
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Load ONNX model from a file.
|
|
77
|
+
#
|
|
78
|
+
# @param onnx_path [String] Path to .onnx file
|
|
79
|
+
# @param language_code [String] Language code (auto-detected from filename)
|
|
80
|
+
# @return [OnnxModel] Loaded model
|
|
81
|
+
# @raise [ArgumentError] if file doesn't exist
|
|
82
|
+
def self.from_file(onnx_path, language_code: nil)
|
|
83
|
+
raise ArgumentError, "File not found: #{onnx_path}" unless File.exist?(onnx_path)
|
|
84
|
+
|
|
85
|
+
# Detect language from filename if not provided
|
|
86
|
+
language_code ||= detect_language_from_path(onnx_path)
|
|
87
|
+
|
|
88
|
+
# Load vocabulary from .vocab.json file
|
|
89
|
+
vocab_path = onnx_path.sub('.onnx', '.vocab.json')
|
|
90
|
+
unless File.exist?(vocab_path)
|
|
91
|
+
raise ArgumentError, "Vocabulary file not found: #{vocab_path}"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
require 'json'
|
|
95
|
+
# The vocab sibling ships in the wrapped {"vocab_size", "word_to_idx"}
|
|
96
|
+
# shape (written by fasttext_to_onnx.py); normalize it to a flat
|
|
97
|
+
# word-to-index map via the shared Vocabulary parser.
|
|
98
|
+
vocabulary = Embeddings::Vocabulary.word_to_index_from_document(
|
|
99
|
+
JSON.parse(File.read(vocab_path))
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# Load metadata
|
|
103
|
+
metadata_path = onnx_path.sub('.onnx', '.metadata.json')
|
|
104
|
+
dimension = DEFAULT_DIMENSION
|
|
105
|
+
|
|
106
|
+
if File.exist?(metadata_path)
|
|
107
|
+
metadata = JSON.parse(File.read(metadata_path))
|
|
108
|
+
dimension = metadata['dimension']
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
new(
|
|
112
|
+
language_code: language_code,
|
|
113
|
+
dimension: dimension,
|
|
114
|
+
onnx_path: onnx_path,
|
|
115
|
+
vocabulary: vocabulary
|
|
116
|
+
)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Load ONNX model from GitHub (via ModelCache).
|
|
120
|
+
#
|
|
121
|
+
# Downloads the .onnx file from kotoshu/dictionaries repository.
|
|
122
|
+
#
|
|
123
|
+
# @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
|
|
124
|
+
# @param cache [ModelCache, nil] Optional cache instance
|
|
125
|
+
# @return [OnnxModel] Loaded model
|
|
126
|
+
# @raise [ArgumentError] if language not supported
|
|
127
|
+
def self.from_github(language_code, cache: nil)
|
|
128
|
+
cache ||= Cache::ModelCache.new
|
|
129
|
+
|
|
130
|
+
# Get the .onnx file path from cache
|
|
131
|
+
onnx_file = cache.get_onnx_model(language_code)
|
|
132
|
+
|
|
133
|
+
from_file(onnx_file, language_code: language_code)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Get embedding vector for a word.
|
|
137
|
+
#
|
|
138
|
+
# @param word [String] The word to lookup
|
|
139
|
+
# @return [WordEmbedding, nil] Embedding vector or nil if not found
|
|
140
|
+
def embedding_for(word)
|
|
141
|
+
return nil if word.nil? || word.empty?
|
|
142
|
+
|
|
143
|
+
index = @vocabulary[word]
|
|
144
|
+
return nil unless index
|
|
145
|
+
|
|
146
|
+
# Get embedding from ONNX model
|
|
147
|
+
vector = get_embedding_vector(index)
|
|
148
|
+
|
|
149
|
+
WordEmbedding.new(word, vector, @language_code, dimension: @dimension)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Get the vocabulary (all words in the model).
|
|
153
|
+
#
|
|
154
|
+
# @return [Array<String>] Vocabulary words
|
|
155
|
+
def vocabulary
|
|
156
|
+
@vocabulary.keys
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Check if model is loaded.
|
|
160
|
+
#
|
|
161
|
+
# @return [Boolean] True if ONNX session is loaded
|
|
162
|
+
def loaded?
|
|
163
|
+
@loaded
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Find k nearest neighbors for a word.
|
|
167
|
+
#
|
|
168
|
+
# @param word [String] The query word
|
|
169
|
+
# @param k [Integer] Number of neighbors to return
|
|
170
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
|
|
171
|
+
def nearest_neighbors(word, k: 10)
|
|
172
|
+
ensure_session_loaded
|
|
173
|
+
|
|
174
|
+
# Get query embedding
|
|
175
|
+
query = embedding_for(word)
|
|
176
|
+
return [] unless query
|
|
177
|
+
|
|
178
|
+
# If embedding matrix is pre-loaded, use it for faster search
|
|
179
|
+
if @embedding_matrix
|
|
180
|
+
nearest_neighbors_from_matrix(query, k)
|
|
181
|
+
else
|
|
182
|
+
super
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Batch lookup of embeddings for multiple words.
|
|
187
|
+
#
|
|
188
|
+
# More efficient than individual lookups when using ONNX.
|
|
189
|
+
#
|
|
190
|
+
# @param words [Array<String>] Words to lookup
|
|
191
|
+
# @return [Hash<String, WordEmbedding>] Word to embedding mapping
|
|
192
|
+
def batch_embeddings(words)
|
|
193
|
+
ensure_session_loaded
|
|
194
|
+
|
|
195
|
+
indices = words.map { |w| @vocabulary[w] }
|
|
196
|
+
vectors = batch_get_embeddings(indices)
|
|
197
|
+
|
|
198
|
+
words.zip(indices, vectors).each_with_object({}) do |(word, idx, vec)|
|
|
199
|
+
next unless idx && vec
|
|
200
|
+
|
|
201
|
+
[word, WordEmbedding.new(word, vec, @language_code, dimension: @dimension)]
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Preload the embedding matrix into memory for faster nearest neighbor search.
|
|
206
|
+
#
|
|
207
|
+
# Useful when doing many nearest neighbor queries.
|
|
208
|
+
#
|
|
209
|
+
# @return [Boolean] True if loaded successfully
|
|
210
|
+
def preload_embedding_matrix
|
|
211
|
+
ensure_session_loaded
|
|
212
|
+
|
|
213
|
+
# Get all embeddings at once
|
|
214
|
+
all_indices = (0...@vocabulary_size).to_a
|
|
215
|
+
vectors = batch_get_embeddings(all_indices)
|
|
216
|
+
|
|
217
|
+
# Convert to matrix (using Numo::SFloat for efficiency)
|
|
218
|
+
require 'numo/narray'
|
|
219
|
+
@embedding_matrix = Numo::Sfloat.cast(vectors).reshape(@vocabulary_size, @dimension)
|
|
220
|
+
|
|
221
|
+
true
|
|
222
|
+
rescue StandardError => e
|
|
223
|
+
warn "Failed to preload embedding matrix: #{e.message}"
|
|
224
|
+
false
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
private
|
|
228
|
+
|
|
229
|
+
# Get embedding vector from ONNX model.
|
|
230
|
+
#
|
|
231
|
+
# @param index [Integer] Word index
|
|
232
|
+
# @return [Array<Float>] Embedding vector
|
|
233
|
+
def get_embedding_vector(index)
|
|
234
|
+
ensure_session_loaded
|
|
235
|
+
|
|
236
|
+
# The converted FastText graph takes a single int64 index (shape
|
|
237
|
+
# [1]) and emits one float32 vector.
|
|
238
|
+
result = @session.run([@output_name], { @input_name => [index] })
|
|
239
|
+
|
|
240
|
+
Array(result.first).map(&:to_f)
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Get embeddings for multiple indices.
|
|
244
|
+
#
|
|
245
|
+
# The graph accepts exactly one index per run, so batches are
|
|
246
|
+
# executed as sequential single lookups.
|
|
247
|
+
#
|
|
248
|
+
# @param indices [Array<Integer>] Word indices
|
|
249
|
+
# @return [Array<Array<Float>>] Embedding vectors
|
|
250
|
+
def batch_get_embeddings(indices)
|
|
251
|
+
valid_indices = indices.compact
|
|
252
|
+
|
|
253
|
+
return [] if valid_indices.empty?
|
|
254
|
+
|
|
255
|
+
valid_indices.map { |index| get_embedding_vector(index) }
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Find nearest neighbors using pre-loaded embedding matrix.
|
|
259
|
+
#
|
|
260
|
+
# @param query [WordEmbedding] Query embedding
|
|
261
|
+
# @param k [Integer] Number of neighbors
|
|
262
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors
|
|
263
|
+
def nearest_neighbors_from_matrix(query, k)
|
|
264
|
+
return [] unless @embedding_matrix
|
|
265
|
+
|
|
266
|
+
# Compute cosine similarity with all words
|
|
267
|
+
query_vec = Numo::Sfloat.cast(query.vector)
|
|
268
|
+
similarities = []
|
|
269
|
+
|
|
270
|
+
@vocabulary.each_with_index do |(word, idx)|
|
|
271
|
+
vec = @embedding_matrix[idx, true]
|
|
272
|
+
sim = cosine_similarity(query_vec, vec)
|
|
273
|
+
similarities << [word, sim]
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Sort by similarity and take top k
|
|
277
|
+
similarities.sort_by { |_, s| -s }.first(k).map do |word, sim|
|
|
278
|
+
NearestNeighbor.new(
|
|
279
|
+
word: word,
|
|
280
|
+
similarity: sim,
|
|
281
|
+
embedding: embedding_for(word)
|
|
282
|
+
)
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Calculate cosine similarity between two vectors.
|
|
287
|
+
#
|
|
288
|
+
# @param vec1 [Numo::SFloat] First vector
|
|
289
|
+
# @param vec2 [Numo::SFloat] Second vector
|
|
290
|
+
# @return [Float] Cosine similarity
|
|
291
|
+
def cosine_similarity(vec1, vec2)
|
|
292
|
+
dot = (vec1 * vec2).sum
|
|
293
|
+
norm1 = Math.sqrt((vec1**2).sum)
|
|
294
|
+
norm2 = Math.sqrt((vec2**2).sum)
|
|
295
|
+
|
|
296
|
+
return 0.0 if norm1.zero? || norm2.zero?
|
|
297
|
+
|
|
298
|
+
dot / (norm1 * norm2)
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Ensure ONNX session is loaded.
|
|
302
|
+
def ensure_session_loaded
|
|
303
|
+
return if @loaded
|
|
304
|
+
|
|
305
|
+
raise OnnxUnavailable unless ONNX_LOADED
|
|
306
|
+
|
|
307
|
+
# The class is InferenceSession (OnnxRuntime::Session does not
|
|
308
|
+
# exist); input/output names come from the graph — the
|
|
309
|
+
# fasttext_to_onnx converter emits "word_index"/"embedding".
|
|
310
|
+
@session = OnnxRuntime::InferenceSession.new(@onnx_path)
|
|
311
|
+
@input_name = @session.inputs.first[:name] || 'word_index'
|
|
312
|
+
@output_name = @session.outputs.first[:name] || 'embedding'
|
|
313
|
+
@loaded = true
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Detect language code from file path.
|
|
317
|
+
#
|
|
318
|
+
# @param path [String] File path
|
|
319
|
+
# @return [String] Detected language code
|
|
320
|
+
def self.detect_language_from_path(path)
|
|
321
|
+
# Extract from path like "fasttext.en.onnx"
|
|
322
|
+
if path =~ /\.([a-z]{2})\./i
|
|
323
|
+
Regexp.last_match(1).downcase
|
|
324
|
+
else
|
|
325
|
+
'en' # Default to English
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
end
|
|
330
|
+
end
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Unified semantic error (NO artificial spelling/grammar split!).
|
|
6
|
+
#
|
|
7
|
+
# Represents ANY kind of language error detected through semantic analysis.
|
|
8
|
+
# Uses semantic categories instead of traditional "spelling" vs "grammar" labels.
|
|
9
|
+
#
|
|
10
|
+
# Error types (semantic categories):
|
|
11
|
+
# - :word_choice - Wrong word for context (e.g., "desert" vs "dessert")
|
|
12
|
+
# - :verb_agreement - Subject-verb mismatch (e.g., "they is" → "they are")
|
|
13
|
+
# - :tense - Temporal inconsistency (e.g., "Yesterday I will go")
|
|
14
|
+
# - :orthographic - Actual typo/misspelling (e.g., "wrold" → "world")
|
|
15
|
+
# - :preposition - Wrong preposition (e.g., "bored of" → "bored with")
|
|
16
|
+
# - :article - Wrong article (e.g., "a apple" → "an apple")
|
|
17
|
+
# - :morphology - Wrong word form (e.g., "goed" → "went")
|
|
18
|
+
# - :capitalization - Capitalization error (e.g., "i am" → "I am")
|
|
19
|
+
# - :punctuation - Punctuation error (e.g., "its" vs "it's")
|
|
20
|
+
# - :style - Style/usage suggestion
|
|
21
|
+
#
|
|
22
|
+
# @example Creating a semantic error
|
|
23
|
+
# error = SemanticError.new(
|
|
24
|
+
# id: "error_1",
|
|
25
|
+
# location: Location.new(line: 5, column: 12),
|
|
26
|
+
# original: "desert",
|
|
27
|
+
# suggestions: [Suggestion.new("dessert", confidence: 0.92)],
|
|
28
|
+
# error_type: :word_choice,
|
|
29
|
+
# confidence: 0.92,
|
|
30
|
+
# context: context
|
|
31
|
+
# )
|
|
32
|
+
# Raised by {SemanticError.initialize} when +suggestions+ is empty
|
|
33
|
+
# and +allow_empty_suggestions:+ is false. The analyzer rescues
|
|
34
|
+
# this so words with no close matches are silently dropped rather
|
|
35
|
+
# than producing noise.
|
|
36
|
+
class EmptySuggestionsError < ArgumentError
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
class SemanticError
|
|
40
|
+
# Error type definitions with display names
|
|
41
|
+
ERROR_TYPES = {
|
|
42
|
+
word_choice: 'Word Choice',
|
|
43
|
+
verb_agreement: 'Verb Agreement',
|
|
44
|
+
tense: 'Tense',
|
|
45
|
+
orthographic: 'Spelling',
|
|
46
|
+
preposition: 'Preposition',
|
|
47
|
+
article: 'Article',
|
|
48
|
+
morphology: 'Word Form',
|
|
49
|
+
capitalization: 'Capitalization',
|
|
50
|
+
punctuation: 'Punctuation',
|
|
51
|
+
style: 'Style'
|
|
52
|
+
}.freeze
|
|
53
|
+
|
|
54
|
+
attr_reader :id, :source_range, :location, :original, :suggestions,
|
|
55
|
+
:error_type, :confidence, :context
|
|
56
|
+
|
|
57
|
+
# Create a new semantic error.
|
|
58
|
+
#
|
|
59
|
+
# @param id [String, Symbol] Unique identifier for this error
|
|
60
|
+
# @param source_range [Kotoshu::Documents::SourceRange, nil]
|
|
61
|
+
# Where the offending text lives in the original markup-bearing
|
|
62
|
+
# source. Carried verbatim so editors/plugins can highlight the
|
|
63
|
+
# actual range the user wrote.
|
|
64
|
+
# @param location [Object, nil] Legacy location holder (line/column
|
|
65
|
+
# object). Kept for backward compat with old callers; new code
|
|
66
|
+
# should pass +source_range+ instead.
|
|
67
|
+
# @param original [String] The original (incorrect) word/text
|
|
68
|
+
# @param suggestions [Array<Suggestion>] Suggested corrections
|
|
69
|
+
# @param error_type [Symbol] Error type (must be in ERROR_TYPES)
|
|
70
|
+
# @param confidence [Float] Confidence score (0.0 to 1.0)
|
|
71
|
+
# @param context [Context, nil] Context around the error
|
|
72
|
+
# @param allow_empty_suggestions [Boolean] When false (default),
|
|
73
|
+
# raises {EmptySuggestionsError} instead of building an error
|
|
74
|
+
# with zero suggestions.
|
|
75
|
+
# @raise [ArgumentError] if error_type is invalid
|
|
76
|
+
# @raise [ArgumentError] if confidence is outside [0, 1]
|
|
77
|
+
# @raise [EmptySuggestionsError] if suggestions is empty and
|
|
78
|
+
# allow_empty_suggestions is false
|
|
79
|
+
def initialize(id:, original:, suggestions:, error_type:, confidence:, source_range: nil, location: nil, context: nil,
|
|
80
|
+
allow_empty_suggestions: false)
|
|
81
|
+
raise ArgumentError, "Invalid error type: #{error_type}" unless ERROR_TYPES.key?(error_type)
|
|
82
|
+
raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
|
|
83
|
+
|
|
84
|
+
if (suggestions.nil? || suggestions.empty?) && !allow_empty_suggestions
|
|
85
|
+
raise EmptySuggestionsError,
|
|
86
|
+
"Suggestions cannot be empty (pass allow_empty_suggestions: true to override)"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
@id = id.to_s
|
|
90
|
+
@source_range = source_range
|
|
91
|
+
@location = location || source_range&.start
|
|
92
|
+
@original = original
|
|
93
|
+
@suggestions = (suggestions || []).sort_by(&:confidence).reverse.freeze
|
|
94
|
+
@error_type = error_type
|
|
95
|
+
@confidence = confidence
|
|
96
|
+
@context = context
|
|
97
|
+
|
|
98
|
+
freeze
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Get user-friendly display type name.
|
|
102
|
+
#
|
|
103
|
+
# @return [String] Display type name
|
|
104
|
+
def display_type
|
|
105
|
+
ERROR_TYPES[@error_type] || @error_type.to_s.capitalize
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Check if this is a high-confidence error.
|
|
109
|
+
#
|
|
110
|
+
# @return [Boolean] True if confidence > 0.8
|
|
111
|
+
def high_confidence?
|
|
112
|
+
@confidence > 0.8
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Get confidence level category.
|
|
116
|
+
#
|
|
117
|
+
# @return [Symbol] :high, :medium, or :low
|
|
118
|
+
def confidence_level
|
|
119
|
+
return :high if @confidence > 0.8
|
|
120
|
+
return :medium if @confidence > 0.5
|
|
121
|
+
|
|
122
|
+
:low
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Get the recommended (top) suggestion.
|
|
126
|
+
#
|
|
127
|
+
# @return [Suggestion] The highest-confidence suggestion
|
|
128
|
+
def recommended_suggestion
|
|
129
|
+
@suggestions.first
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Check if this error equals another.
|
|
133
|
+
#
|
|
134
|
+
# @param other [Object] Another object
|
|
135
|
+
# @return [Boolean] True if IDs match
|
|
136
|
+
def ==(other)
|
|
137
|
+
return false unless other.is_a?(SemanticError)
|
|
138
|
+
|
|
139
|
+
@id == other.id
|
|
140
|
+
end
|
|
141
|
+
alias_method :eql?, :==
|
|
142
|
+
|
|
143
|
+
# Hash code for hash table usage.
|
|
144
|
+
#
|
|
145
|
+
# @return [Integer] Hash code
|
|
146
|
+
def hash
|
|
147
|
+
@id.hash
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Comparison for sorting (by source position, then confidence).
|
|
151
|
+
#
|
|
152
|
+
# Errors are sorted by:
|
|
153
|
+
# 1. Source position (source_range.start when present; falls back
|
|
154
|
+
# to legacy +location+ when source_range is nil)
|
|
155
|
+
# 2. Confidence (highest first)
|
|
156
|
+
#
|
|
157
|
+
# @param other [SemanticError] Another error
|
|
158
|
+
# @return [Integer] Comparison result (-1, 0, 1)
|
|
159
|
+
def <=>(other)
|
|
160
|
+
return 0 unless other.is_a?(SemanticError)
|
|
161
|
+
|
|
162
|
+
a_pos = sort_position
|
|
163
|
+
b_pos = other.sort_position
|
|
164
|
+
pos_cmp = a_pos <=> b_pos
|
|
165
|
+
return pos_cmp unless pos_cmp.zero?
|
|
166
|
+
|
|
167
|
+
other.confidence <=> @confidence
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# String representation.
|
|
171
|
+
#
|
|
172
|
+
# @return [String] Human-readable representation
|
|
173
|
+
def to_s
|
|
174
|
+
"#{@location}: '#{@original}' → #{recommended_suggestion.word} [#{(@confidence * 100).to_i}%]"
|
|
175
|
+
end
|
|
176
|
+
alias_method :inspect, :to_s
|
|
177
|
+
|
|
178
|
+
# Create an abbreviated display for lists.
|
|
179
|
+
#
|
|
180
|
+
# @param max_length [Integer] Maximum line length
|
|
181
|
+
# @return [String] Abbreviated representation
|
|
182
|
+
def abbreviated(max_length: 80)
|
|
183
|
+
orig_display = "'#{@original}'"
|
|
184
|
+
sugg_display = "'#{recommended_suggestion.word}'"
|
|
185
|
+
|
|
186
|
+
"#{@location}: #{orig_display} → #{sugg_display} [#{(@confidence * 100).to_i}%]"
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
protected
|
|
190
|
+
|
|
191
|
+
# The position used for sorting. Prefers source_range.start
|
|
192
|
+
# (the new contract); falls back to legacy +location+ for
|
|
193
|
+
# errors built without a document.
|
|
194
|
+
#
|
|
195
|
+
# Visible to other SemanticError instances (protected, not private)
|
|
196
|
+
# because <=> reads `other.sort_position` when comparing.
|
|
197
|
+
def sort_position
|
|
198
|
+
return @source_range.start if @source_range
|
|
199
|
+
|
|
200
|
+
@location
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Value object for correction suggestions.
|
|
6
|
+
#
|
|
7
|
+
# Represents a suggested correction for a detected error,
|
|
8
|
+
# with confidence score and metadata.
|
|
9
|
+
#
|
|
10
|
+
# @example Creating a suggestion
|
|
11
|
+
# suggestion = Suggestion.new("dessert", confidence: 0.92, source: :semantic)
|
|
12
|
+
# suggestion.to_s # => "dessert [92%]"
|
|
13
|
+
class Suggestion
|
|
14
|
+
attr_reader :word, :confidence, :source, :metadata
|
|
15
|
+
|
|
16
|
+
# Create a new suggestion.
|
|
17
|
+
#
|
|
18
|
+
# @param word [String] The suggested word
|
|
19
|
+
# @param confidence [Float] Confidence score (0.0 to 1.0)
|
|
20
|
+
# @param source [Symbol, nil] Source of the suggestion (e.g., :semantic, :edit_distance)
|
|
21
|
+
# @param metadata [Hash] Additional metadata (optional)
|
|
22
|
+
# @option metadata [WordEmbedding, nil] :embedding The word embedding
|
|
23
|
+
# @option metadata [Float] :edit_distance Edit distance score
|
|
24
|
+
# @option metadata [Float] :frequency_bonus Frequency score bonus
|
|
25
|
+
# @option metadata [String] :explanation Explanation for the suggestion
|
|
26
|
+
def initialize(word, confidence:, source: nil, metadata: {})
|
|
27
|
+
raise ArgumentError, "Confidence must be 0-1" unless confidence.between?(0.0, 1.0)
|
|
28
|
+
|
|
29
|
+
@word = word
|
|
30
|
+
@confidence = confidence
|
|
31
|
+
@source = source || :unknown
|
|
32
|
+
@metadata = metadata.freeze
|
|
33
|
+
freeze
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Comparison for sorting (higher confidence = better).
|
|
37
|
+
#
|
|
38
|
+
# @param other [Suggestion] Another suggestion
|
|
39
|
+
# @return [Integer] Comparison result (-1, 0, 1)
|
|
40
|
+
def <=>(other)
|
|
41
|
+
return 0 unless other.is_a?(Suggestion)
|
|
42
|
+
|
|
43
|
+
# Higher confidence = better rank (sort descending)
|
|
44
|
+
other.confidence <=> @confidence
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Check if this equals another suggestion.
|
|
48
|
+
#
|
|
49
|
+
# @param other [Object] Another object
|
|
50
|
+
# @return [Boolean] True if words match
|
|
51
|
+
def ==(other)
|
|
52
|
+
return false unless other.is_a?(Suggestion)
|
|
53
|
+
|
|
54
|
+
@word == other.word
|
|
55
|
+
end
|
|
56
|
+
alias_method :eql?, :==
|
|
57
|
+
|
|
58
|
+
# Hash code for hash table usage.
|
|
59
|
+
#
|
|
60
|
+
# @return [Integer] Hash code
|
|
61
|
+
def hash
|
|
62
|
+
@word.hash
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# String representation with percentage.
|
|
66
|
+
#
|
|
67
|
+
# @return [String] Human-readable representation
|
|
68
|
+
def to_s
|
|
69
|
+
if @source && @source != :unknown
|
|
70
|
+
"#{@word} [#{(@confidence * 100).to_i}%] (#{@source})"
|
|
71
|
+
else
|
|
72
|
+
"#{@word} [#{(@confidence * 100).to_i}%]"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
alias_method :inspect, :to_s
|
|
76
|
+
|
|
77
|
+
# Get the embedding if available.
|
|
78
|
+
#
|
|
79
|
+
# @return [WordEmbedding, nil] The embedding or nil
|
|
80
|
+
def embedding
|
|
81
|
+
@metadata[:embedding]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Get the edit distance if available.
|
|
85
|
+
#
|
|
86
|
+
# @return [Float, nil] Edit distance or nil
|
|
87
|
+
def edit_distance
|
|
88
|
+
@metadata[:edit_distance]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Check if this is a high-confidence suggestion.
|
|
92
|
+
#
|
|
93
|
+
# @return [Boolean] True if confidence > 0.8
|
|
94
|
+
def high_confidence?
|
|
95
|
+
@confidence > 0.8
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Get explanation text if available.
|
|
99
|
+
#
|
|
100
|
+
# @return [String, nil] Explanation or nil
|
|
101
|
+
def explanation
|
|
102
|
+
@metadata[:explanation]
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|