kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Abstract base class for word embedding models.
|
|
6
|
+
#
|
|
7
|
+
# Provides a unified interface for loading and querying word embeddings
|
|
8
|
+
# from different sources (FastText, Word2Vec, GloVe, ONNX, etc.).
|
|
9
|
+
#
|
|
10
|
+
# @example Using an embedding model
|
|
11
|
+
# model = FastTextModel.new('cc.en.300.vec')
|
|
12
|
+
# embedding = model.embedding_for('hello')
|
|
13
|
+
# similarity = model.similarity('hello', 'world')
|
|
14
|
+
# neighbors = model.nearest_neighbors('hello', k: 10)
|
|
15
|
+
#
|
|
16
|
+
# @abstract Subclasses must implement {#embedding_for} and {#vocabulary}
|
|
17
|
+
class EmbeddingModel
|
|
18
|
+
attr_reader :language_code, :dimension, :vocabulary_size
|
|
19
|
+
|
|
20
|
+
# Create a new embedding model.
|
|
21
|
+
#
|
|
22
|
+
# @param language_code [String] ISO 639-1 language code
|
|
23
|
+
# @param dimension [Integer] Vector dimensionality (e.g., 300)
|
|
24
|
+
def initialize(language_code:, dimension:)
|
|
25
|
+
raise ArgumentError, "Language code cannot be nil" if language_code.nil?
|
|
26
|
+
raise ArgumentError, "Dimension must be positive" unless dimension&.positive?
|
|
27
|
+
|
|
28
|
+
@language_code = language_code
|
|
29
|
+
@dimension = dimension
|
|
30
|
+
@vocabulary_size = 0
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Get embedding vector for a word.
|
|
34
|
+
#
|
|
35
|
+
# @param word [String] The word to lookup
|
|
36
|
+
# @return [WordEmbedding, nil] Embedding vector or nil if not found
|
|
37
|
+
# @abstract Subclass must implement
|
|
38
|
+
def embedding_for(word)
|
|
39
|
+
raise NotImplementedError, "#{self.class} must implement #embedding_for"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Check if a word is in the vocabulary.
|
|
43
|
+
#
|
|
44
|
+
# @param word [String] The word to check
|
|
45
|
+
# @return [Boolean] True if word exists in vocabulary
|
|
46
|
+
def has_word?(word)
|
|
47
|
+
vocabulary.include?(word)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Calculate cosine similarity between two words.
|
|
51
|
+
#
|
|
52
|
+
# @param word1 [String] First word
|
|
53
|
+
# @param word2 [String] Second word
|
|
54
|
+
# @return [Float, nil] Similarity score (0.0 to 1.0) or nil if words not found
|
|
55
|
+
def similarity(word1, word2)
|
|
56
|
+
emb1 = embedding_for(word1)
|
|
57
|
+
emb2 = embedding_for(word2)
|
|
58
|
+
|
|
59
|
+
return nil unless emb1 && emb2
|
|
60
|
+
|
|
61
|
+
emb1.similarity(emb2)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Calculate Euclidean distance between two words.
|
|
65
|
+
#
|
|
66
|
+
# @param word1 [String] First word
|
|
67
|
+
# @param word2 [String] Second word
|
|
68
|
+
# @return [Float, nil] Distance or nil if words not found
|
|
69
|
+
def distance(word1, word2)
|
|
70
|
+
emb1 = embedding_for(word1)
|
|
71
|
+
emb2 = embedding_for(word2)
|
|
72
|
+
|
|
73
|
+
return nil unless emb1 && emb2
|
|
74
|
+
|
|
75
|
+
emb1.distance(emb2)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Find k nearest neighbors for a word.
|
|
79
|
+
#
|
|
80
|
+
# @param word [String] The query word
|
|
81
|
+
# @param k [Integer] Number of neighbors to return
|
|
82
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
|
|
83
|
+
def nearest_neighbors(word, k: 10)
|
|
84
|
+
embedding = embedding_for(word)
|
|
85
|
+
return [] unless embedding
|
|
86
|
+
|
|
87
|
+
# Calculate similarity with all words in vocabulary
|
|
88
|
+
neighbors = vocabulary.map do |vocab_word|
|
|
89
|
+
next if vocab_word == word
|
|
90
|
+
|
|
91
|
+
vocab_embedding = embedding_for(vocab_word)
|
|
92
|
+
next unless vocab_embedding
|
|
93
|
+
|
|
94
|
+
sim = embedding.similarity(vocab_embedding)
|
|
95
|
+
NearestNeighbor.new(
|
|
96
|
+
word: vocab_word,
|
|
97
|
+
similarity: sim,
|
|
98
|
+
distance: embedding.distance(vocab_embedding),
|
|
99
|
+
embedding: vocab_embedding
|
|
100
|
+
)
|
|
101
|
+
end.compact
|
|
102
|
+
|
|
103
|
+
# NearestNeighbor#<=> already sorts descending by similarity.
|
|
104
|
+
neighbors.sort.first(k)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Find k nearest neighbors for an embedding vector.
|
|
108
|
+
#
|
|
109
|
+
# @param embedding [WordEmbedding] The query embedding
|
|
110
|
+
# @param k [Integer] Number of neighbors to return
|
|
111
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
|
|
112
|
+
def nearest_neighbors_for_embedding(embedding, k: 10)
|
|
113
|
+
return [] unless embedding
|
|
114
|
+
|
|
115
|
+
# Calculate similarity with all words in vocabulary
|
|
116
|
+
neighbors = vocabulary.map do |vocab_word|
|
|
117
|
+
vocab_embedding = embedding_for(vocab_word)
|
|
118
|
+
next unless vocab_embedding
|
|
119
|
+
|
|
120
|
+
sim = embedding.similarity(vocab_embedding)
|
|
121
|
+
NearestNeighbor.new(
|
|
122
|
+
word: vocab_word,
|
|
123
|
+
similarity: sim,
|
|
124
|
+
distance: embedding.distance(vocab_embedding),
|
|
125
|
+
embedding: vocab_embedding
|
|
126
|
+
)
|
|
127
|
+
end.compact
|
|
128
|
+
|
|
129
|
+
# NearestNeighbor#<=> already sorts descending by similarity.
|
|
130
|
+
neighbors.sort.first(k)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Get model metadata.
|
|
134
|
+
#
|
|
135
|
+
# @return [Hash] Model metadata
|
|
136
|
+
def metadata
|
|
137
|
+
{
|
|
138
|
+
language_code: @language_code,
|
|
139
|
+
dimension: @dimension,
|
|
140
|
+
vocabulary_size: @vocabulary_size,
|
|
141
|
+
model_type: self.class.name
|
|
142
|
+
}
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Get the vocabulary (all words in the model).
|
|
146
|
+
#
|
|
147
|
+
# @return [Array<String>] Vocabulary words
|
|
148
|
+
# @abstract Subclass must implement
|
|
149
|
+
def vocabulary
|
|
150
|
+
raise NotImplementedError, "#{self.class} must implement #vocabulary"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Check if model is loaded.
|
|
154
|
+
#
|
|
155
|
+
# @return [Boolean] True if model is loaded and ready
|
|
156
|
+
def loaded?
|
|
157
|
+
@vocabulary_size&.positive? || vocabulary&.any?
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Get model statistics.
|
|
161
|
+
#
|
|
162
|
+
# @return [Hash] Statistics about the model
|
|
163
|
+
def statistics
|
|
164
|
+
{
|
|
165
|
+
language: @language_code,
|
|
166
|
+
dimension: @dimension,
|
|
167
|
+
vocabulary_size: @vocabulary_size,
|
|
168
|
+
loaded: loaded?
|
|
169
|
+
}
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# String representation.
|
|
173
|
+
#
|
|
174
|
+
# @return [String] Human-readable representation
|
|
175
|
+
def to_s
|
|
176
|
+
"#{self.class.name}(language: #{@language_code}, dim: #{@dimension}, vocab: #{@vocabulary_size})"
|
|
177
|
+
end
|
|
178
|
+
alias_method :inspect, :to_s
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# FastText embedding model implementation.
|
|
6
|
+
#
|
|
7
|
+
# Loads FastText pre-trained word vectors from .vec files.
|
|
8
|
+
# Supports Common Crawl and Wikipedia trained vectors.
|
|
9
|
+
#
|
|
10
|
+
# @example Loading from file
|
|
11
|
+
# model = FastTextModel.from_file('cc.en.300.vec')
|
|
12
|
+
# model.embedding_for('hello')
|
|
13
|
+
#
|
|
14
|
+
# @example Loading from GitHub
|
|
15
|
+
# model = FastTextModel.from_github('en')
|
|
16
|
+
# model.nearest_neighbors('hello', k: 10)
|
|
17
|
+
#
|
|
18
|
+
# @see https://fasttext.cc/docs/en/crawl-vectors.html FastText crawl vectors
|
|
19
|
+
# @see https://fasttext.cc/docs/en/english-vectors.html FastText English vectors
|
|
20
|
+
class FastTextModel < EmbeddingModel
|
|
21
|
+
# Standard FastText dimension for crawl vectors
|
|
22
|
+
DEFAULT_DIMENSION = 300
|
|
23
|
+
|
|
24
|
+
# Number of vectors to load when reading from file
|
|
25
|
+
# FastText .vec files contain up to 2M words; we load a subset by default
|
|
26
|
+
DEFAULT_MAX_VECTORS = 1_000_000
|
|
27
|
+
|
|
28
|
+
attr_reader :embeddings, :max_vectors
|
|
29
|
+
|
|
30
|
+
# Create a new FastText model.
|
|
31
|
+
#
|
|
32
|
+
# @param language_code [String] ISO 639-1 language code
|
|
33
|
+
# @param dimension [Integer] Vector dimension (default: 300)
|
|
34
|
+
# @param embeddings [Hash<String, WordEmbedding>] Pre-loaded embeddings
|
|
35
|
+
# @param max_vectors [Integer] Maximum vectors to load from file
|
|
36
|
+
def initialize(language_code:, dimension: DEFAULT_DIMENSION, embeddings: {}, max_vectors: DEFAULT_MAX_VECTORS)
|
|
37
|
+
super(language_code: language_code, dimension: dimension)
|
|
38
|
+
@embeddings = embeddings.freeze
|
|
39
|
+
@max_vectors = max_vectors
|
|
40
|
+
@vocabulary_size = @embeddings.size
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Load FastText model from a .vec file.
|
|
44
|
+
#
|
|
45
|
+
# @param file_path [String] Path to FastText .vec file
|
|
46
|
+
# @param max_vectors [Integer] Maximum vectors to load (default: 1M)
|
|
47
|
+
# @param language_code [String] Language code (auto-detected from filename)
|
|
48
|
+
# @return [FastTextModel] Loaded model
|
|
49
|
+
# @raise [ArgumentError] if file doesn't exist
|
|
50
|
+
def self.from_file(file_path, max_vectors: DEFAULT_MAX_VECTORS, language_code: nil)
|
|
51
|
+
raise ArgumentError, "File not found: #{file_path}" unless File.exist?(file_path)
|
|
52
|
+
|
|
53
|
+
# Detect language from filename if not provided
|
|
54
|
+
language_code ||= detect_language_from_path(file_path)
|
|
55
|
+
|
|
56
|
+
# Parse the .vec file
|
|
57
|
+
embeddings = {}
|
|
58
|
+
dimension = nil
|
|
59
|
+
count = 0
|
|
60
|
+
|
|
61
|
+
File.open(file_path, 'r', encoding: 'UTF-8') do |file|
|
|
62
|
+
# First line: vocab_size dimension
|
|
63
|
+
first_line = file.getline
|
|
64
|
+
metadata = first_line.split
|
|
65
|
+
_vocab_size = metadata[0].to_i
|
|
66
|
+
dimension = metadata[1].to_i
|
|
67
|
+
|
|
68
|
+
# Read vectors
|
|
69
|
+
file.each_line do |line|
|
|
70
|
+
break if count >= max_vectors
|
|
71
|
+
|
|
72
|
+
parts = line.split
|
|
73
|
+
word = parts[0]
|
|
74
|
+
vector = parts[1..-1].map(&:to_f)
|
|
75
|
+
|
|
76
|
+
next unless vector.size == dimension
|
|
77
|
+
|
|
78
|
+
embeddings[word] = WordEmbedding.new(word, vector, language_code, dimension: dimension)
|
|
79
|
+
count += 1
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
new(language_code: language_code, dimension: dimension, embeddings: embeddings, max_vectors: max_vectors)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Load FastText model from GitHub (via ModelCache).
|
|
87
|
+
#
|
|
88
|
+
# Downloads the .vec file from kotoshu/dictionaries repository.
|
|
89
|
+
#
|
|
90
|
+
# @param language_code [String] ISO 639-1 language code (de, en, es, fr, pt, ru)
|
|
91
|
+
# @param max_vectors [Integer] Maximum vectors to load (default: 500K for GitHub)
|
|
92
|
+
# @param cache [ModelCache, nil] Optional cache instance
|
|
93
|
+
# @return [FastTextModel] Loaded model
|
|
94
|
+
# @raise [ArgumentError] if language not supported
|
|
95
|
+
def self.from_github(language_code, max_vectors: 500_000, cache: nil)
|
|
96
|
+
cache ||= Cache::ModelCache.new
|
|
97
|
+
|
|
98
|
+
# Get the .vec file path from cache
|
|
99
|
+
vec_file = cache.get_fasttext_model(language_code)
|
|
100
|
+
|
|
101
|
+
from_file(vec_file, max_vectors: max_vectors, language_code: language_code)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Get embedding vector for a word.
|
|
105
|
+
#
|
|
106
|
+
# @param word [String] The word to lookup
|
|
107
|
+
# @return [WordEmbedding, nil] Embedding vector or nil if not found
|
|
108
|
+
def embedding_for(word)
|
|
109
|
+
return nil if word.nil? || word.empty?
|
|
110
|
+
|
|
111
|
+
# Direct lookup
|
|
112
|
+
@embeddings[word]
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Get the vocabulary (all words in the model).
|
|
116
|
+
#
|
|
117
|
+
# @return [Array<String>] Vocabulary words
|
|
118
|
+
def vocabulary
|
|
119
|
+
@embeddings.keys
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Check if model is loaded.
|
|
123
|
+
#
|
|
124
|
+
# @return [Boolean] True if embeddings are loaded
|
|
125
|
+
def loaded?
|
|
126
|
+
@embeddings&.any?
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Find k nearest neighbors for a word (optimized version).
|
|
130
|
+
#
|
|
131
|
+
# Overrides the base implementation for better performance using
|
|
132
|
+
# pre-loaded embeddings instead of repeated lookups.
|
|
133
|
+
#
|
|
134
|
+
# @param word [String] The query word
|
|
135
|
+
# @param k [Integer] Number of neighbors to return
|
|
136
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
|
|
137
|
+
def nearest_neighbors(word, k: 10)
|
|
138
|
+
embedding = embedding_for(word)
|
|
139
|
+
return [] unless embedding
|
|
140
|
+
|
|
141
|
+
# Calculate similarity with all words in vocabulary
|
|
142
|
+
neighbors = @embeddings.map do |vocab_word, vocab_embedding|
|
|
143
|
+
next if vocab_word == word
|
|
144
|
+
|
|
145
|
+
sim = embedding.similarity(vocab_embedding)
|
|
146
|
+
NearestNeighbor.new(
|
|
147
|
+
word: vocab_word,
|
|
148
|
+
similarity: sim,
|
|
149
|
+
embedding: vocab_embedding
|
|
150
|
+
)
|
|
151
|
+
end.compact
|
|
152
|
+
|
|
153
|
+
# Sort by similarity (descending) and take top k
|
|
154
|
+
neighbors.sort.reverse.first(k)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Find k nearest neighbors for an embedding vector (optimized version).
|
|
158
|
+
#
|
|
159
|
+
# @param embedding [WordEmbedding] The query embedding
|
|
160
|
+
# @param k [Integer] Number of neighbors to return
|
|
161
|
+
# @return [Array<NearestNeighbor>] Nearest neighbors sorted by similarity
|
|
162
|
+
def nearest_neighbors_for_embedding(embedding, k: 10)
|
|
163
|
+
return [] unless embedding
|
|
164
|
+
|
|
165
|
+
# Calculate similarity with all words in vocabulary
|
|
166
|
+
neighbors = @embeddings.map do |vocab_word, vocab_embedding|
|
|
167
|
+
sim = embedding.similarity(vocab_embedding)
|
|
168
|
+
NearestNeighbor.new(
|
|
169
|
+
word: vocab_word,
|
|
170
|
+
similarity: sim,
|
|
171
|
+
embedding: vocab_embedding
|
|
172
|
+
)
|
|
173
|
+
end.compact
|
|
174
|
+
|
|
175
|
+
# Sort by similarity (descending) and take top k
|
|
176
|
+
neighbors.sort.reverse.first(k)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Get batch embeddings for multiple words.
|
|
180
|
+
#
|
|
181
|
+
# @param words [Array<String>] Words to lookup
|
|
182
|
+
# @return [Hash<String, WordEmbedding>] Mapping of word to embedding
|
|
183
|
+
def batch_embeddings(words)
|
|
184
|
+
words.each_with_object({}) do |word, hash|
|
|
185
|
+
emb = embedding_for(word)
|
|
186
|
+
hash[word] = emb if emb
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Get batch similarities for word pairs.
|
|
191
|
+
#
|
|
192
|
+
# @param pairs [Array<Array<String, String>>] Word pairs
|
|
193
|
+
# @return [Array<Float>] Similarity scores
|
|
194
|
+
def batch_similarities(pairs)
|
|
195
|
+
pairs.map { |word1, word2| similarity(word1, word2) }
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Detect language code from file path.
|
|
199
|
+
#
|
|
200
|
+
# @param path [String] File path
|
|
201
|
+
# @return [String] Detected language code
|
|
202
|
+
def self.detect_language_from_path(path)
|
|
203
|
+
# Extract from path like "cc.en.300.vec" or "wiki.de.vec"
|
|
204
|
+
if path =~ /\.([a-z]{2})\./i
|
|
205
|
+
Regexp.last_match(1).downcase
|
|
206
|
+
else
|
|
207
|
+
'en' # Default to English
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Value object for embedding search results (nearest neighbors).
|
|
6
|
+
#
|
|
7
|
+
# Represents a single suggestion from semantic similarity search,
|
|
8
|
+
# with similarity score and optional embedding reference.
|
|
9
|
+
#
|
|
10
|
+
# @example Creating a neighbor
|
|
11
|
+
# neighbor = NearestNeighbor.new(word: "hello", similarity: 0.85, embedding: emb)
|
|
12
|
+
# neighbor.to_s # => "hello [85%]"
|
|
13
|
+
class NearestNeighbor
|
|
14
|
+
attr_reader :word, :similarity, :distance, :embedding
|
|
15
|
+
|
|
16
|
+
# Create a new nearest neighbor result.
|
|
17
|
+
#
|
|
18
|
+
# @param word [String] The suggested word
|
|
19
|
+
# @param similarity [Float] Cosine similarity (0.0 to 1.0)
|
|
20
|
+
# @param distance [Float, nil] Optional precomputed distance metric
|
|
21
|
+
# (e.g., Euclidean). Defaults to the cosine-derived distance (1 - similarity).
|
|
22
|
+
# @param embedding [WordEmbedding, nil] Optional embedding reference
|
|
23
|
+
def initialize(word:, similarity:, distance: nil, embedding: nil)
|
|
24
|
+
similarity = Float(similarity)
|
|
25
|
+
raise ArgumentError, "Similarity must be a finite number" unless similarity.finite?
|
|
26
|
+
unless similarity.between?(-1.0 - 1e-6, 1.0 + 1e-6)
|
|
27
|
+
raise ArgumentError, "Similarity out of range: #{similarity}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
@word = word
|
|
31
|
+
# Cosine similarity is mathematically in [-1, 1]: float
|
|
32
|
+
# rounding can drift just past 1.0, and true negatives are
|
|
33
|
+
# noise suggestions (filtered by min_similarity downstream) —
|
|
34
|
+
# both clamp into the documented [0, 1] suggestion range.
|
|
35
|
+
@similarity = similarity.clamp(0.0, 1.0)
|
|
36
|
+
@distance = distance || (1.0 - @similarity)
|
|
37
|
+
@embedding = embedding
|
|
38
|
+
freeze
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Comparison for sorting (higher similarity = better).
|
|
42
|
+
#
|
|
43
|
+
# @param other [NearestNeighbor] Another neighbor
|
|
44
|
+
# @return [Integer] Comparison result (-1, 0, 1)
|
|
45
|
+
def <=>(other)
|
|
46
|
+
return 0 unless other.is_a?(NearestNeighbor)
|
|
47
|
+
|
|
48
|
+
# Higher similarity = better rank (sort descending)
|
|
49
|
+
other.similarity <=> @similarity
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Check if this equals another neighbor.
|
|
53
|
+
#
|
|
54
|
+
# @param other [Object] Another object
|
|
55
|
+
# @return [Boolean] True if words match
|
|
56
|
+
def ==(other)
|
|
57
|
+
return false unless other.is_a?(NearestNeighbor)
|
|
58
|
+
|
|
59
|
+
@word == other.word
|
|
60
|
+
end
|
|
61
|
+
alias_method :eql?, :==
|
|
62
|
+
|
|
63
|
+
# Hash code for hash table usage.
|
|
64
|
+
#
|
|
65
|
+
# @return [Integer] Hash code
|
|
66
|
+
def hash
|
|
67
|
+
@word.hash
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# String representation with percentage.
|
|
71
|
+
#
|
|
72
|
+
# @return [String] Human-readable representation
|
|
73
|
+
def to_s
|
|
74
|
+
"#{@word} [#{(@similarity * 100).to_i}%]"
|
|
75
|
+
end
|
|
76
|
+
alias_method :inspect, :to_s
|
|
77
|
+
|
|
78
|
+
# Check if this is a high-confidence suggestion.
|
|
79
|
+
#
|
|
80
|
+
# @return [Boolean] True if similarity > 0.8
|
|
81
|
+
def high_confidence?
|
|
82
|
+
@similarity > 0.8
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Get confidence level category.
|
|
86
|
+
#
|
|
87
|
+
# @return [Symbol] :high, :medium, or :low
|
|
88
|
+
def confidence_level
|
|
89
|
+
return :high if @similarity > 0.8
|
|
90
|
+
return :medium if @similarity > 0.5
|
|
91
|
+
|
|
92
|
+
:low
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|