kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# OnnxRuntimeModel - ONNX Runtime wrapper for FastText embeddings
|
|
4
|
+
#
|
|
5
|
+
# Provides embedding inference using ONNX Runtime. Supports single lookups,
|
|
6
|
+
# batch inference, and vocabulary-aware operations.
|
|
7
|
+
#
|
|
8
|
+
# @example Single embedding lookup
|
|
9
|
+
# model = OnnxRuntimeModel.from_file('fasttext.en.onnx', language_code: 'en')
|
|
10
|
+
# model.load!
|
|
11
|
+
# embedding = model.get_embedding(1234)
|
|
12
|
+
#
|
|
13
|
+
# @example Batch lookup
|
|
14
|
+
# embeddings = model.get_embeddings([1, 2, 3, 4, 5])
|
|
15
|
+
#
|
|
16
|
+
# @example With vocabulary
|
|
17
|
+
# embedding = model.get_embedding_for_word('hello', vocabulary)
|
|
18
|
+
#
|
|
19
|
+
module Kotoshu
|
|
20
|
+
module Embeddings
|
|
21
|
+
class OnnxRuntimeModel
|
|
22
|
+
include EmbeddingModelProtocol
|
|
23
|
+
|
|
24
|
+
# Default dimension for FastText models
|
|
25
|
+
DEFAULT_DIMENSION = 300
|
|
26
|
+
|
|
27
|
+
# Batch size for batch inference
|
|
28
|
+
BATCH_SIZE = 32
|
|
29
|
+
|
|
30
|
+
# @return [String] Language code (ISO 639-1)
|
|
31
|
+
attr_reader :language_code
|
|
32
|
+
|
|
33
|
+
# @return [Integer] Embedding dimension
|
|
34
|
+
attr_reader :dimension
|
|
35
|
+
|
|
36
|
+
# @return [String] Path to ONNX model file
|
|
37
|
+
attr_reader :onnx_path
|
|
38
|
+
|
|
39
|
+
# @return [Boolean] Whether the model is loaded
|
|
40
|
+
def loaded?
|
|
41
|
+
@loaded
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @return [Integer] Number of inference calls
|
|
45
|
+
attr_reader :inference_count
|
|
46
|
+
|
|
47
|
+
# Create a new ONNX Runtime model
|
|
48
|
+
#
|
|
49
|
+
# @param language_code [String] ISO 639-1 language code
|
|
50
|
+
# @param onnx_path [String] Path to .onnx file
|
|
51
|
+
# @param dimension [Integer] Embedding dimension (default: 300)
|
|
52
|
+
#
|
|
53
|
+
def initialize(language_code:, onnx_path:, dimension: DEFAULT_DIMENSION)
|
|
54
|
+
@language_code = language_code
|
|
55
|
+
@onnx_path = onnx_path
|
|
56
|
+
@dimension = dimension
|
|
57
|
+
@session = nil
|
|
58
|
+
@loaded = false
|
|
59
|
+
@input_name = nil
|
|
60
|
+
@output_name = nil
|
|
61
|
+
@inference_count = 0
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Load the ONNX model into memory
|
|
65
|
+
#
|
|
66
|
+
# @return [self]
|
|
67
|
+
#
|
|
68
|
+
# @raise [Kotoshu::Models::OnnxUnavailable] if onnxruntime gem is missing
|
|
69
|
+
# @raise [ArgumentError] if model file doesn't exist
|
|
70
|
+
#
|
|
71
|
+
def load!
|
|
72
|
+
return self if @loaded
|
|
73
|
+
|
|
74
|
+
raise Kotoshu::Models::OnnxModel::OnnxUnavailable unless Kotoshu::Models::OnnxModel::ONNX_LOADED
|
|
75
|
+
|
|
76
|
+
unless File.exist?(@onnx_path)
|
|
77
|
+
raise ArgumentError,
|
|
78
|
+
"ONNX file not found: #{@onnx_path}"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
begin
|
|
82
|
+
@session = OnnxRuntime::InferenceSession.new(@onnx_path)
|
|
83
|
+
rescue OnnxRuntime::Error => e
|
|
84
|
+
# Phase C of TODO.impl/38: a corrupt/truncated cached model must
|
|
85
|
+
# surface as an actionable error, not a raw "Protobuf parsing
|
|
86
|
+
# failed".
|
|
87
|
+
raise Kotoshu::Error,
|
|
88
|
+
"ONNX model failed to load (#{e.message}): #{@onnx_path}. " \
|
|
89
|
+
"The cached model may be corrupt or truncated — re-download " \
|
|
90
|
+
"it with `kotoshu cache download :#{@language_code} --model`."
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Detect input/output names
|
|
94
|
+
@input_name = detect_input_name
|
|
95
|
+
@output_name = detect_output_name
|
|
96
|
+
|
|
97
|
+
@loaded = true
|
|
98
|
+
self
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Unload the model from memory
|
|
102
|
+
#
|
|
103
|
+
# @return [self]
|
|
104
|
+
#
|
|
105
|
+
def unload!
|
|
106
|
+
@session = nil
|
|
107
|
+
@input_name = nil
|
|
108
|
+
@output_name = nil
|
|
109
|
+
@loaded = false
|
|
110
|
+
self
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Check if model is ready for inference
|
|
114
|
+
#
|
|
115
|
+
# @return [Boolean]
|
|
116
|
+
#
|
|
117
|
+
def ready?
|
|
118
|
+
@loaded && !@session.nil?
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Get embedding for a single word index
|
|
122
|
+
#
|
|
123
|
+
# @param index [Integer] Word index in vocabulary
|
|
124
|
+
# @return [Array<Float>] Embedding vector
|
|
125
|
+
#
|
|
126
|
+
# @raise [RuntimeError] if model is not loaded
|
|
127
|
+
# @raise [ArgumentError] if index is invalid
|
|
128
|
+
#
|
|
129
|
+
def get_embedding(index)
|
|
130
|
+
ensure_loaded
|
|
131
|
+
|
|
132
|
+
unless valid_index?(index)
|
|
133
|
+
raise ArgumentError,
|
|
134
|
+
"Invalid word index: #{index}"
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
output = @session.run(
|
|
138
|
+
[@output_name],
|
|
139
|
+
{ @input_name => [index] },
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
@inference_count += 1
|
|
143
|
+
|
|
144
|
+
extract_embedding(output.first)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Get embeddings for multiple indices (batched)
|
|
148
|
+
#
|
|
149
|
+
# More efficient than individual calls for batch operations.
|
|
150
|
+
#
|
|
151
|
+
# @param indices [Array<Integer>] Word indices
|
|
152
|
+
# @return [Array<Array<Float>>] Array of embedding vectors
|
|
153
|
+
#
|
|
154
|
+
def get_embeddings(indices)
|
|
155
|
+
ensure_loaded
|
|
156
|
+
return [] if indices.nil? || indices.empty?
|
|
157
|
+
|
|
158
|
+
valid_indices = indices.select { |i| valid_index?(i) }
|
|
159
|
+
return [] if valid_indices.empty?
|
|
160
|
+
|
|
161
|
+
# Process in batches for memory efficiency
|
|
162
|
+
valid_indices.each_slice(BATCH_SIZE).flat_map do |batch|
|
|
163
|
+
run_batch_inference(batch)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Preload all embeddings into memory
|
|
168
|
+
#
|
|
169
|
+
# For small vocabularies, this provides O(1) lookup after loading.
|
|
170
|
+
#
|
|
171
|
+
# @param vocabulary [Vocabulary] Vocabulary with complete word list
|
|
172
|
+
# @return [Hash<Integer, Array<Float>>] Index to embedding mapping
|
|
173
|
+
#
|
|
174
|
+
def preload_embeddings!(vocabulary)
|
|
175
|
+
ensure_loaded
|
|
176
|
+
|
|
177
|
+
all_indices = (0...vocabulary.size).to_a
|
|
178
|
+
embeddings = get_embeddings(all_indices)
|
|
179
|
+
|
|
180
|
+
# Build index mapping
|
|
181
|
+
all_indices.zip(embeddings).to_h
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Get embedding for a word using vocabulary
|
|
185
|
+
#
|
|
186
|
+
# @param word [String] The word to lookup
|
|
187
|
+
# @param vocabulary [Vocabulary] Vocabulary for word-to-index mapping
|
|
188
|
+
# @return [Array<Float>, nil] Embedding vector or nil if word not found
|
|
189
|
+
#
|
|
190
|
+
def get_embedding_for_word(word, vocabulary)
|
|
191
|
+
index = vocabulary.lookup(word)
|
|
192
|
+
return nil unless index
|
|
193
|
+
|
|
194
|
+
get_embedding(index)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Get embeddings for multiple words using vocabulary
|
|
198
|
+
#
|
|
199
|
+
# @param words [Array<String>] Words to lookup
|
|
200
|
+
# @param vocabulary [Vocabulary] Vocabulary for word-to-index mapping
|
|
201
|
+
# @return [Hash<String, Array<Float>>] Word to embedding mapping
|
|
202
|
+
#
|
|
203
|
+
def get_embeddings_for_words(words, vocabulary)
|
|
204
|
+
result = {}
|
|
205
|
+
words.each do |word|
|
|
206
|
+
embedding = get_embedding_for_word(word, vocabulary)
|
|
207
|
+
result[word] = embedding if embedding
|
|
208
|
+
end
|
|
209
|
+
result
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Check if batching is supported
|
|
213
|
+
#
|
|
214
|
+
# @return [Boolean]
|
|
215
|
+
#
|
|
216
|
+
def supports_batching?
|
|
217
|
+
true
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Get batch size for batch inference
|
|
221
|
+
#
|
|
222
|
+
# @return [Integer]
|
|
223
|
+
#
|
|
224
|
+
def batch_size
|
|
225
|
+
BATCH_SIZE
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Get model type identifier
|
|
229
|
+
#
|
|
230
|
+
# @return [String]
|
|
231
|
+
#
|
|
232
|
+
def model_type
|
|
233
|
+
'onnx'
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Get model information
|
|
237
|
+
#
|
|
238
|
+
# @return [Hash]
|
|
239
|
+
#
|
|
240
|
+
def model_info
|
|
241
|
+
{
|
|
242
|
+
type: 'onnx',
|
|
243
|
+
language: @language_code,
|
|
244
|
+
dimension: @dimension,
|
|
245
|
+
path: @onnx_path,
|
|
246
|
+
loaded: @loaded,
|
|
247
|
+
inference_count: @inference_count,
|
|
248
|
+
}
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Create model from file
|
|
252
|
+
#
|
|
253
|
+
# @param onnx_path [String] Path to .onnx file
|
|
254
|
+
# @param language_code [String] Language code (auto-detected if nil)
|
|
255
|
+
# @param dimension [Integer] Embedding dimension
|
|
256
|
+
# @return [OnnxRuntimeModel]
|
|
257
|
+
#
|
|
258
|
+
def self.from_file(onnx_path, language_code: nil, dimension: nil)
|
|
259
|
+
unless File.exist?(onnx_path)
|
|
260
|
+
raise ArgumentError,
|
|
261
|
+
"ONNX file not found: #{onnx_path}"
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
language_code ||= detect_language_from_path(onnx_path)
|
|
265
|
+
dimension ||= DEFAULT_DIMENSION
|
|
266
|
+
|
|
267
|
+
new(
|
|
268
|
+
language_code: language_code,
|
|
269
|
+
onnx_path: onnx_path,
|
|
270
|
+
dimension: dimension,
|
|
271
|
+
)
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# Create model from cache
|
|
275
|
+
#
|
|
276
|
+
# @param language_code [String] ISO 639-1 language code
|
|
277
|
+
# @param cache [Cache::ModelCache] Cache instance
|
|
278
|
+
# @return [OnnxRuntimeModel, nil]
|
|
279
|
+
#
|
|
280
|
+
def self.from_cache(language_code, cache: nil)
|
|
281
|
+
cache ||= Kotoshu::Cache::ModelCache.new
|
|
282
|
+
|
|
283
|
+
onnx_path = cache.get_onnx_model(language_code)
|
|
284
|
+
return nil unless onnx_path
|
|
285
|
+
|
|
286
|
+
from_file(onnx_path, language_code: language_code)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# String representation
|
|
290
|
+
#
|
|
291
|
+
# @return [String]
|
|
292
|
+
#
|
|
293
|
+
def to_s
|
|
294
|
+
"OnnxRuntimeModel(language: #{@language_code}, dimension: #{@dimension}, loaded: #{@loaded})"
|
|
295
|
+
end
|
|
296
|
+
alias inspect to_s
|
|
297
|
+
|
|
298
|
+
private
|
|
299
|
+
|
|
300
|
+
# Ensure model is loaded
|
|
301
|
+
#
|
|
302
|
+
def ensure_loaded
|
|
303
|
+
load! unless @loaded
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# Check if index is valid
|
|
307
|
+
#
|
|
308
|
+
def valid_index?(index)
|
|
309
|
+
index.is_a?(Integer) && index >= 0
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Run batch inference for a batch of indices
|
|
313
|
+
#
|
|
314
|
+
# @param indices [Array<Integer>] Word indices
|
|
315
|
+
# @return [Array<Array<Float>>] Embedding vectors
|
|
316
|
+
#
|
|
317
|
+
def run_batch_inference(indices)
|
|
318
|
+
# Create input tensor
|
|
319
|
+
input_data = indices.flatten
|
|
320
|
+
|
|
321
|
+
output = @session.run(
|
|
322
|
+
[@output_name],
|
|
323
|
+
{ @input_name => input_data },
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
@inference_count += 1
|
|
327
|
+
|
|
328
|
+
# Extract embeddings
|
|
329
|
+
result = output.first
|
|
330
|
+
if result.is_a?(Array)
|
|
331
|
+
result
|
|
332
|
+
else
|
|
333
|
+
# Handle OrtValue or other wrappers
|
|
334
|
+
Array.new(indices.length) { |i| extract_single_embedding(result, i) }
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# Extract embedding from output
|
|
339
|
+
#
|
|
340
|
+
# @param output [Object] ONNX output
|
|
341
|
+
# @return [Array<Float>]
|
|
342
|
+
#
|
|
343
|
+
def extract_embedding(output)
|
|
344
|
+
case output
|
|
345
|
+
when Array
|
|
346
|
+
output
|
|
347
|
+
when NumpyArray, Numo::SFloat
|
|
348
|
+
output.to_a
|
|
349
|
+
when OnnxRuntime::OrtValue
|
|
350
|
+
output.to_a
|
|
351
|
+
else
|
|
352
|
+
# Idiomatic Array coercion: invokes to_ary / to_a when defined,
|
|
353
|
+
# otherwise wraps. Equivalent to the prior respond_to?(:to_a)
|
|
354
|
+
# branch without the duck-typing check.
|
|
355
|
+
Array(output)
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
# Extract single embedding from batch output
|
|
360
|
+
#
|
|
361
|
+
# @param output [Object] ONNX batch output
|
|
362
|
+
# @param index [Integer] Index in batch
|
|
363
|
+
# @return [Array<Float>]
|
|
364
|
+
#
|
|
365
|
+
def extract_single_embedding(output, index)
|
|
366
|
+
case output
|
|
367
|
+
when Array
|
|
368
|
+
output[index]
|
|
369
|
+
when NumpyArray, Numo::SFloat
|
|
370
|
+
output[index, true].to_a
|
|
371
|
+
else
|
|
372
|
+
# Default: assume array-like
|
|
373
|
+
output[index].to_a
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# Detect input name from model
|
|
378
|
+
#
|
|
379
|
+
# @return [String]
|
|
380
|
+
#
|
|
381
|
+
def detect_input_name
|
|
382
|
+
inputs = @session.inputs
|
|
383
|
+
inputs&.first&.dig(:name) || 'word_index'
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# Detect output name from model
|
|
387
|
+
#
|
|
388
|
+
# @return [String]
|
|
389
|
+
#
|
|
390
|
+
def detect_output_name
|
|
391
|
+
outputs = @session.outputs
|
|
392
|
+
outputs&.first&.dig(:name) || 'embedding'
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# Detect language from file path
|
|
396
|
+
#
|
|
397
|
+
# @param path [String]
|
|
398
|
+
# @return [String]
|
|
399
|
+
#
|
|
400
|
+
def self.detect_language_from_path(path)
|
|
401
|
+
basename = File.basename(path)
|
|
402
|
+
|
|
403
|
+
if basename =~ /\.([a-z]{2})\./i
|
|
404
|
+
Regexp.last_match(1).downcase
|
|
405
|
+
else
|
|
406
|
+
'en'
|
|
407
|
+
end
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Embeddings
|
|
7
|
+
# Protocol - Ruby interface/contract system
|
|
8
|
+
#
|
|
9
|
+
# Provides a simple way to define interfaces with required and optional methods.
|
|
10
|
+
module Protocol
|
|
11
|
+
# Set of method names that conforming classes must implement.
|
|
12
|
+
# Populated via +required+ (the writer); read via +required_methods+.
|
|
13
|
+
def required_methods
|
|
14
|
+
@required_methods ||= Set.new
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Set of method names that conforming classes may optionally implement.
|
|
18
|
+
# Populated via +optional+ (the writer); read via +optional_methods+.
|
|
19
|
+
def optional_methods
|
|
20
|
+
@optional_methods ||= Set.new
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Declare one or more required methods on this protocol.
|
|
24
|
+
def required(*names)
|
|
25
|
+
names.each { |n| required_methods << n }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Declare one or more optional methods on this protocol.
|
|
29
|
+
def optional(*names)
|
|
30
|
+
names.each { |n| optional_methods << n }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Returns the subset of +required_methods+ that +klass+ does not
|
|
34
|
+
# implement as an instance method. Empty result means full
|
|
35
|
+
# conformance. +klass+ should be a Class (not an instance); we
|
|
36
|
+
# check +method_defined?+ because protocol-declared methods are
|
|
37
|
+
# mixed in via +include+, becoming instance methods of +klass+.
|
|
38
|
+
def compliance_errors(klass)
|
|
39
|
+
required_methods.reject { |m| klass.method_defined?(m) }
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Raise ProtocolError unless +klass+ implements every required method.
|
|
43
|
+
def assert_implemented_by!(klass)
|
|
44
|
+
errors = compliance_errors(klass)
|
|
45
|
+
return if errors.empty?
|
|
46
|
+
|
|
47
|
+
raise ProtocolError.new(klass, self, errors.to_a)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Protocol error
|
|
52
|
+
class ProtocolError < StandardError
|
|
53
|
+
attr_reader :klass, :protocol, :missing_methods
|
|
54
|
+
|
|
55
|
+
def initialize(klass, protocol, missing_methods)
|
|
56
|
+
@klass = klass
|
|
57
|
+
@protocol = protocol
|
|
58
|
+
@missing_methods = missing_methods
|
|
59
|
+
super("#{klass} missing: #{missing_methods.join(', ')}")
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# EmbeddingModel Protocol
|
|
64
|
+
module EmbeddingModelProtocol
|
|
65
|
+
extend Protocol
|
|
66
|
+
|
|
67
|
+
required :dimension, :language_code, :get_embedding, :get_embeddings
|
|
68
|
+
required :load!, :unload!, :loaded?, :ready?
|
|
69
|
+
|
|
70
|
+
optional :get_embeddings_batch, :batch_size, :preload_embeddings!
|
|
71
|
+
optional :supports_batching?, :model_type, :model_info
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# SimilarityEngine Protocol
|
|
75
|
+
module SimilarityEngineProtocol
|
|
76
|
+
extend Protocol
|
|
77
|
+
|
|
78
|
+
required :cosine, :dot_product, :euclidean, :manhattan
|
|
79
|
+
required :pre_normalize, :normalize_and_compute
|
|
80
|
+
|
|
81
|
+
optional :cosine_batch, :compute_all_pairs
|
|
82
|
+
optional :is_normalized?, :normalization_required?
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Vocabulary Protocol
|
|
86
|
+
module VocabularyProtocol
|
|
87
|
+
extend Protocol
|
|
88
|
+
|
|
89
|
+
required :lookup, :get_word, :include?, :size, :words
|
|
90
|
+
required :valid_index?, :common_words, :to_h
|
|
91
|
+
|
|
92
|
+
optional :sample, :sub_vocabulary, :words_starting_with
|
|
93
|
+
optional :save_to_file, :language_code
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This file is intentionally a no-op stub.
|
|
4
|
+
#
|
|
5
|
+
# History: an earlier draft of the embeddings layer planned to split
|
|
6
|
+
# protocol definitions across `protocols/embedding_model.rb`,
|
|
7
|
+
# `protocols/similarity_engine.rb`, and `protocols/vocabulary.rb`.
|
|
8
|
+
# Those subfiles were never created; the protocols were inlined into
|
|
9
|
+
# `protocol.rb` (singular) instead, which is what the autoload table
|
|
10
|
+
# in `lib/kotoshu/embeddings.rb` points at.
|
|
11
|
+
#
|
|
12
|
+
# Loading the original version of this file raised LoadError because
|
|
13
|
+
# `require_relative 'protocols/embedding_model'` referenced a path
|
|
14
|
+
# that did not exist. Rather than delete the file (project policy
|
|
15
|
+
# preserves files contributors may have local context for), it is
|
|
16
|
+
# reduced to this stub so any future `require` of it succeeds
|
|
17
|
+
# harmlessly.
|
|
18
|
+
#
|
|
19
|
+
# For the actual protocol definitions see:
|
|
20
|
+
# Kotoshu::Embeddings::EmbeddingModelProtocol
|
|
21
|
+
# Kotoshu::Embeddings::SimilarityEngineProtocol
|
|
22
|
+
# Kotoshu::Embeddings::VocabularyProtocol
|
|
23
|
+
# all in lib/kotoshu/embeddings/protocol.rb.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Registry - Plugin system for embeddings components
|
|
4
|
+
#
|
|
5
|
+
# Provides a centralized registry for embedding models, similarity engines,
|
|
6
|
+
# and vocabularies. Allows registering and retrieving custom implementations.
|
|
7
|
+
#
|
|
8
|
+
# @example Registering a custom model
|
|
9
|
+
# Kotoshu::Embeddings::Registry.register_model(:my_model, MyCustomModel)
|
|
10
|
+
#
|
|
11
|
+
# @example Creating from registry
|
|
12
|
+
# model = Kotoshu::Embeddings::Registry.create_model(:my_model, vectors: my_vectors)
|
|
13
|
+
#
|
|
14
|
+
# @example Listing available implementations
|
|
15
|
+
# Kotoshu::Embeddings::Registry.models.keys # => [:onnx, :my_model]
|
|
16
|
+
#
|
|
17
|
+
module Kotoshu
|
|
18
|
+
module Embeddings
|
|
19
|
+
class Registry
|
|
20
|
+
class << self
|
|
21
|
+
# @return [Hash{Symbol => Class}] Registered models
|
|
22
|
+
attr_reader :models
|
|
23
|
+
|
|
24
|
+
# @return [Hash{Symbol => Class}] Registered engines
|
|
25
|
+
attr_reader :engines
|
|
26
|
+
|
|
27
|
+
# @return [Hash{Symbol => Class}] Registered vocabularies
|
|
28
|
+
attr_reader :vocabularies
|
|
29
|
+
|
|
30
|
+
# Initialize registry
|
|
31
|
+
#
|
|
32
|
+
def init
|
|
33
|
+
@models ||= {}
|
|
34
|
+
@engines ||= {}
|
|
35
|
+
@vocabularies ||= {}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Register an embedding model
|
|
39
|
+
#
|
|
40
|
+
# @param name [Symbol] Model identifier
|
|
41
|
+
# @param klass [Class] Model class (must implement EmbeddingModel protocol)
|
|
42
|
+
#
|
|
43
|
+
def register_model(name, klass)
|
|
44
|
+
init
|
|
45
|
+
@models[name] = klass
|
|
46
|
+
klass
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Register a similarity engine
|
|
50
|
+
#
|
|
51
|
+
# @param name [Symbol] Engine identifier
|
|
52
|
+
# @param klass [Class] Engine class (must implement SimilarityEngine protocol)
|
|
53
|
+
#
|
|
54
|
+
def register_engine(name, klass)
|
|
55
|
+
init
|
|
56
|
+
@engines[name] = klass
|
|
57
|
+
klass
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Register a vocabulary
|
|
61
|
+
#
|
|
62
|
+
# @param name [Symbol] Vocabulary identifier
|
|
63
|
+
# @param klass [Class] Vocabulary class (must implement Vocabulary protocol)
|
|
64
|
+
#
|
|
65
|
+
def register_vocabulary(name, klass)
|
|
66
|
+
init
|
|
67
|
+
@vocabularies[name] = klass
|
|
68
|
+
klass
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Get registered model class
|
|
72
|
+
#
|
|
73
|
+
# @param name [Symbol] Model identifier
|
|
74
|
+
# @return [Class, nil]
|
|
75
|
+
#
|
|
76
|
+
def model(name)
|
|
77
|
+
init
|
|
78
|
+
@models[name]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Get registered engine class
|
|
82
|
+
#
|
|
83
|
+
# @param name [Symbol] Engine identifier
|
|
84
|
+
# @return [Class, nil]
|
|
85
|
+
#
|
|
86
|
+
def engine(name)
|
|
87
|
+
init
|
|
88
|
+
@engines[name]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Get registered vocabulary class
|
|
92
|
+
#
|
|
93
|
+
# @param name [Symbol] Vocabulary identifier
|
|
94
|
+
# @return [Class, nil]
|
|
95
|
+
#
|
|
96
|
+
def vocabulary(name)
|
|
97
|
+
init
|
|
98
|
+
@vocabularies[name]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Create a model instance
|
|
102
|
+
#
|
|
103
|
+
# @param name [Symbol] Model identifier
|
|
104
|
+
# @param kwargs [Hash] Model constructor arguments
|
|
105
|
+
# @return [EmbeddingModel]
|
|
106
|
+
#
|
|
107
|
+
def create_model(name, **kwargs)
|
|
108
|
+
klass = model(name)
|
|
109
|
+
raise ArgumentError, "Unknown model: #{name}" unless klass
|
|
110
|
+
|
|
111
|
+
klass.new(**kwargs)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Create an engine instance
|
|
115
|
+
#
|
|
116
|
+
# @param name [Symbol] Engine identifier
|
|
117
|
+
# @param kwargs [Hash] Engine constructor arguments
|
|
118
|
+
# @return [SimilarityEngine]
|
|
119
|
+
#
|
|
120
|
+
def create_engine(name, **kwargs)
|
|
121
|
+
klass = engine(name)
|
|
122
|
+
raise ArgumentError, "Unknown engine: #{name}" unless klass
|
|
123
|
+
|
|
124
|
+
klass.new(**kwargs)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Create a vocabulary instance
|
|
128
|
+
#
|
|
129
|
+
# @param name [Symbol] Vocabulary identifier
|
|
130
|
+
# @param kwargs [Hash] Vocabulary constructor arguments
|
|
131
|
+
# @return [Vocabulary]
|
|
132
|
+
#
|
|
133
|
+
def create_vocabulary(name, **kwargs)
|
|
134
|
+
klass = vocabulary(name)
|
|
135
|
+
raise ArgumentError, "Unknown vocabulary: #{name}" unless klass
|
|
136
|
+
|
|
137
|
+
klass.new(**kwargs)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# List all registered models
|
|
141
|
+
#
|
|
142
|
+
# @return [Array<Symbol>]
|
|
143
|
+
#
|
|
144
|
+
def model_names
|
|
145
|
+
init
|
|
146
|
+
@models.keys
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# List all registered engines
|
|
150
|
+
#
|
|
151
|
+
# @return [Array<Symbol>]
|
|
152
|
+
#
|
|
153
|
+
def engine_names
|
|
154
|
+
init
|
|
155
|
+
@engines.keys
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# List all registered vocabularies
|
|
159
|
+
#
|
|
160
|
+
# @return [Array<Symbol>]
|
|
161
|
+
#
|
|
162
|
+
def vocabulary_names
|
|
163
|
+
init
|
|
164
|
+
@vocabularies.keys
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Clear all registrations
|
|
168
|
+
#
|
|
169
|
+
def reset!
|
|
170
|
+
@models = {}
|
|
171
|
+
@engines = {}
|
|
172
|
+
@vocabularies = {}
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Register built-in implementations. Triggering the autoloaded constants
|
|
178
|
+
# (OnnxRuntimeModel, SimilarityEngine, Vocabulary) here ensures the registry
|
|
179
|
+
# is populated the first time Kotoshu::Embeddings::Registry is referenced.
|
|
180
|
+
Registry.register_model(:onnx, OnnxRuntimeModel)
|
|
181
|
+
Registry.register_engine(:cosine, SimilarityEngine)
|
|
182
|
+
Registry.register_vocabulary(:json, Vocabulary)
|
|
183
|
+
end
|
|
184
|
+
end
|