kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Results
|
|
5
|
+
# Result pattern for explicit error handling.
|
|
6
|
+
#
|
|
7
|
+
# Provides a type-safe way to handle operations that can fail
|
|
8
|
+
# without using exceptions. Based on functional programming patterns.
|
|
9
|
+
#
|
|
10
|
+
# @example Using Success
|
|
11
|
+
# result = Result::Success.new("value")
|
|
12
|
+
# result.success? # => true
|
|
13
|
+
# result.value # => "value"
|
|
14
|
+
#
|
|
15
|
+
# @example Using Failure
|
|
16
|
+
# result = Result::Failure.new(error)
|
|
17
|
+
# result.failure? # => true
|
|
18
|
+
# result.error # => the error
|
|
19
|
+
#
|
|
20
|
+
# @example Chaining operations
|
|
21
|
+
# result = Result::Success.new(5)
|
|
22
|
+
# .and_then { |v| Success.new(v * 2) } # Only called if success
|
|
23
|
+
# .or_else { |e| Success.new(0) } # Only called if failure
|
|
24
|
+
module Result
|
|
25
|
+
# Base result class.
|
|
26
|
+
#
|
|
27
|
+
# @abstract
|
|
28
|
+
class Base
|
|
29
|
+
# Check if result is successful.
|
|
30
|
+
#
|
|
31
|
+
# @return [Boolean] True if successful
|
|
32
|
+
def success?
|
|
33
|
+
is_a?(Success)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Check if result is a failure.
|
|
37
|
+
#
|
|
38
|
+
# @return [Boolean] True if failed
|
|
39
|
+
def failure?
|
|
40
|
+
is_a?(Failure)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Map the value if successful.
|
|
44
|
+
#
|
|
45
|
+
# @yield [value] The wrapped value
|
|
46
|
+
# @return [Result::Success, Result::Failure] Mapped result
|
|
47
|
+
def map
|
|
48
|
+
return self if failure?
|
|
49
|
+
|
|
50
|
+
Success.new(yield value)
|
|
51
|
+
rescue StandardError => e
|
|
52
|
+
Failure.new(e)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Chain operations if successful.
|
|
56
|
+
#
|
|
57
|
+
# @yield [value] The wrapped value
|
|
58
|
+
# @return [Result::Success, Result::Failure] Chained result
|
|
59
|
+
def and_then
|
|
60
|
+
return self if failure?
|
|
61
|
+
|
|
62
|
+
result = yield value
|
|
63
|
+
|
|
64
|
+
# Ensure we get a Result back
|
|
65
|
+
result.is_a?(Base) ? result : Success.new(result)
|
|
66
|
+
rescue StandardError => e
|
|
67
|
+
Failure.new(e)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Recover from failure.
|
|
71
|
+
#
|
|
72
|
+
# @yield [error] The wrapped error
|
|
73
|
+
# @return [Result::Success, Result::Failure] Recovered result
|
|
74
|
+
def or_else
|
|
75
|
+
return self if success?
|
|
76
|
+
|
|
77
|
+
result = yield error
|
|
78
|
+
|
|
79
|
+
# Ensure we get a Result back
|
|
80
|
+
result.is_a?(Base) ? result : Success.new(result)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Unwrap the value or raise error.
|
|
84
|
+
#
|
|
85
|
+
# @return [Object] The wrapped value
|
|
86
|
+
# @raise [Error] The wrapped error if this is a Failure
|
|
87
|
+
def unwrap
|
|
88
|
+
return value if success?
|
|
89
|
+
|
|
90
|
+
raise error
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Get the wrapped value (nil for Failure).
|
|
94
|
+
#
|
|
95
|
+
# @return [Object, nil] The wrapped value or nil
|
|
96
|
+
def value
|
|
97
|
+
raise NotImplementedError
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Get the wrapped error (nil for Success).
|
|
101
|
+
#
|
|
102
|
+
# @return [StandardError, nil] The wrapped error or nil
|
|
103
|
+
def error
|
|
104
|
+
raise NotImplementedError
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Represents a successful operation.
|
|
109
|
+
#
|
|
110
|
+
class Success < Base
|
|
111
|
+
# @return [Object] The wrapped value
|
|
112
|
+
attr_reader :value
|
|
113
|
+
|
|
114
|
+
# Create a new Success result.
|
|
115
|
+
#
|
|
116
|
+
# @param value [Object] The wrapped value
|
|
117
|
+
def initialize(value)
|
|
118
|
+
@value = value
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Get the error (always nil for Success).
|
|
122
|
+
#
|
|
123
|
+
# @return [nil] Always nil
|
|
124
|
+
def error
|
|
125
|
+
nil
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Represents a failed operation.
|
|
130
|
+
#
|
|
131
|
+
class Failure < Base
|
|
132
|
+
# @return [StandardError] The wrapped error
|
|
133
|
+
attr_reader :error
|
|
134
|
+
|
|
135
|
+
# Create a new Failure result.
|
|
136
|
+
#
|
|
137
|
+
# @param error [StandardError] The wrapped error
|
|
138
|
+
def initialize(error)
|
|
139
|
+
@error = error
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Map does nothing for Failure.
|
|
143
|
+
#
|
|
144
|
+
# @return [Failure] Self
|
|
145
|
+
def map
|
|
146
|
+
self
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# and_then does nothing for Failure.
|
|
150
|
+
#
|
|
151
|
+
# @return [Failure] Self
|
|
152
|
+
def and_then
|
|
153
|
+
self
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Get the value (always nil for Failure).
|
|
157
|
+
#
|
|
158
|
+
# @return [nil] Always nil
|
|
159
|
+
def value
|
|
160
|
+
nil
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec/matchers"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# RSpec integration (plan 89): word and document matchers with
|
|
7
|
+
# real failure messages.
|
|
8
|
+
#
|
|
9
|
+
# RSpec is a soft dependency of the gem itself; this file requires
|
|
10
|
+
# rspec/matchers (present in any suite that would use it).
|
|
11
|
+
#
|
|
12
|
+
# @example Wiring
|
|
13
|
+
# # spec_helper.rb
|
|
14
|
+
# require "kotoshu/rspec"
|
|
15
|
+
# RSpec.configure do |config|
|
|
16
|
+
# config.include Kotoshu::Rspec::Matchers
|
|
17
|
+
# end
|
|
18
|
+
#
|
|
19
|
+
# @example Word matcher
|
|
20
|
+
# expect_words("helo", "world").to all_be_spelled_correctly
|
|
21
|
+
# expect("helo").not_to be_spelled_correctly
|
|
22
|
+
#
|
|
23
|
+
# @example Document matcher
|
|
24
|
+
# expect_document("README.adoc").to be_spelled_correctly
|
|
25
|
+
# expect("helo wrold").not_to be_spelled_correctly(in: "en")
|
|
26
|
+
module Rspec
|
|
27
|
+
# Matchers exposed to specs via config.include.
|
|
28
|
+
module Matchers
|
|
29
|
+
# Expectation target over discrete words: returns the real
|
|
30
|
+
# RSpec target so `.to all_be_spelled_correctly` works. Requires
|
|
31
|
+
# RSpec::Matchers to be in scope (it is, in any suite including
|
|
32
|
+
# this module).
|
|
33
|
+
#
|
|
34
|
+
# @param words [Array<String>] words to check
|
|
35
|
+
# @return [RSpec::Expectations::ExpectationTarget]
|
|
36
|
+
def expect_words(*words)
|
|
37
|
+
expect(words.flatten)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Expectation target over a documents contents.
|
|
41
|
+
#
|
|
42
|
+
# @param path [String] document path
|
|
43
|
+
# @return [RSpec::Expectations::ExpectationTarget]
|
|
44
|
+
def expect_document(path)
|
|
45
|
+
expect(File.read(path))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Matcher: every item of the target is a correctly spelled
|
|
49
|
+
# word. Pairs with expect_words.
|
|
50
|
+
def all_be_spelled_correctly
|
|
51
|
+
AllWordsSpelledCorrectlyMatcher.new
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Matcher: a single word is correct, or a whole text is free of
|
|
55
|
+
# misspellings (any whitespace makes it a document).
|
|
56
|
+
def be_spelled_correctly(in: nil)
|
|
57
|
+
SpelledCorrectlyMatcher.new(language: binding.local_variable_get(:in))
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Checks each word of an enumerable.
|
|
61
|
+
class AllWordsSpelledCorrectlyMatcher
|
|
62
|
+
def matches?(words)
|
|
63
|
+
@words = Array(words)
|
|
64
|
+
@misspelled = @words.reject { |word| Kotoshu.correct?(word) }
|
|
65
|
+
@misspelled.empty?
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def failure_message
|
|
69
|
+
"expected all words to be spelled correctly, but found " \
|
|
70
|
+
"#{formatted_misspellings}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def failure_message_when_negated
|
|
74
|
+
"expected some words to be misspelled, but all were correct"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def description
|
|
78
|
+
"all be spelled correctly"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# One "word -> suggestion" pair per misspelling.
|
|
84
|
+
#
|
|
85
|
+
# @return [String]
|
|
86
|
+
def formatted_misspellings
|
|
87
|
+
@misspelled.map do |word|
|
|
88
|
+
suggestions = Kotoshu.suggest(word).top(3).map(&:word)
|
|
89
|
+
detail = suggestions.empty? ? "no suggestions" : suggestions.join(", ")
|
|
90
|
+
"#{word.inspect} (#{detail})"
|
|
91
|
+
end.join(", ")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Checks one word, or a whole document when the string contains
|
|
96
|
+
# whitespace.
|
|
97
|
+
class SpelledCorrectlyMatcher
|
|
98
|
+
def initialize(language: nil)
|
|
99
|
+
@language = language
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def matches?(target)
|
|
103
|
+
@target = target.to_s
|
|
104
|
+
@errors = document? ? document_errors : word_errors
|
|
105
|
+
@errors.empty?
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def failure_message
|
|
109
|
+
if document?
|
|
110
|
+
"expected the document to be spelled correctly, but found " \
|
|
111
|
+
"#{formatted_errors}"
|
|
112
|
+
else
|
|
113
|
+
"expected #{@target.inspect} to be spelled correctly"
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def failure_message_when_negated
|
|
118
|
+
if document?
|
|
119
|
+
"expected the document to have misspellings, but it is clean"
|
|
120
|
+
else
|
|
121
|
+
"expected #{@target.inspect} to be misspelled"
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def description
|
|
126
|
+
"be spelled correctly"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
# Whether the target is a document rather than one word.
|
|
132
|
+
def document?
|
|
133
|
+
@target.match?(/\s/)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Document misspellings via the check pipeline.
|
|
137
|
+
def document_errors
|
|
138
|
+
Kotoshu.check(@target, language: @language).errors
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Single-word misspelling via correct?.
|
|
142
|
+
def word_errors
|
|
143
|
+
Kotoshu.correct?(@target, language: @language) ? [] : [@target]
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# One "word -> suggestions" pair per misspelling.
|
|
147
|
+
#
|
|
148
|
+
# @return [String]
|
|
149
|
+
def formatted_errors
|
|
150
|
+
@errors.map do |error|
|
|
151
|
+
if error.is_a?(String)
|
|
152
|
+
"#{error.inspect} (no suggestions)"
|
|
153
|
+
else
|
|
154
|
+
suggestions = error.top_suggestions(3)
|
|
155
|
+
detail = suggestions.empty? ? "no suggestions" : suggestions.join(", ")
|
|
156
|
+
"#{error.word.inspect} (#{detail})"
|
|
157
|
+
end
|
|
158
|
+
end.join(", ")
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
FastText to ONNX Converter
|
|
4
|
+
|
|
5
|
+
Converts FastText .vec files to ONNX format for efficient deployment.
|
|
6
|
+
The ONNX model contains an embedding lookup layer with the word vectors.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
python fasttext_to_onnx.py <vec_file> <output_onnx> [--vocab_size N]
|
|
10
|
+
|
|
11
|
+
Example:
|
|
12
|
+
python fasttext_to_onnx.py cc.en.300.vec fasttext.en.onnx --vocab_size 50000
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import sys
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import onnx
|
|
22
|
+
from onnx import helper, numpy_helper
|
|
23
|
+
from onnx import TensorProto
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_fasttext_vec(vec_file_path, vocab_size=None):
|
|
27
|
+
"""
|
|
28
|
+
Parse FastText .vec file and extract vocabulary and embeddings.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
vec_file_path: Path to FastText .vec file
|
|
32
|
+
vocab_size: Maximum vocabulary size (None for all words)
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
tuple: (vocab_dict, embeddings_array, metadata)
|
|
36
|
+
- vocab_dict: Dictionary mapping word to index
|
|
37
|
+
- embeddings_array: NumPy array of shape [vocab_size, embedding_dim]
|
|
38
|
+
- metadata: Dictionary with file metadata
|
|
39
|
+
"""
|
|
40
|
+
print(f"Parsing FastText file: {vec_file_path}")
|
|
41
|
+
|
|
42
|
+
word_to_idx = {}
|
|
43
|
+
embeddings = []
|
|
44
|
+
metadata = {
|
|
45
|
+
"source_format": "fasttext_vec",
|
|
46
|
+
"embedding_dim": None,
|
|
47
|
+
"vocab_size": 0,
|
|
48
|
+
"total_words_in_file": 0
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
with open(vec_file_path, 'r', encoding='utf-8') as f:
|
|
52
|
+
# First line: vocab_size and dimension
|
|
53
|
+
first_line = f.readline().strip()
|
|
54
|
+
parts = first_line.split()
|
|
55
|
+
total_vocab = int(parts[0])
|
|
56
|
+
embedding_dim = int(parts[1])
|
|
57
|
+
|
|
58
|
+
metadata["embedding_dim"] = embedding_dim
|
|
59
|
+
metadata["total_words_in_file"] = total_vocab
|
|
60
|
+
|
|
61
|
+
print(f" Total words in file: {total_vocab}")
|
|
62
|
+
print(f" Embedding dimension: {embedding_dim}")
|
|
63
|
+
|
|
64
|
+
# Limit vocab size if specified
|
|
65
|
+
if vocab_size is None or vocab_size > total_vocab:
|
|
66
|
+
vocab_size = total_vocab
|
|
67
|
+
|
|
68
|
+
metadata["vocab_size"] = vocab_size
|
|
69
|
+
|
|
70
|
+
print(f" Loading {vocab_size} words...")
|
|
71
|
+
|
|
72
|
+
# Read embeddings
|
|
73
|
+
for idx in range(vocab_size):
|
|
74
|
+
line = f.readline()
|
|
75
|
+
if not line:
|
|
76
|
+
break
|
|
77
|
+
|
|
78
|
+
parts = line.strip().split()
|
|
79
|
+
word = parts[0]
|
|
80
|
+
vector = np.array([float(x) for x in parts[1:]], dtype=np.float32)
|
|
81
|
+
|
|
82
|
+
word_to_idx[word] = idx
|
|
83
|
+
embeddings.append(vector)
|
|
84
|
+
|
|
85
|
+
if (idx + 1) % 10000 == 0:
|
|
86
|
+
print(f" Loaded {idx + 1} words...")
|
|
87
|
+
|
|
88
|
+
# Stack embeddings into a matrix
|
|
89
|
+
embeddings_matrix = np.vstack(embeddings).astype(np.float32)
|
|
90
|
+
|
|
91
|
+
print(f" Embeddings matrix shape: {embeddings_matrix.shape}")
|
|
92
|
+
print(f" Matrix size in MB: {embeddings_matrix.nbytes / (1024 * 1024):.2f}")
|
|
93
|
+
|
|
94
|
+
return word_to_idx, embeddings_matrix, metadata
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def create_onnx_model(embeddings_matrix, word_to_idx, model_name="fasttext"):
|
|
98
|
+
"""
|
|
99
|
+
Create ONNX model with embedding lookup layer.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
embeddings_matrix: NumPy array of word embeddings [vocab_size, embedding_dim]
|
|
103
|
+
word_to_idx: Dictionary mapping words to indices
|
|
104
|
+
model_name: Name for the ONNX model
|
|
105
|
+
|
|
106
|
+
Returns:
|
|
107
|
+
onnx.ModelProto: ONNX model
|
|
108
|
+
"""
|
|
109
|
+
vocab_size, embedding_dim = embeddings_matrix.shape
|
|
110
|
+
|
|
111
|
+
print(f"Creating ONNX model...")
|
|
112
|
+
print(f" Input: word_index (int64)")
|
|
113
|
+
print(f" Output: embedding (float32, shape=[{embedding_dim}])")
|
|
114
|
+
|
|
115
|
+
# Create input (word index)
|
|
116
|
+
input_tensor = helper.make_tensor_value_info(
|
|
117
|
+
'word_index',
|
|
118
|
+
TensorProto.INT64,
|
|
119
|
+
[1] # Scalar input (single word index)
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Create output (embedding vector)
|
|
123
|
+
output_tensor = helper.make_tensor_value_info(
|
|
124
|
+
'embedding',
|
|
125
|
+
TensorProto.FLOAT,
|
|
126
|
+
[embedding_dim]
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Create initializers for the embedding matrix
|
|
130
|
+
embedding_initializer = numpy_helper.from_array(embeddings_matrix, name='word_embeddings')
|
|
131
|
+
|
|
132
|
+
# Create Constant node for the embedding matrix
|
|
133
|
+
embedding_constant = helper.make_node(
|
|
134
|
+
'Constant',
|
|
135
|
+
inputs=[],
|
|
136
|
+
outputs=['embeddings_matrix'],
|
|
137
|
+
value=embedding_initializer
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# Create Gather node to lookup embedding by index
|
|
141
|
+
gather_node = helper.make_node(
|
|
142
|
+
'Gather',
|
|
143
|
+
inputs=['embeddings_matrix', 'word_index'],
|
|
144
|
+
outputs=['embedding_flat'],
|
|
145
|
+
axis=0
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
# Create Squeeze node to remove the batch dimension
|
|
149
|
+
squeeze_node = helper.make_node(
|
|
150
|
+
'Squeeze',
|
|
151
|
+
inputs=['embedding_flat'],
|
|
152
|
+
outputs=['embedding'],
|
|
153
|
+
axes=[0] # Remove first dimension
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# Create graph
|
|
157
|
+
graph = helper.make_graph(
|
|
158
|
+
[embedding_constant, gather_node, squeeze_node],
|
|
159
|
+
f'{model_name}_embedding',
|
|
160
|
+
[input_tensor],
|
|
161
|
+
[output_tensor]
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Create model
|
|
165
|
+
model = helper.make_model(
|
|
166
|
+
graph,
|
|
167
|
+
producer_name='kotoshu-fasttext-converter',
|
|
168
|
+
producer_version='1.0.0',
|
|
169
|
+
opset_imports=[helper.make_operatorsetid('', 11)], # ONNX opset 11 for compatibility
|
|
170
|
+
ir_version=11 # Set IR version to match opset
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
# Add metadata
|
|
174
|
+
from onnx import StringStringEntryProto
|
|
175
|
+
model.metadata_props.append(StringStringEntryProto(key='vocabulary_size', value=str(vocab_size)))
|
|
176
|
+
model.metadata_props.append(StringStringEntryProto(key='embedding_dimension', value=str(embedding_dim)))
|
|
177
|
+
model.metadata_props.append(StringStringEntryProto(key='model_type', value='fasttext_embedding'))
|
|
178
|
+
|
|
179
|
+
print(f" ONNX model created successfully")
|
|
180
|
+
|
|
181
|
+
return model
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def save_vocabulary(word_to_idx, vocab_file_path):
|
|
185
|
+
"""
|
|
186
|
+
Save vocabulary dictionary to JSON file.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
word_to_idx: Dictionary mapping words to indices
|
|
190
|
+
vocab_file_path: Path to save vocabulary file
|
|
191
|
+
"""
|
|
192
|
+
print(f"Saving vocabulary to: {vocab_file_path}")
|
|
193
|
+
|
|
194
|
+
vocab_data = {
|
|
195
|
+
"vocab_size": len(word_to_idx),
|
|
196
|
+
"word_to_idx": word_to_idx
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
with open(vocab_file_path, 'w', encoding='utf-8') as f:
|
|
200
|
+
json.dump(vocab_data, f, ensure_ascii=False, indent=2)
|
|
201
|
+
|
|
202
|
+
print(f" Vocabulary saved: {len(word_to_idx)} words")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def main():
|
|
206
|
+
parser = argparse.ArgumentParser(
|
|
207
|
+
description='Convert FastText .vec file to ONNX format'
|
|
208
|
+
)
|
|
209
|
+
parser.add_argument(
|
|
210
|
+
'vec_file',
|
|
211
|
+
type=str,
|
|
212
|
+
help='Path to FastText .vec file'
|
|
213
|
+
)
|
|
214
|
+
parser.add_argument(
|
|
215
|
+
'output_onnx',
|
|
216
|
+
type=str,
|
|
217
|
+
help='Path to output ONNX file'
|
|
218
|
+
)
|
|
219
|
+
parser.add_argument(
|
|
220
|
+
'--vocab-size',
|
|
221
|
+
type=int,
|
|
222
|
+
default=None,
|
|
223
|
+
help='Maximum vocabulary size (default: all words)'
|
|
224
|
+
)
|
|
225
|
+
parser.add_argument(
|
|
226
|
+
'--save-vocab',
|
|
227
|
+
type=str,
|
|
228
|
+
default=None,
|
|
229
|
+
help='Path to save vocabulary JSON file (optional)'
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
args = parser.parse_args()
|
|
233
|
+
|
|
234
|
+
# Validate input file
|
|
235
|
+
vec_file = Path(args.vec_file)
|
|
236
|
+
if not vec_file.exists():
|
|
237
|
+
print(f"Error: Input file not found: {vec_file}")
|
|
238
|
+
sys.exit(1)
|
|
239
|
+
|
|
240
|
+
# Parse FastText file
|
|
241
|
+
word_to_idx, embeddings_matrix, metadata = parse_fasttext_vec(
|
|
242
|
+
vec_file,
|
|
243
|
+
vocab_size=args.vocab_size
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
# Create ONNX model
|
|
247
|
+
model = create_onnx_model(embeddings_matrix, word_to_idx)
|
|
248
|
+
|
|
249
|
+
# Save ONNX model
|
|
250
|
+
output_path = Path(args.output_onnx)
|
|
251
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
252
|
+
|
|
253
|
+
print(f"Saving ONNX model to: {output_path}")
|
|
254
|
+
onnx.save(model, str(output_path))
|
|
255
|
+
print(f" Model saved successfully")
|
|
256
|
+
|
|
257
|
+
# Calculate file sizes
|
|
258
|
+
vec_size_mb = vec_file.stat().st_size / (1024 * 1024)
|
|
259
|
+
onnx_size_mb = output_path.stat().st_size / (1024 * 1024)
|
|
260
|
+
|
|
261
|
+
print(f"\nFile size comparison:")
|
|
262
|
+
print(f" Input .vec file: {vec_size_mb:.2f} MB")
|
|
263
|
+
print(f" Output .onnx file: {onnx_size_mb:.2f} MB")
|
|
264
|
+
print(f" Compression ratio: {(vec_size_mb / onnx_size_mb):.2f}x")
|
|
265
|
+
|
|
266
|
+
# Save vocabulary if requested
|
|
267
|
+
if args.save_vocab:
|
|
268
|
+
save_vocabulary(word_to_idx, args.save_vocab)
|
|
269
|
+
|
|
270
|
+
print(f"\nConversion complete!")
|
|
271
|
+
print(f" Model metadata: {metadata}")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
if __name__ == '__main__':
|
|
275
|
+
main()
|