kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Immutable value object for word embeddings.
|
|
6
|
+
#
|
|
7
|
+
# Represents a word and its vector representation in a semantic space.
|
|
8
|
+
# Used for semantic similarity calculations and nearest neighbor searches.
|
|
9
|
+
#
|
|
10
|
+
# @example Creating an embedding
|
|
11
|
+
# embedding = WordEmbedding.new("hello", [0.1, -0.2, 0.3], "en")
|
|
12
|
+
# embedding.similarity(other_embedding) # => 0.85
|
|
13
|
+
#
|
|
14
|
+
# @see https://fasttext.cc/docs/en/crawl-vectors.html FastText crawl vectors
|
|
15
|
+
class WordEmbedding
|
|
16
|
+
attr_reader :word, :vector, :language_code, :dimension, :magnitude
|
|
17
|
+
|
|
18
|
+
# Create a new word embedding.
|
|
19
|
+
#
|
|
20
|
+
# @param word [String] The word
|
|
21
|
+
# @param vector [Array<Float>] The word's vector representation
|
|
22
|
+
# @param language_code [String] ISO 639-1 language code
|
|
23
|
+
# @param dimension [Integer] Vector dimension (default: 300 for FastText)
|
|
24
|
+
# @raise [ArgumentError] if vector doesn't match dimension
|
|
25
|
+
def initialize(word, vector, language_code, dimension: 300)
|
|
26
|
+
raise ArgumentError, "Vector dimension mismatch" unless vector.size == dimension
|
|
27
|
+
|
|
28
|
+
@word = word
|
|
29
|
+
@vector = vector.freeze
|
|
30
|
+
@language_code = language_code
|
|
31
|
+
@dimension = dimension
|
|
32
|
+
@magnitude = Math.sqrt(@vector.map { |x| x * x }.sum)
|
|
33
|
+
|
|
34
|
+
freeze
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Calculate cosine similarity with another embedding.
|
|
38
|
+
#
|
|
39
|
+
# Cosine similarity measures the cosine of the angle between two vectors.
|
|
40
|
+
# Returns 1.0 for identical vectors, 0.0 for orthogonal vectors.
|
|
41
|
+
#
|
|
42
|
+
# @param other [WordEmbedding] Another embedding
|
|
43
|
+
# @return [Float] Similarity score (0.0 to 1.0)
|
|
44
|
+
# @raise [TypeError] if other is not a WordEmbedding
|
|
45
|
+
def similarity(other)
|
|
46
|
+
raise TypeError, "Must be WordEmbedding" unless other.is_a?(WordEmbedding)
|
|
47
|
+
|
|
48
|
+
return 0.0 if @dimension != other.dimension
|
|
49
|
+
|
|
50
|
+
dot_product = @vector.zip(other.vector).map { |a, b| a * b }.sum
|
|
51
|
+
|
|
52
|
+
return 0.0 if magnitude.zero? || other.magnitude.zero?
|
|
53
|
+
|
|
54
|
+
dot_product / (magnitude * other.magnitude)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Calculate Euclidean distance from another embedding.
|
|
58
|
+
#
|
|
59
|
+
# @param other [WordEmbedding] Another embedding
|
|
60
|
+
# @return [Float] Euclidean distance
|
|
61
|
+
# @raise [TypeError] if other is not a WordEmbedding
|
|
62
|
+
def distance(other)
|
|
63
|
+
raise TypeError, "Must be WordEmbedding" unless other.is_a?(WordEmbedding)
|
|
64
|
+
|
|
65
|
+
return Float::INFINITY if @dimension != other.dimension
|
|
66
|
+
|
|
67
|
+
Math.sqrt(@vector.zip(other.vector).map { |a, b| (a - b)**2 }.sum)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Check if this embedding is equal to another.
|
|
71
|
+
#
|
|
72
|
+
# @param other [Object] Another object
|
|
73
|
+
# @return [Boolean] True if words and languages match
|
|
74
|
+
def ==(other)
|
|
75
|
+
return false unless other.is_a?(WordEmbedding)
|
|
76
|
+
|
|
77
|
+
@word == other.word && @language_code == other.language_code
|
|
78
|
+
end
|
|
79
|
+
alias_method :eql?, :==
|
|
80
|
+
|
|
81
|
+
# Hash code for hash table usage.
|
|
82
|
+
#
|
|
83
|
+
# @return [Integer] Hash code
|
|
84
|
+
def hash
|
|
85
|
+
[@word, @language_code].hash
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# String representation.
|
|
89
|
+
#
|
|
90
|
+
# @return [String] Human-readable representation
|
|
91
|
+
def to_s
|
|
92
|
+
"#{self.class.name}[#{@word}, #{@language_code}, #{@dimension}D]"
|
|
93
|
+
end
|
|
94
|
+
alias_method :inspect, :to_s
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Embedding models for semantic spell checking (FastText via ONNX),
|
|
5
|
+
# and core domain value objects (Word, AffixRule, Result types).
|
|
6
|
+
#
|
|
7
|
+
# Note: Word, AffixRule, and Result types live under Kotoshu::Models
|
|
8
|
+
# for historical reasons even though their files are in core/models/.
|
|
9
|
+
module Models
|
|
10
|
+
autoload :Word, "kotoshu/core/models/word"
|
|
11
|
+
autoload :AffixRule, "kotoshu/core/models/affix_rule"
|
|
12
|
+
autoload :Context, "kotoshu/models/context"
|
|
13
|
+
autoload :EmbeddingModel, "kotoshu/models/embedding_model"
|
|
14
|
+
autoload :FastTextModel, "kotoshu/models/fasttext_model"
|
|
15
|
+
autoload :NearestNeighbor, "kotoshu/models/nearest_neighbor"
|
|
16
|
+
autoload :OnnxModel, "kotoshu/models/onnx_model"
|
|
17
|
+
autoload :SemanticError, "kotoshu/models/semantic_error"
|
|
18
|
+
autoload :Suggestion, "kotoshu/models/suggestion"
|
|
19
|
+
autoload :WordEmbedding, "kotoshu/models/word_embedding"
|
|
20
|
+
|
|
21
|
+
# Result sub-namespace (word/document check results).
|
|
22
|
+
module Result
|
|
23
|
+
autoload :WordResult, "kotoshu/core/models/result/word_result"
|
|
24
|
+
autoload :DocumentResult, "kotoshu/core/models/result/document_result"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Multi-language document checker.
|
|
5
|
+
#
|
|
6
|
+
# Experimental and not public API: outside the 1.0 stability
|
|
7
|
+
# freeze, may change or be removed without a deprecation cycle.
|
|
8
|
+
#
|
|
9
|
+
# Splits a {Documents::Document} into language-tagged segments via
|
|
10
|
+
# {Language::Segmenter}, resolves a {Spellchecker} for each
|
|
11
|
+
# segment's detected language, and emits a flat list of
|
|
12
|
+
# {Models::SemanticError}s carrying source_range from the original
|
|
13
|
+
# document. The result is one sorted stream of errors that editors
|
|
14
|
+
# / plugins can highlight without caring that the document mixed
|
|
15
|
+
# languages.
|
|
16
|
+
#
|
|
17
|
+
# Real documents mix languages routinely: code comments in English
|
|
18
|
+
# embedded in French prose, a Japanese paper with an English
|
|
19
|
+
# abstract, German quotes in a Spanish essay. Without per-segment
|
|
20
|
+
# detection, every word from the "other" language is reported as a
|
|
21
|
+
# spelling error against the wrong dictionary.
|
|
22
|
+
class MultiLanguageChecker
|
|
23
|
+
# @return [Language::Segmenter]
|
|
24
|
+
attr_reader :segmenter
|
|
25
|
+
|
|
26
|
+
# @return [Hash{String => Spellchecker}] per-language cache so
|
|
27
|
+
# repeated segments of the same language reuse the same checker
|
|
28
|
+
attr_reader :spellcheckers
|
|
29
|
+
|
|
30
|
+
# @param segmenter [Language::Segmenter, nil] defaults to a new
|
|
31
|
+
# Segmenter with the default Detector
|
|
32
|
+
def initialize(segmenter: nil)
|
|
33
|
+
@segmenter = segmenter || Language::Segmenter.new
|
|
34
|
+
@spellcheckers = {}
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Check a document, returning one sorted stream of errors across
|
|
38
|
+
# every detected language. Errors carry source_range resolved via
|
|
39
|
+
# the document's text-node mapping.
|
|
40
|
+
#
|
|
41
|
+
# @param document [Kotoshu::Documents::Document]
|
|
42
|
+
# @return [Array<Models::SemanticError>]
|
|
43
|
+
def check(document)
|
|
44
|
+
unless document.is_a?(Kotoshu::Documents::Document)
|
|
45
|
+
raise ArgumentError,
|
|
46
|
+
"document must be a Kotoshu::Documents::Document"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
@segmenter.segment(document).flat_map do |segment|
|
|
50
|
+
check_segment(segment, document)
|
|
51
|
+
end.sort
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
# Check one segment: resolve (or reuse) a Spellchecker for the
|
|
57
|
+
# segment's language, walk its text nodes, and build errors with
|
|
58
|
+
# source_range for every unknown word that has at least one
|
|
59
|
+
# suggestion.
|
|
60
|
+
def check_segment(segment, document)
|
|
61
|
+
checker = spellchecker_for(segment.language_code)
|
|
62
|
+
return [] unless checker
|
|
63
|
+
|
|
64
|
+
errors = []
|
|
65
|
+
segment.text_nodes.each do |node|
|
|
66
|
+
tokenize_with_offsets(node.text).each do |word, offset_in_node|
|
|
67
|
+
next if checker.correct?(word)
|
|
68
|
+
|
|
69
|
+
suggestions = checker.suggest(word).to_a
|
|
70
|
+
next if suggestions.empty?
|
|
71
|
+
|
|
72
|
+
flattened_start = node.flattened_offset + offset_in_node
|
|
73
|
+
flattened_end = flattened_start + word.length
|
|
74
|
+
source_range = document.source_range_for(flattened_start, flattened_end)
|
|
75
|
+
errors << build_error(word, suggestions, source_range, segment.language_code)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
errors
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Resolve a Spellchecker for a language code, memoized per
|
|
82
|
+
# process. Returns nil when the language isn't set up — callers
|
|
83
|
+
# silently skip unknown-language segments rather than crashing.
|
|
84
|
+
def spellchecker_for(language_code)
|
|
85
|
+
return @spellcheckers[language_code] if @spellcheckers.key?(language_code)
|
|
86
|
+
|
|
87
|
+
@spellcheckers[language_code] = begin
|
|
88
|
+
bundle = ResourceManager.resolve(language: language_code, want: %i[spelling])
|
|
89
|
+
Spellchecker.new(resource_bundle: bundle)
|
|
90
|
+
rescue StandardError
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Build a SemanticError. Uses the orthographic error type (this
|
|
96
|
+
# is a spelling check, not a grammar check). Confidence is a
|
|
97
|
+
# fixed 0.5 (medium) since we don't compute real confidence here.
|
|
98
|
+
def build_error(word, suggestions, source_range, language_code)
|
|
99
|
+
Models::SemanticError.new(
|
|
100
|
+
id: Digest::SHA256.hexdigest("#{language_code}-#{word}-#{source_range}")[0...16],
|
|
101
|
+
source_range: source_range,
|
|
102
|
+
original: word,
|
|
103
|
+
suggestions: wrap_suggestions(suggestions),
|
|
104
|
+
error_type: :orthographic,
|
|
105
|
+
confidence: 0.5,
|
|
106
|
+
context: nil
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Wrap raw suggestion strings into Models::Suggestion objects so
|
|
111
|
+
# the SemanticError invariant (array of Suggestion) is satisfied.
|
|
112
|
+
def wrap_suggestions(words)
|
|
113
|
+
words.first(5).map do |word|
|
|
114
|
+
Models::Suggestion.new(word, confidence: 0.5, source: :multi_language)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Tokenize +text+ into [word, char_offset] pairs.
|
|
119
|
+
def tokenize_with_offsets(text)
|
|
120
|
+
return [] unless text
|
|
121
|
+
|
|
122
|
+
text.scan(/[a-zà-ÿ]+(?:['’-][a-zà-ÿ]+)*/i).map do |match|
|
|
123
|
+
[match.downcase, Regexp.last_match.offset(0).first]
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Ruby-side home of the native engine (plan 66, P4b).
|
|
5
|
+
#
|
|
6
|
+
# The Rust extension +kotoshu/kotoshu_native+ (built from
|
|
7
|
+
# ext/kotoshu_native) defines this module's engine surface when it loads:
|
|
8
|
+
#
|
|
9
|
+
# Kotoshu::Native::VERSION # kotoshu-rs crate version
|
|
10
|
+
# Kotoshu::Native.available? # => true
|
|
11
|
+
# Kotoshu::Native::Error # engine failures (RuntimeError)
|
|
12
|
+
# dictionary = Kotoshu::Native::Dictionary.load(aff_path, dic_path)
|
|
13
|
+
# dictionary.correct?("hello") # => true / false
|
|
14
|
+
# dictionary.suggest("hlelo", 5) # => [{ "word" => "hello", ... }]
|
|
15
|
+
#
|
|
16
|
+
# This file adds two things the extension cannot:
|
|
17
|
+
#
|
|
18
|
+
# * the safe loader — +require "kotoshu"+ never fails when the extension
|
|
19
|
+
# is absent. A missing Rust toolchain, an uncompiled checkout, or a
|
|
20
|
+
# slim install simply leaves {Native.available?} false, and the
|
|
21
|
+
# pure-Ruby backend remains the complete engine. Mirrors the parsanol
|
|
22
|
+
# loader pattern (lib/parsanol/native.rb): rescue LoadError, stay quiet.
|
|
23
|
+
# * the adapter — native suggestion rows are Hashes carrying exactly the
|
|
24
|
+
# four public keys of {Suggestions::Suggestion}; {Native.suggestion}
|
|
25
|
+
# materializes the model object from a row. No serialization is
|
|
26
|
+
# hand-rolled here: the row is constructor input, the model's
|
|
27
|
+
# (de)serialization stays lutaml-model's.
|
|
28
|
+
module Native
|
|
29
|
+
# Raised by backend selection when the native engine is explicitly
|
|
30
|
+
# requested ({Configuration#backend} = "native") but cannot serve the
|
|
31
|
+
# request: extension not built, or a dictionary type the native engine
|
|
32
|
+
# does not load. Failing loudly is deliberate — "native" exists for
|
|
33
|
+
# debugging and benchmarking, where a silent fallback to Ruby would
|
|
34
|
+
# produce misleading measurements.
|
|
35
|
+
class Unavailable < Kotoshu::Error; end
|
|
36
|
+
|
|
37
|
+
# The extension's require path (rb_sys installs it as
|
|
38
|
+
# kotoshu/kotoshu_native.<dlext> under lib/).
|
|
39
|
+
EXTENSION = "kotoshu/kotoshu_native"
|
|
40
|
+
|
|
41
|
+
class << self
|
|
42
|
+
# Whether the native extension is loaded.
|
|
43
|
+
#
|
|
44
|
+
# Overwritten by the extension itself when it loads (its own
|
|
45
|
+
# +available?+ answers true), so this definition only stands when the
|
|
46
|
+
# require below failed.
|
|
47
|
+
#
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
def available?
|
|
50
|
+
false
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Materialize a {Suggestions::Suggestion} from a native suggestion
|
|
54
|
+
# row.
|
|
55
|
+
#
|
|
56
|
+
# The Rust engine returns one Hash per suggestion with exactly the
|
|
57
|
+
# four public keys of the gem's Suggestion shape ("word" String,
|
|
58
|
+
# "distance" Integer, "confidence" Float in [0, 1], "source" String)
|
|
59
|
+
# — the same four keys the conformance vectors freeze
|
|
60
|
+
# ({ConformanceExporter::SUGGESTION_KEYS}). This constructor hop is
|
|
61
|
+
# the whole adapter; rows and model attributes are one-to-one.
|
|
62
|
+
#
|
|
63
|
+
# @param row [Hash] native suggestion row
|
|
64
|
+
# @return [Suggestions::Suggestion]
|
|
65
|
+
def suggestion(row)
|
|
66
|
+
Suggestions::Suggestion.new(
|
|
67
|
+
word: row.fetch("word"),
|
|
68
|
+
distance: row.fetch("distance"),
|
|
69
|
+
confidence: row.fetch("confidence"),
|
|
70
|
+
source: row.fetch("source")
|
|
71
|
+
)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
begin
|
|
77
|
+
require Native::EXTENSION
|
|
78
|
+
rescue LoadError
|
|
79
|
+
# Extension not built (no Rust toolchain, or `rake compile` not run).
|
|
80
|
+
# Nothing else in the gem requires it: the pure-Ruby backend is the
|
|
81
|
+
# complete engine and every public API works without the extension.
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Native engine adapter (plan 66, P4b): +correct?+ and +suggest+ for one
|
|
5
|
+
# loaded Hunspell dictionary, computed by the kotoshu-rs core through
|
|
6
|
+
# {Native}.
|
|
7
|
+
#
|
|
8
|
+
# == Boundary (deliberate, documented)
|
|
9
|
+
#
|
|
10
|
+
# The native engine accelerates exactly the two hot-path calls of the
|
|
11
|
+
# traditional checking path — correct?(word) and suggest(word) — over a
|
|
12
|
+
# file-backed Hunspell dictionary, where kotoshu-rs implements the full
|
|
13
|
+
# dictionary + affix + suggest composite and is held to byte-identical
|
|
14
|
+
# results by the conformance vectors (rake kotoshu:conformance:compare).
|
|
15
|
+
# Everything else stays Ruby: non-Hunspell dictionary backends, the
|
|
16
|
+
# suggestion strategy pipeline beyond the conformance surface, the
|
|
17
|
+
# semantic path ({Analyzers::SemanticAnalyzer}, ONNX models), grammar
|
|
18
|
+
# rules, caches, and document tokenizing. The extension is a pure
|
|
19
|
+
# accelerator, never a dependency.
|
|
20
|
+
#
|
|
21
|
+
# == Selection
|
|
22
|
+
#
|
|
23
|
+
# See {Configuration#backend} / +KOTOSHU_BACKEND+:
|
|
24
|
+
#
|
|
25
|
+
# ruby :: never uses the native engine (default).
|
|
26
|
+
# auto :: uses it when the extension is loaded AND the dictionary is a
|
|
27
|
+
# file-backed Hunspell; silently pure-Ruby otherwise.
|
|
28
|
+
# native :: uses it or raises {Native::Unavailable} — loud failure for
|
|
29
|
+
# debugging and benchmarking.
|
|
30
|
+
class NativeBackend
|
|
31
|
+
# @return [Kotoshu::Native::Dictionary] the loaded native dictionary
|
|
32
|
+
attr_reader :native_dictionary
|
|
33
|
+
|
|
34
|
+
# Resolve which backend serves a Spellchecker built over +dictionary+.
|
|
35
|
+
#
|
|
36
|
+
# @param dictionary [Dictionary::Base, Object] the dictionary the
|
|
37
|
+
# Spellchecker was built with
|
|
38
|
+
# @param backend [String, Symbol] "ruby", "auto", or "native"
|
|
39
|
+
# @param max_suggestions [Integer] default suggestion limit
|
|
40
|
+
# @return [NativeBackend, nil] nil when the pure-Ruby engine serves the
|
|
41
|
+
# request
|
|
42
|
+
# @raise [Native::Unavailable] backend "native" and the engine cannot
|
|
43
|
+
# serve the request
|
|
44
|
+
# @raise [ConfigurationError] unknown backend value
|
|
45
|
+
def self.resolve(dictionary:, backend:, max_suggestions: 10)
|
|
46
|
+
case backend.to_s
|
|
47
|
+
when "ruby"
|
|
48
|
+
nil
|
|
49
|
+
when "auto"
|
|
50
|
+
paths = hunspell_paths(dictionary)
|
|
51
|
+
return nil unless paths && Kotoshu::Native.available?
|
|
52
|
+
|
|
53
|
+
new(aff_path: paths.fetch(0), dic_path: paths.fetch(1), max_suggestions: max_suggestions)
|
|
54
|
+
when "native"
|
|
55
|
+
unless Kotoshu::Native.available?
|
|
56
|
+
raise Native::Unavailable,
|
|
57
|
+
"native backend requested (KOTOSHU_BACKEND=native) but the extension is not " \
|
|
58
|
+
"available — build it with `rake compile` or install the gem where it compiled"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
paths = hunspell_paths(dictionary)
|
|
62
|
+
unless paths
|
|
63
|
+
raise Native::Unavailable,
|
|
64
|
+
"native backend requested (KOTOSHU_BACKEND=native) but it only loads " \
|
|
65
|
+
"Hunspell dictionaries — got #{dictionary.class}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
new(aff_path: paths.fetch(0), dic_path: paths.fetch(1), max_suggestions: max_suggestions)
|
|
69
|
+
else
|
|
70
|
+
raise ConfigurationError,
|
|
71
|
+
"unknown backend #{backend.inspect} (expected \"ruby\", \"auto\", or \"native\")"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# The (aff, dic) paths of a file-backed Hunspell dictionary — the only
|
|
76
|
+
# dictionary form the native engine loads. Everything else answers nil.
|
|
77
|
+
#
|
|
78
|
+
# @param dictionary [Object]
|
|
79
|
+
# @return [Array<String>, nil]
|
|
80
|
+
def self.hunspell_paths(dictionary)
|
|
81
|
+
return nil unless dictionary.is_a?(Dictionary::Hunspell)
|
|
82
|
+
|
|
83
|
+
aff = dictionary.aff_path
|
|
84
|
+
dic = dictionary.dic_path
|
|
85
|
+
return nil if aff.nil? || dic.nil?
|
|
86
|
+
|
|
87
|
+
[aff, dic]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# @param aff_path [String] path to the .aff file
|
|
91
|
+
# @param dic_path [String] path to the .dic file
|
|
92
|
+
# @param max_suggestions [Integer] default suggestion limit
|
|
93
|
+
# @raise [Native::Error] the dictionary failed to load (message from
|
|
94
|
+
# the Rust engine)
|
|
95
|
+
def initialize(aff_path:, dic_path:, max_suggestions: 10)
|
|
96
|
+
@max_suggestions = max_suggestions
|
|
97
|
+
@native_dictionary = Kotoshu::Native::Dictionary.load(aff_path, dic_path)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Check a word against the native dictionary.
|
|
101
|
+
#
|
|
102
|
+
# @param word [String]
|
|
103
|
+
# @return [Boolean]
|
|
104
|
+
def correct?(word)
|
|
105
|
+
@native_dictionary.correct?(word)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Suggest corrections for a word through the native engine, returning
|
|
109
|
+
# the same {Suggestions::SuggestionSet} the Ruby path returns.
|
|
110
|
+
#
|
|
111
|
+
# The Rust engine returns rows already ranked, deduplicated, and
|
|
112
|
+
# limited — its ranking is conformance-frozen to the Ruby engine's
|
|
113
|
+
# behavior (rake kotoshu:conformance:compare), and its tie order can
|
|
114
|
+
# depend on ranking signals that do not live on the Suggestion objects.
|
|
115
|
+
# The set therefore adopts the native order verbatim (ranked: true)
|
|
116
|
+
# instead of re-sorting.
|
|
117
|
+
#
|
|
118
|
+
# @param word [String]
|
|
119
|
+
# @param max_suggestions [Integer, nil] limit override
|
|
120
|
+
# @return [Suggestions::SuggestionSet]
|
|
121
|
+
def suggest(word, max_suggestions: nil)
|
|
122
|
+
limit = max_suggestions || @max_suggestions
|
|
123
|
+
rows = @native_dictionary.suggest(word, limit)
|
|
124
|
+
Suggestions::SuggestionSet.new(
|
|
125
|
+
rows.map { |row| Kotoshu::Native.suggestion(row) },
|
|
126
|
+
max_size: limit,
|
|
127
|
+
ranked: true
|
|
128
|
+
)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# Centralized XDG Base Directory paths.
|
|
7
|
+
#
|
|
8
|
+
# Resolves cache, config, and data paths per the XDG Base Directory
|
|
9
|
+
# Specification. Kotoshu-specific env vars (KOTOSHU_CACHE_PATH etc.)
|
|
10
|
+
# take precedence over the generic XDG_*_HOME vars, which take
|
|
11
|
+
# precedence over the hardcoded defaults.
|
|
12
|
+
#
|
|
13
|
+
# Default layout:
|
|
14
|
+
# ~/.cache/kotoshu/ downloaded dicts, models, frequency lists
|
|
15
|
+
# ~/.config/kotoshu/ user-edited config, personal dictionary
|
|
16
|
+
# ~/.local/share/kotoshu/ append-only data (audit log)
|
|
17
|
+
module Paths
|
|
18
|
+
class << self
|
|
19
|
+
# Root directory for downloaded resources (dictionaries,
|
|
20
|
+
# frequency lists, ONNX models).
|
|
21
|
+
#
|
|
22
|
+
# @return [String] KOTOSHU_CACHE_PATH, or
|
|
23
|
+
# `$XDG_CACHE_HOME/kotoshu` (default `~/.cache/kotoshu`)
|
|
24
|
+
def cache_path
|
|
25
|
+
ENV.fetch("KOTOSHU_CACHE_PATH", nil) || xdg("CACHE", "cache")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Root directory for user configuration (kotoshu.cfg, personal
|
|
29
|
+
# dictionary).
|
|
30
|
+
#
|
|
31
|
+
# @return [String] KOTOSHU_CONFIG_PATH, or
|
|
32
|
+
# `$XDG_CONFIG_HOME/kotoshu` (default `~/.config/kotoshu`)
|
|
33
|
+
def config_path
|
|
34
|
+
ENV.fetch("KOTOSHU_CONFIG_PATH", nil) || xdg("CONFIG", "config")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Root directory for shared state (audit log).
|
|
38
|
+
#
|
|
39
|
+
# @return [String] KOTOSHU_DATA_PATH, or
|
|
40
|
+
# `$XDG_DATA_HOME/kotoshu` (default `~/.local/share/kotoshu`)
|
|
41
|
+
def data_path
|
|
42
|
+
ENV.fetch("KOTOSHU_DATA_PATH", nil) || xdg("DATA", "local/share")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Where audited operations are appended (with rotation; see
|
|
46
|
+
# Configuration#audit_max_bytes / #audit_rotations).
|
|
47
|
+
#
|
|
48
|
+
# @return [String] KOTOSHU_AUDIT_LOG, or `<data_path>/audit.log`
|
|
49
|
+
def audit_log_path
|
|
50
|
+
ENV.fetch("KOTOSHU_AUDIT_LOG", nil) || File.join(data_path, "audit.log")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Where {PersonalDictionary} stores its words (Hunspell word-list
|
|
54
|
+
# format).
|
|
55
|
+
#
|
|
56
|
+
# @return [String] KOTOSHU_PERSONAL_DIC, or
|
|
57
|
+
# `<config_path>/personal.dic`
|
|
58
|
+
def personal_dictionary_path
|
|
59
|
+
ENV.fetch("KOTOSHU_PERSONAL_DIC", nil) || File.join(config_path, "personal.dic")
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Create the cache, config, and data directories. Idempotent.
|
|
63
|
+
#
|
|
64
|
+
# @return [void]
|
|
65
|
+
def ensure_exist!
|
|
66
|
+
FileUtils.mkdir_p(cache_path)
|
|
67
|
+
FileUtils.mkdir_p(config_path)
|
|
68
|
+
FileUtils.mkdir_p(data_path)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def xdg(suffix, default_subdir)
|
|
74
|
+
base = ENV.fetch("XDG_#{suffix}_HOME", nil) || File.join(Dir.home, ".#{default_subdir}")
|
|
75
|
+
File.join(base, "kotoshu")
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# Personal dictionary for user-specific words.
|
|
7
|
+
#
|
|
8
|
+
# Stored in ~/.config/kotoshu/personal.dic (Hunspell format) under the
|
|
9
|
+
# XDG config directory. Override via KOTOSHU_PERSONAL_DIC.
|
|
10
|
+
#
|
|
11
|
+
# The on-disk path is resolved lazily on every call so env-var
|
|
12
|
+
# overrides take effect at runtime — handy for tests and for users
|
|
13
|
+
# who set KOTOSHU_PERSONAL_DIC after the gem loads.
|
|
14
|
+
class PersonalDictionary
|
|
15
|
+
class << self
|
|
16
|
+
# Resolve the personal-dictionary file path from Paths (which
|
|
17
|
+
# honors KOTOSHU_PERSONAL_DIC). Read fresh each call so an
|
|
18
|
+
# env override applied after gem load still takes effect.
|
|
19
|
+
#
|
|
20
|
+
# @return [String]
|
|
21
|
+
def file_path
|
|
22
|
+
Kotoshu::Paths.personal_dictionary_path
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Add a word to personal dictionary.
|
|
26
|
+
#
|
|
27
|
+
# @param word [String] Word to add
|
|
28
|
+
# @return [Boolean] True if the word was newly added (i.e. it
|
|
29
|
+
# wasn't already in the dictionary); false otherwise.
|
|
30
|
+
def add_word(word)
|
|
31
|
+
return false if word.nil? || word.empty?
|
|
32
|
+
|
|
33
|
+
ensure_directory
|
|
34
|
+
words = load_words
|
|
35
|
+
|
|
36
|
+
return false if words.include?(word.downcase)
|
|
37
|
+
|
|
38
|
+
words << word.downcase
|
|
39
|
+
save_words(words)
|
|
40
|
+
true
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Get all personal words.
|
|
44
|
+
#
|
|
45
|
+
# @return [Array<String>] All personal words
|
|
46
|
+
def words
|
|
47
|
+
load_words
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Remove a word from personal dictionary.
|
|
51
|
+
#
|
|
52
|
+
# @param word [String] Word to remove
|
|
53
|
+
# @return [Boolean] True if removed
|
|
54
|
+
def remove_word(word)
|
|
55
|
+
return false if word.nil? || word.empty?
|
|
56
|
+
|
|
57
|
+
words = load_words
|
|
58
|
+
if words.delete(word.downcase)
|
|
59
|
+
save_words(words)
|
|
60
|
+
true
|
|
61
|
+
else
|
|
62
|
+
false
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Check if word is in personal dictionary.
|
|
67
|
+
#
|
|
68
|
+
# @param word [String] Word to check
|
|
69
|
+
# @return [Boolean] True if present
|
|
70
|
+
def include?(word)
|
|
71
|
+
return false if word.nil? || word.empty?
|
|
72
|
+
|
|
73
|
+
load_words.include?(word.downcase)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
# Ensure personal dictionary's parent directory exists.
|
|
79
|
+
def ensure_directory
|
|
80
|
+
FileUtils.mkdir_p(File.dirname(file_path))
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Load words from personal dictionary file.
|
|
84
|
+
#
|
|
85
|
+
# @return [Array<String>] List of words
|
|
86
|
+
def load_words
|
|
87
|
+
return [] unless File.exist?(file_path)
|
|
88
|
+
|
|
89
|
+
File.readlines(file_path, chomp: true)
|
|
90
|
+
.reject { |line| line.empty? || line.start_with?("#") }
|
|
91
|
+
.map(&:strip)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Save words to personal dictionary file.
|
|
95
|
+
#
|
|
96
|
+
# @param words [Array<String>] Words to save
|
|
97
|
+
def save_words(words)
|
|
98
|
+
ensure_directory
|
|
99
|
+
File.open(file_path, "w") do |f|
|
|
100
|
+
words.sort.uniq.each { |word| f.puts word }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|