kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Kotoshu
|
|
7
|
+
# Exports conformance vectors for the kotoshu-rs test suite (TODO.impl plan
|
|
8
|
+
# 67, M3).
|
|
9
|
+
#
|
|
10
|
+
# The exporter walks the Hunspell fixture corpora that live in this
|
|
11
|
+
# repository, runs the REAL engine -- one real +Spellchecker+ over one real
|
|
12
|
+
# +Dictionary::Hunspell+ per fixture, no mocks or stubs anywhere -- and emits
|
|
13
|
+
# one JSON object per line (JSONL).
|
|
14
|
+
#
|
|
15
|
+
# == Row shape (v0)
|
|
16
|
+
#
|
|
17
|
+
# Matches kotoshu-rs' +tests/conformance/v0-placeholders.jsonl+, plus a
|
|
18
|
+
# +dictionary+ key that identifies the fixture the row was produced against
|
|
19
|
+
# (without it a consumer cannot reproduce the engine call):
|
|
20
|
+
#
|
|
21
|
+
# {"kind":"correct","language":"en","dictionary":"spec/integrational/fixtures/base",
|
|
22
|
+
# "input":"created","expected":true}
|
|
23
|
+
# {"kind":"suggest","language":"en","dictionary":"spec/integrational/fixtures/base",
|
|
24
|
+
# "input":"hlelo","limit":5,"expected":[{"word":"hello","distance":1,
|
|
25
|
+
# "confidence":1.0,"source":"edit_distance"}]}
|
|
26
|
+
#
|
|
27
|
+
# +expected+ entries for +suggest+ come from the framework serialization of
|
|
28
|
+
# +Suggestions::Suggestion+ (+to_hash+, provided by lutaml-model -- nothing
|
|
29
|
+
# hand-rolled here), reduced to the four keys the kotoshu-rs harness parses.
|
|
30
|
+
#
|
|
31
|
+
# == Semantics of +expected+ -- FROZEN ENGINE BEHAVIOR
|
|
32
|
+
#
|
|
33
|
+
# +expected+ records what the Ruby engine ACTUALLY returns, not what the
|
|
34
|
+
# Hunspell fixture says SHOULD be returned. The point of this file is
|
|
35
|
+
# cross-implementation equality (kotoshu-rs must return byte-identical
|
|
36
|
+
# results), not linguistic correctness. If the Ruby engine changes behavior,
|
|
37
|
+
# the vectors are re-exported and both implementations move together.
|
|
38
|
+
#
|
|
39
|
+
# == Corpora
|
|
40
|
+
#
|
|
41
|
+
# * Spylls corpus: every +spec/integrational/fixtures/*.{aff,dic}+ pair;
|
|
42
|
+
# words are taken from the matching +.good+ / +.wrong+ lists using the
|
|
43
|
+
# Spylls +read_list+ conventions (strip, drop empty lines, drop lines
|
|
44
|
+
# ending in "."). Dictionaries without word lists contribute no rows.
|
|
45
|
+
# * Spec fixture corpus: +spec/fixtures/dictionaries/hunspell/test.{aff,dic}+
|
|
46
|
+
# (its .dic entries plus +spec/fixtures/words.txt+).
|
|
47
|
+
#
|
|
48
|
+
# == Determinism
|
|
49
|
+
#
|
|
50
|
+
# Corpora are iterated sorted by id, words keep fixture file order (.good
|
|
51
|
+
# first, then .wrong, deduplicated), hash keys are built as stable literals,
|
|
52
|
+
# and no timestamps are written. Exporting twice yields byte-identical
|
|
53
|
+
# output.
|
|
54
|
+
#
|
|
55
|
+
# == Sync contract with kotoshu-rs
|
|
56
|
+
#
|
|
57
|
+
# The generated file (default +conformance/vectors.jsonl+ at the gem repo
|
|
58
|
+
# root) IS committed to this repository -- conformance/ is deliberately NOT
|
|
59
|
+
# gitignored. kotoshu-rs copies it into +tests/conformance/+ via its own
|
|
60
|
+
# documented sync step, so its CI consumes the vectors without the gem.
|
|
61
|
+
class ConformanceExporter
|
|
62
|
+
# Fixed suggestion limit used for every "suggest" vector.
|
|
63
|
+
SUGGEST_LIMIT = 5
|
|
64
|
+
|
|
65
|
+
# Language recorded in every row (the engine language_code the exporter
|
|
66
|
+
# passes when loading fixture dictionaries, mirroring SpyllsTestHelper).
|
|
67
|
+
LANGUAGE = "en"
|
|
68
|
+
|
|
69
|
+
# Corpus ids excluded from the default export.
|
|
70
|
+
#
|
|
71
|
+
# "timelimit" is Hunspell's exponential-compounding torture test
|
|
72
|
+
# (COMPOUNDMIN 1 over runs of zeros): +correct?+ survives it, but
|
|
73
|
+
# +suggest+ never terminates in the Ruby engine, so no vector can be
|
|
74
|
+
# frozen. The fixture stays covered by spec/integrational/lookup_spec.rb.
|
|
75
|
+
EXCLUDED_FIXTURES = ["spec/integrational/fixtures/timelimit"].freeze
|
|
76
|
+
|
|
77
|
+
# Keys kept from Suggestions::Suggestion#to_hash (lutaml-model
|
|
78
|
+
# serialization) in "suggest" expected payloads. Mirrors the
|
|
79
|
+
# kotoshu-rs v0 placeholder schema; "metadata" is dropped.
|
|
80
|
+
SUGGESTION_KEYS = %w[word distance confidence source].freeze
|
|
81
|
+
|
|
82
|
+
# One fixture dictionary plus the words to run against it.
|
|
83
|
+
Corpus = Struct.new(:id, :aff_path, :dic_path, :words, keyword_init: true)
|
|
84
|
+
|
|
85
|
+
# Outcome of an export run.
|
|
86
|
+
ExportResult = Struct.new(:path, :row_count, :corpus_count, :skipped_corpora, :skipped_words,
|
|
87
|
+
keyword_init: true)
|
|
88
|
+
|
|
89
|
+
class << self
|
|
90
|
+
# Root of the gem repository (this file lives in lib/kotoshu/).
|
|
91
|
+
#
|
|
92
|
+
# @return [String] Absolute path to the repo root
|
|
93
|
+
def gem_root
|
|
94
|
+
File.expand_path("../..", __dir__)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Default corpora: the Spylls fixtures plus the spec fixture corpus.
|
|
98
|
+
#
|
|
99
|
+
# Applies EXCLUDED_FIXTURES and drops corpora with no words, so the
|
|
100
|
+
# export is stable regardless of which auxiliary dictionaries happen to
|
|
101
|
+
# sit in the fixture directory.
|
|
102
|
+
#
|
|
103
|
+
# @param gem_root [String] Repo root to resolve fixture paths against
|
|
104
|
+
# @return [Array<Corpus>] Corpora sorted by id
|
|
105
|
+
def default_corpora(gem_root: self.gem_root)
|
|
106
|
+
spylls_corpora(gem_root: gem_root) + [spec_fixture_corpus(gem_root: gem_root)]
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Scan the Spylls fixture directory for Hunspell dictionaries.
|
|
110
|
+
#
|
|
111
|
+
# @param gem_root [String] Repo root
|
|
112
|
+
# @return [Array<Corpus>] Non-excluded corpora with at least one word,
|
|
113
|
+
# sorted by id
|
|
114
|
+
def spylls_corpora(gem_root: self.gem_root)
|
|
115
|
+
dir = File.join(gem_root, "spec/integrational/fixtures")
|
|
116
|
+
Dir.glob(File.join(dir, "*.dic")).filter_map do |dic_path|
|
|
117
|
+
base = dic_path.delete_suffix(".dic")
|
|
118
|
+
id = "spec/integrational/fixtures/#{File.basename(base)}"
|
|
119
|
+
next if EXCLUDED_FIXTURES.include?(id)
|
|
120
|
+
|
|
121
|
+
words = (read_word_list("#{base}.good") + read_word_list("#{base}.wrong")).uniq
|
|
122
|
+
next if words.empty?
|
|
123
|
+
|
|
124
|
+
Corpus.new(id: id, aff_path: "#{base}.aff", dic_path: dic_path, words: words)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# The spec fixture corpus: the small Hunspell test dictionary shipped
|
|
129
|
+
# under spec/fixtures, fed its own .dic entries plus words.txt.
|
|
130
|
+
#
|
|
131
|
+
# @param gem_root [String] Repo root
|
|
132
|
+
# @return [Corpus]
|
|
133
|
+
def spec_fixture_corpus(gem_root: self.gem_root)
|
|
134
|
+
base = File.join(gem_root, "spec/fixtures/dictionaries/hunspell/test")
|
|
135
|
+
words = (dic_headwords("#{base}.dic") + read_word_list(File.join(gem_root, "spec/fixtures/words.txt"))).uniq
|
|
136
|
+
Corpus.new(id: "spec/fixtures/dictionaries/hunspell/test",
|
|
137
|
+
aff_path: "#{base}.aff", dic_path: "#{base}.dic", words: words)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Read a Spylls word list (.good / .wrong / plain list), mirroring
|
|
141
|
+
# SpyllsTestHelper#read_list: strip lines, drop empty lines and lines
|
|
142
|
+
# ending in ".".
|
|
143
|
+
#
|
|
144
|
+
# @param path [String] List file path
|
|
145
|
+
# @return [Array<String>] Words in file order
|
|
146
|
+
def read_word_list(path)
|
|
147
|
+
return [] unless File.file?(path)
|
|
148
|
+
|
|
149
|
+
File.read(path, encoding: "UTF-8").split("\n").filter_map do |line|
|
|
150
|
+
word = line.strip
|
|
151
|
+
next if word.empty?
|
|
152
|
+
next if word.end_with?(".")
|
|
153
|
+
|
|
154
|
+
word
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Extract headwords from a Hunspell .dic file: skip the count line,
|
|
159
|
+
# comments and morphological continuation lines; strip affix flags.
|
|
160
|
+
#
|
|
161
|
+
# @param dic_path [String] .dic file path
|
|
162
|
+
# @return [Array<String>] Headwords in file order
|
|
163
|
+
def dic_headwords(dic_path)
|
|
164
|
+
File.read(dic_path, encoding: "UTF-8").split("\n").filter_map do |line|
|
|
165
|
+
entry = line.strip
|
|
166
|
+
next if entry.empty?
|
|
167
|
+
next if entry.start_with?("#")
|
|
168
|
+
next if entry.match?(/\A\d+\z/)
|
|
169
|
+
|
|
170
|
+
entry.split("/", 2).first
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Export the default corpora to a JSONL file.
|
|
175
|
+
#
|
|
176
|
+
# @param path [String] Destination file path (directories are created)
|
|
177
|
+
# @param corpora [Array<Corpus>] Corpora to export
|
|
178
|
+
# @return [ExportResult] Row/corpus counts and anything skipped
|
|
179
|
+
def export(path:, corpora: default_corpora)
|
|
180
|
+
new(corpora).export(path)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# @param corpora [Array<Corpus>] Corpora to export (sorted by id)
|
|
185
|
+
def initialize(corpora)
|
|
186
|
+
@corpora = corpora.sort_by(&:id)
|
|
187
|
+
@skipped_corpora = {}
|
|
188
|
+
@skipped_words = {}
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Build every vector row, deterministically ordered.
|
|
192
|
+
#
|
|
193
|
+
# @return [Array<Hash>] Rows in corpus-id then fixture-word order
|
|
194
|
+
def rows
|
|
195
|
+
@corpora.each_with_object([]) do |corpus, acc|
|
|
196
|
+
checker = spellchecker_for(corpus)
|
|
197
|
+
next unless checker
|
|
198
|
+
|
|
199
|
+
corpus.words.each do |word|
|
|
200
|
+
unless word.dup.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
201
|
+
@skipped_words[[corpus.id, word]] = "not valid UTF-8"
|
|
202
|
+
next
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
acc << { kind: "correct", language: LANGUAGE, dictionary: corpus.id,
|
|
206
|
+
input: word, expected: checker.correct?(word) }
|
|
207
|
+
acc << { kind: "suggest", language: LANGUAGE, dictionary: corpus.id,
|
|
208
|
+
input: word, limit: SUGGEST_LIMIT,
|
|
209
|
+
expected: checker.suggest(word, max_suggestions: SUGGEST_LIMIT)
|
|
210
|
+
.map { |suggestion| suggestion_payload(suggestion) } }
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Build the rows and write them as JSONL to +path+.
|
|
216
|
+
#
|
|
217
|
+
# @param path [String] Destination file path (directories are created)
|
|
218
|
+
# @return [ExportResult] Row/corpus counts and anything skipped
|
|
219
|
+
def export(path)
|
|
220
|
+
all_rows = rows
|
|
221
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
222
|
+
File.write(path, all_rows.map { |row| JSON.generate(row) }.join("\n") << "\n")
|
|
223
|
+
|
|
224
|
+
ExportResult.new(path: path, row_count: all_rows.size,
|
|
225
|
+
corpus_count: @corpora.size - @skipped_corpora.size,
|
|
226
|
+
skipped_corpora: @skipped_corpora, skipped_words: @skipped_words)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
private
|
|
230
|
+
|
|
231
|
+
# Build the real engine for a corpus: a real Hunspell dictionary wrapped
|
|
232
|
+
# in a real Spellchecker. Corpus load failures are recorded and skipped
|
|
233
|
+
# rather than aborting the whole export.
|
|
234
|
+
#
|
|
235
|
+
# @param corpus [Corpus]
|
|
236
|
+
# @return [Spellchecker, nil] nil when the fixture fails to load
|
|
237
|
+
def spellchecker_for(corpus)
|
|
238
|
+
dictionary = Dictionary::Hunspell.new(dic_path: corpus.dic_path, aff_path: corpus.aff_path,
|
|
239
|
+
language_code: LANGUAGE)
|
|
240
|
+
Spellchecker.new(dictionary: dictionary)
|
|
241
|
+
rescue StandardError => e
|
|
242
|
+
@skipped_corpora[corpus.id] = "#{e.class}: #{e.message.lines.first&.strip}"
|
|
243
|
+
nil
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Reduce a Suggestions::Suggestion to the four-key payload the conformance
|
|
247
|
+
# vectors carry. Serialization itself is the framework's (lutaml-model
|
|
248
|
+
# +to_hash+); this only selects keys from its output.
|
|
249
|
+
#
|
|
250
|
+
# @param suggestion [Suggestions::Suggestion]
|
|
251
|
+
# @return [Hash] "word"/"distance"/"confidence"/"source" hash
|
|
252
|
+
def suggestion_payload(suggestion)
|
|
253
|
+
suggestion.to_hash.slice(*SUGGESTION_KEYS)
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
end
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# Replays the committed conformance vectors through one backend (or both)
|
|
7
|
+
# and reports differences — the dual-backend suite of plan 66
|
|
8
|
+
# (TODO.impl 05 P4b).
|
|
9
|
+
#
|
|
10
|
+
# The vectors (conformance/vectors.jsonl, exported by
|
|
11
|
+
# {ConformanceExporter} and committed) freeze Ruby engine behavior. This
|
|
12
|
+
# runner replays every vector through a backend and compares the actual
|
|
13
|
+
# against the frozen expectation, so:
|
|
14
|
+
#
|
|
15
|
+
# * rake kotoshu:conformance:ruby — the pure-Ruby engine must reproduce
|
|
16
|
+
# the vectors (guards against accidental behavior drift).
|
|
17
|
+
# * rake kotoshu:conformance:native — the native engine
|
|
18
|
+
# ({NativeBackend} via a real {Spellchecker} with backend "native")
|
|
19
|
+
# must reproduce them too.
|
|
20
|
+
# * rake kotoshu:conformance:compare — both engines run side by side and
|
|
21
|
+
# every row must agree: expected == ruby == native. Any difference is
|
|
22
|
+
# a backend divergence; the task exits nonzero.
|
|
23
|
+
#
|
|
24
|
+
# Everything here runs the real engines over the real fixture
|
|
25
|
+
# dictionaries — no doubles anywhere.
|
|
26
|
+
class ConformanceRunner
|
|
27
|
+
# Default vectors path: the committed file at the gem repo root.
|
|
28
|
+
#
|
|
29
|
+
# @return [String]
|
|
30
|
+
def self.default_path
|
|
31
|
+
File.join(ConformanceExporter.gem_root, "conformance", "vectors.jsonl")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# One vector whose replay disagreed with its expectation (or, in a
|
|
35
|
+
# {CompareResult}, with the other backend — then +expected+ is the Ruby
|
|
36
|
+
# actual and +actual+ the native actual).
|
|
37
|
+
Failure = Struct.new(:index, :kind, :dictionary, :input, :expected, :actual, keyword_init: true)
|
|
38
|
+
|
|
39
|
+
# Outcome of replaying the vectors through one backend.
|
|
40
|
+
RunResult = Struct.new(:backend, :row_count, :failures, :actuals, keyword_init: true) do
|
|
41
|
+
def ok?
|
|
42
|
+
failures.empty?
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Outcome of {#compare}: both runs plus the rows where the backends
|
|
47
|
+
# disagree with each other.
|
|
48
|
+
CompareResult = Struct.new(:ruby, :native, :divergences, keyword_init: true) do
|
|
49
|
+
def ok?
|
|
50
|
+
ruby.ok? && native.ok? && divergences.empty?
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @param path [String] JSONL vectors file to replay
|
|
55
|
+
def initialize(path: self.class.default_path)
|
|
56
|
+
@path = path
|
|
57
|
+
@rows = File.readlines(path, chomp: true).map { |line| JSON.parse(line) }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# @return [Integer] vector count
|
|
61
|
+
def row_count
|
|
62
|
+
@rows.size
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Replay every vector through one backend, comparing against the frozen
|
|
66
|
+
# expectations.
|
|
67
|
+
#
|
|
68
|
+
# @param backend [String, Symbol] "ruby" or "native"
|
|
69
|
+
# @return [RunResult]
|
|
70
|
+
# @raise [Native::Unavailable] backend "native" and the extension is
|
|
71
|
+
# not built
|
|
72
|
+
def run(backend:)
|
|
73
|
+
backend_name = backend.to_s
|
|
74
|
+
if backend_name == "native" && !Kotoshu::Native.available?
|
|
75
|
+
raise Native::Unavailable,
|
|
76
|
+
"native backend requested but the extension is not built " \
|
|
77
|
+
"(rake compile) — cannot run the native conformance suite"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
failures = []
|
|
81
|
+
engines = {}
|
|
82
|
+
actuals = @rows.each_with_index.map do |row, index|
|
|
83
|
+
actual = replay(engine_for(row, backend_name, engines), row)
|
|
84
|
+
if actual != row["expected"]
|
|
85
|
+
failures << Failure.new(index: index, kind: row["kind"], dictionary: row["dictionary"],
|
|
86
|
+
input: row["input"], expected: row["expected"], actual: actual)
|
|
87
|
+
end
|
|
88
|
+
actual
|
|
89
|
+
end
|
|
90
|
+
RunResult.new(backend: backend_name, row_count: @rows.size, failures: failures, actuals: actuals)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Run both backends and cross-check every row.
|
|
94
|
+
#
|
|
95
|
+
# @return [CompareResult]
|
|
96
|
+
# @raise [Native::Unavailable] the extension is not built
|
|
97
|
+
def compare
|
|
98
|
+
ruby_result = run(backend: :ruby)
|
|
99
|
+
native_result = run(backend: :native)
|
|
100
|
+
|
|
101
|
+
divergences = @rows.each_index.filter_map do |index|
|
|
102
|
+
ruby_actual = ruby_result.actuals[index]
|
|
103
|
+
native_actual = native_result.actuals[index]
|
|
104
|
+
next if ruby_actual == native_actual
|
|
105
|
+
|
|
106
|
+
row = @rows[index]
|
|
107
|
+
Failure.new(index: index, kind: row["kind"], dictionary: row["dictionary"],
|
|
108
|
+
input: row["input"], expected: ruby_actual, actual: native_actual)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
CompareResult.new(ruby: ruby_result, native: native_result, divergences: divergences)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
|
|
116
|
+
# The engine for a row's dictionary, built once per dictionary.
|
|
117
|
+
#
|
|
118
|
+
# @param row [Hash] vector row
|
|
119
|
+
# @param backend [String] "ruby" or "native"
|
|
120
|
+
# @param engines [Hash] cache keyed by dictionary id
|
|
121
|
+
# @return [Spellchecker, String] the engine, or an error marker for
|
|
122
|
+
# dictionaries that fail to load
|
|
123
|
+
def engine_for(row, backend, engines)
|
|
124
|
+
dictionary_id = row["dictionary"]
|
|
125
|
+
engines[dictionary_id] ||= build_engine(dictionary_id, backend)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Build the real engine for a dictionary id from the vectors file: a
|
|
129
|
+
# real {Dictionary::Hunspell} over the fixture files, wrapped in a real
|
|
130
|
+
# {Spellchecker} pinned to the requested backend (pinning overrides any
|
|
131
|
+
# KOTOSHU_BACKEND env so the task measures what it names).
|
|
132
|
+
#
|
|
133
|
+
# @param dictionary_id [String] dictionary path relative to gem root
|
|
134
|
+
# @param backend [String] "ruby" or "native"
|
|
135
|
+
# @return [Spellchecker, String] the engine or a failure description
|
|
136
|
+
def build_engine(dictionary_id, backend)
|
|
137
|
+
base = File.join(ConformanceExporter.gem_root, dictionary_id)
|
|
138
|
+
dictionary = Dictionary::Hunspell.new(
|
|
139
|
+
dic_path: "#{base}.dic", aff_path: "#{base}.aff",
|
|
140
|
+
language_code: ConformanceExporter::LANGUAGE
|
|
141
|
+
)
|
|
142
|
+
Spellchecker.new(dictionary: dictionary, config: Configuration.new(backend: backend))
|
|
143
|
+
rescue StandardError => e
|
|
144
|
+
"#{e.class}: #{e.message.lines.first&.strip}"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Replay one vector through an engine.
|
|
148
|
+
#
|
|
149
|
+
# @param engine [Spellchecker, String] engine or load-failure marker
|
|
150
|
+
# @param row [Hash] vector row
|
|
151
|
+
# @return [Boolean, Array<Hash>] the actual result
|
|
152
|
+
def replay(engine, row)
|
|
153
|
+
return "dictionary failed to load: #{engine}" if engine.is_a?(String)
|
|
154
|
+
|
|
155
|
+
case row["kind"]
|
|
156
|
+
when "correct"
|
|
157
|
+
engine.correct?(row["input"])
|
|
158
|
+
when "suggest"
|
|
159
|
+
engine.suggest(row["input"], max_suggestions: row["limit"])
|
|
160
|
+
.map { |suggestion| suggestion_payload(suggestion) }
|
|
161
|
+
else
|
|
162
|
+
"unknown vector kind: #{row['kind'].inspect}"
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Reduce a Suggestion to the four-key conformance payload — the same
|
|
167
|
+
# selection {ConformanceExporter} applies (lutaml-model +to_hash+,
|
|
168
|
+
# slice to SUGGESTION_KEYS).
|
|
169
|
+
#
|
|
170
|
+
# @param suggestion [Suggestions::Suggestion]
|
|
171
|
+
# @return [Hash]
|
|
172
|
+
def suggestion_payload(suggestion)
|
|
173
|
+
suggestion.to_hash.slice(*ConformanceExporter::SUGGESTION_KEYS)
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Base error class for all Kotoshu exceptions.
|
|
5
|
+
#
|
|
6
|
+
# @example Raising a custom Kotoshu error
|
|
7
|
+
# raise Kotoshu::Error, "Something went wrong"
|
|
8
|
+
class Error < StandardError; end
|
|
9
|
+
|
|
10
|
+
# Error raised when a dictionary file cannot be found.
|
|
11
|
+
#
|
|
12
|
+
# @example Dictionary not found
|
|
13
|
+
# raise DictionaryNotFoundError, "Dictionary not found: /path/to/dic.dic"
|
|
14
|
+
class DictionaryNotFoundError < Error
|
|
15
|
+
# Create a new dictionary not found error.
|
|
16
|
+
#
|
|
17
|
+
# @param path [String] The path that was not found
|
|
18
|
+
# @param message [String] Custom message (optional)
|
|
19
|
+
def initialize(path, message = nil)
|
|
20
|
+
@path = path
|
|
21
|
+
super(message || "Dictionary not found: #{path}")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# @return [String] The path that was not found
|
|
25
|
+
attr_reader :path
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Error raised when a dictionary file has an invalid format.
|
|
29
|
+
#
|
|
30
|
+
# @example Invalid dictionary format
|
|
31
|
+
# raise InvalidDictionaryFormatError, "Invalid .dic file format"
|
|
32
|
+
class InvalidDictionaryFormatError < Error
|
|
33
|
+
# Create a new invalid format error.
|
|
34
|
+
#
|
|
35
|
+
# @param path [String] The file path
|
|
36
|
+
# @param details [String] Details about the format issue
|
|
37
|
+
def initialize(path, details = nil)
|
|
38
|
+
@path = path
|
|
39
|
+
@details = details
|
|
40
|
+
super("Invalid dictionary format#{": #{details}" if details}: #{path}")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @return [String] The file path
|
|
44
|
+
attr_reader :path
|
|
45
|
+
|
|
46
|
+
# @return [String, nil] Details about the format issue
|
|
47
|
+
attr_reader :details
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Error raised when there is a configuration issue.
|
|
51
|
+
#
|
|
52
|
+
# @example Invalid configuration
|
|
53
|
+
# raise ConfigurationError, "Invalid dictionary type: unknown_type"
|
|
54
|
+
class ConfigurationError < Error
|
|
55
|
+
# Create a new configuration error.
|
|
56
|
+
#
|
|
57
|
+
# @param message [String] The error message
|
|
58
|
+
# @param key [String, Symbol] The configuration key (optional)
|
|
59
|
+
def initialize(message, key: nil)
|
|
60
|
+
@key = key
|
|
61
|
+
super(message)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return [String, Symbol, nil] The configuration key
|
|
65
|
+
attr_reader :key
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Error raised during spell checking operations.
|
|
69
|
+
#
|
|
70
|
+
# @example Spell check failure
|
|
71
|
+
# raise SpellcheckError, "Failed to check word: encoding error"
|
|
72
|
+
class SpellcheckError < Error
|
|
73
|
+
# Create a new spellcheck error.
|
|
74
|
+
#
|
|
75
|
+
# @param message [String] The error message
|
|
76
|
+
# @param word [String] The word being checked (optional)
|
|
77
|
+
def initialize(message, word: nil)
|
|
78
|
+
@word = word
|
|
79
|
+
super(message)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# @return [String, nil] The word being checked
|
|
83
|
+
attr_reader :word
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Error raised when an affix rule cannot be parsed.
|
|
87
|
+
#
|
|
88
|
+
# @example Invalid affix rule
|
|
89
|
+
# raise AffixRuleError, "Invalid affix rule: PFX A Y 1 re"
|
|
90
|
+
class AffixRuleError < Error
|
|
91
|
+
# Create a new affix rule error.
|
|
92
|
+
#
|
|
93
|
+
# @param message [String] The error message
|
|
94
|
+
# @param rule [String] The rule that failed to parse (optional)
|
|
95
|
+
def initialize(message, rule: nil)
|
|
96
|
+
@rule = rule
|
|
97
|
+
super(message)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# @return [String, nil] The rule that failed to parse
|
|
101
|
+
attr_reader :rule
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Error raised when a required resource is not cached and cannot be
|
|
105
|
+
# downloaded (offline mode or network failure).
|
|
106
|
+
class ResourceNotCachedError < Error
|
|
107
|
+
def initialize(language, resource_type)
|
|
108
|
+
@language = language
|
|
109
|
+
@resource_type = resource_type
|
|
110
|
+
super("Resource not cached: #{language}:#{resource_type}. " \
|
|
111
|
+
"Pre-fetch with `kotoshu cache download language #{language}` " \
|
|
112
|
+
"or disable offline mode (KOTOSHU_OFFLINE=0).")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
attr_reader :language, :resource_type
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Error raised by the hot path (Kotoshu.correct?, .suggest, .check,
|
|
119
|
+
# .check_file, .spellchecker_for) when a language hasn't been set up
|
|
120
|
+
# via Kotoshu.setup / kotoshu setup. The hot path is cache-only and
|
|
121
|
+
# never downloads — explicit setup is required.
|
|
122
|
+
class ResourceNotSetupError < Error
|
|
123
|
+
def initialize(language, resource_type = "spelling")
|
|
124
|
+
@language = language
|
|
125
|
+
@resource_type = resource_type
|
|
126
|
+
super("Language '#{language}' is not set up (missing #{resource_type}). " \
|
|
127
|
+
"Run `kotoshu setup #{language}` or " \
|
|
128
|
+
"`Kotoshu.setup(:#{language})` first.")
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
attr_reader :language, :resource_type
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Error raised when a resource cannot be resolved for a language
|
|
135
|
+
# (unsupported language, download failure, etc.).
|
|
136
|
+
class ResourceResolutionError < Error
|
|
137
|
+
def initialize(language, reason)
|
|
138
|
+
@language = language
|
|
139
|
+
super("Cannot resolve resources for '#{language}': #{reason}")
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
attr_reader :language
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Error raised when a downloaded resource fails integrity verification
|
|
146
|
+
# (SHA-256 mismatch against manifest, truncated content, etc.).
|
|
147
|
+
#
|
|
148
|
+
# The downloaded bytes are never trusted until verified against a known
|
|
149
|
+
# manifest entry. Mismatch raises this error with both hashes so the
|
|
150
|
+
# caller can surface them in audit logs and CI output.
|
|
151
|
+
#
|
|
152
|
+
# The optional +remediation:+ hint is appended to the message when the
|
|
153
|
+
# caller knows the user-facing fix (e.g. "Run `kotoshu cache download
|
|
154
|
+
# :en --model` to re-download"). Existing callers that omit it see no
|
|
155
|
+
# change in the message format.
|
|
156
|
+
class IntegrityError < Error
|
|
157
|
+
def initialize(resource_id, expected:, actual:, url: nil, remediation: nil)
|
|
158
|
+
@resource_id = resource_id
|
|
159
|
+
@expected = expected
|
|
160
|
+
@actual = actual
|
|
161
|
+
@url = url
|
|
162
|
+
@remediation = remediation
|
|
163
|
+
msg = "Integrity verification failed for #{resource_id}: "
|
|
164
|
+
msg << "expected sha256=#{expected}, got sha256=#{actual}"
|
|
165
|
+
msg << " (url: #{url})" if url
|
|
166
|
+
msg << ". #{remediation}" if remediation
|
|
167
|
+
super(msg)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
attr_reader :resource_id, :expected, :actual, :url, :remediation
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Error raised when Japanese-language features are requested but the
|
|
174
|
+
# suika gem is unavailable. suika is a soft runtime dependency because
|
|
175
|
+
# its native extension (dartsclone) does not build on slim/minimal
|
|
176
|
+
# environments. All non-Japanese code paths are unaffected.
|
|
177
|
+
class SuikaUnavailable < Error
|
|
178
|
+
def initialize(detail = nil)
|
|
179
|
+
message = "suika gem not loaded"
|
|
180
|
+
message += " (#{detail})" if detail
|
|
181
|
+
message += ". Install with: gem install suika"
|
|
182
|
+
message += ". Or set KOTOSHU_NO_SUIKA=1 to silence this in code paths that opt out."
|
|
183
|
+
super(message)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|