kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Algorithms namespace for spell checking algorithms.
|
|
5
|
+
#
|
|
6
|
+
# Contains the core algorithms ported from Spylls:
|
|
7
|
+
# - Lookup::Lookuper: word correctness checking with affix support
|
|
8
|
+
# - Suggest::Suggester: main suggestion orchestration
|
|
9
|
+
# - Permutations: edit-distance permutations
|
|
10
|
+
# - NgramSuggest: n-gram based suggestion algorithm
|
|
11
|
+
# - PhonetSuggest: phonetic suggestion algorithm
|
|
12
|
+
# - Capitalization: capitalization handling
|
|
13
|
+
module Algorithms
|
|
14
|
+
autoload :Lookup, "kotoshu/algorithms/lookup"
|
|
15
|
+
autoload :Suggest, "kotoshu/algorithms/suggest"
|
|
16
|
+
autoload :Permutations, "kotoshu/algorithms/permutations"
|
|
17
|
+
autoload :NgramSuggest, "kotoshu/algorithms/ngram_suggest"
|
|
18
|
+
autoload :PhonetSuggest, "kotoshu/algorithms/phonet_suggest"
|
|
19
|
+
autoload :Capitalization, "kotoshu/algorithms/capitalization"
|
|
20
|
+
autoload :EditDistance, "kotoshu/algorithms/edit_distance"
|
|
21
|
+
autoload :Soundex, "kotoshu/algorithms/soundex"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Analyzers
|
|
5
|
+
# Unified semantic error analyzer.
|
|
6
|
+
#
|
|
7
|
+
# Uses word embeddings for context-aware error detection and suggestions.
|
|
8
|
+
# Provides unified semantic analysis without artificial spelling/grammar split.
|
|
9
|
+
#
|
|
10
|
+
# @example Analyzing a document
|
|
11
|
+
# model = FastTextModel.from_github('en')
|
|
12
|
+
# analyzer = SemanticAnalyzer.new(model)
|
|
13
|
+
# errors = analyzer.analyze(document)
|
|
14
|
+
#
|
|
15
|
+
# @example Checking a single word
|
|
16
|
+
# suggestions = analyzer.suggest_corrections('helo', context_words: ['hello', 'world'])
|
|
17
|
+
class SemanticAnalyzer
|
|
18
|
+
# Similarity threshold for high-confidence suggestions
|
|
19
|
+
HIGH_CONFIDENCE_THRESHOLD = 0.85
|
|
20
|
+
|
|
21
|
+
# Similarity threshold for medium-confidence suggestions
|
|
22
|
+
MEDIUM_CONFIDENCE_THRESHOLD = 0.70
|
|
23
|
+
|
|
24
|
+
# Minimum similarity for suggestions
|
|
25
|
+
MIN_SIMILARITY = 0.50
|
|
26
|
+
|
|
27
|
+
# Default number of suggestions to generate
|
|
28
|
+
DEFAULT_MAX_SUGGESTIONS = 5
|
|
29
|
+
|
|
30
|
+
attr_reader :model, :max_suggestions
|
|
31
|
+
|
|
32
|
+
# Create a new semantic analyzer.
|
|
33
|
+
#
|
|
34
|
+
# @param model [EmbeddingModel] The embedding model to use
|
|
35
|
+
# @param max_suggestions [Integer] Maximum suggestions per error
|
|
36
|
+
# @param min_similarity [Float] Minimum similarity threshold
|
|
37
|
+
def initialize(model, max_suggestions: DEFAULT_MAX_SUGGESTIONS, min_similarity: MIN_SIMILARITY)
|
|
38
|
+
raise ArgumentError, "Model must be an EmbeddingModel" unless model.is_a?(Models::EmbeddingModel)
|
|
39
|
+
|
|
40
|
+
@model = model
|
|
41
|
+
@max_suggestions = max_suggestions
|
|
42
|
+
@min_similarity = min_similarity
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Analyze a {Kotoshu::Documents::Document} for semantic errors.
|
|
46
|
+
#
|
|
47
|
+
# Walks every {Documents::TextNode}, tokenizes its text, and for
|
|
48
|
+
# each invalid word resolves a {Documents::SourceRange} via
|
|
49
|
+
# `document.source_range_for` so the emitted {Models::SemanticError}
|
|
50
|
+
# points at the original markup-bearing source rather than the
|
|
51
|
+
# flattened text. Context for ranking is built from the
|
|
52
|
+
# surrounding flattened text.
|
|
53
|
+
#
|
|
54
|
+
# @param document [Kotoshu::Documents::Document]
|
|
55
|
+
# @return [Array<Models::SemanticError>] Sorted by source_range
|
|
56
|
+
def analyze(document)
|
|
57
|
+
unless document.is_a?(Kotoshu::Documents::Document)
|
|
58
|
+
raise ArgumentError,
|
|
59
|
+
"document must be a Kotoshu::Documents::Document"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
errors = []
|
|
63
|
+
flattened = document.flattened_text
|
|
64
|
+
|
|
65
|
+
document.text_nodes.each do |text_node|
|
|
66
|
+
tokenize_with_offsets(text_node.text).each do |word, offset_in_node|
|
|
67
|
+
next if valid_word?(word)
|
|
68
|
+
|
|
69
|
+
flattened_start = text_node.flattened_offset + offset_in_node
|
|
70
|
+
flattened_end = flattened_start + word.length
|
|
71
|
+
source_range = document.source_range_for(flattened_start, flattened_end)
|
|
72
|
+
context = build_context(flattened, flattened_start, flattened_end)
|
|
73
|
+
|
|
74
|
+
error = detect_error(word: word, source_range: source_range, context: context)
|
|
75
|
+
errors << error if error
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
errors.sort
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Detect semantic error for a single word.
|
|
83
|
+
#
|
|
84
|
+
# @param word [String] The word to check
|
|
85
|
+
# @param source_range [Kotoshu::Documents::SourceRange, nil] Where
|
|
86
|
+
# the word lives in the original source. May be nil for
|
|
87
|
+
# word-level checks that aren't tied to a document.
|
|
88
|
+
# @param context [Models::Context, nil] Context around the word
|
|
89
|
+
# @return [Models::SemanticError, nil] Error object or nil if valid
|
|
90
|
+
def detect_error(word:, source_range: nil, context: nil)
|
|
91
|
+
return nil if valid_word?(word)
|
|
92
|
+
|
|
93
|
+
suggestions = suggest_corrections(word, context: context)
|
|
94
|
+
error_type = classify_error(word, suggestions, context)
|
|
95
|
+
confidence = calculate_confidence(suggestions)
|
|
96
|
+
|
|
97
|
+
Models::SemanticError.new(
|
|
98
|
+
id: generate_error_id(word, source_range),
|
|
99
|
+
source_range: source_range,
|
|
100
|
+
original: word,
|
|
101
|
+
suggestions: suggestions,
|
|
102
|
+
error_type: error_type,
|
|
103
|
+
confidence: confidence,
|
|
104
|
+
context: context
|
|
105
|
+
)
|
|
106
|
+
rescue Models::EmptySuggestionsError
|
|
107
|
+
# Word is genuinely unknown — no close matches. Skip silently
|
|
108
|
+
# rather than crashing on the suggestions-cannot-be-empty
|
|
109
|
+
# invariant.
|
|
110
|
+
nil
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Suggest corrections for a word.
|
|
114
|
+
#
|
|
115
|
+
# For in-vocabulary words the embedding model returns the
|
|
116
|
+
# nearest neighbors. For OOV words (the typical "misspelling"
|
|
117
|
+
# case the analyzer exists to catch) the embedding model returns
|
|
118
|
+
# +[]+ because it has no vector for the input. We fall back to
|
|
119
|
+
# an edit-distance walk over the model's vocabulary so the OOV
|
|
120
|
+
# case still produces useful candidates that the rest of the
|
|
121
|
+
# pipeline (confidence scoring, context ranking) can refine.
|
|
122
|
+
#
|
|
123
|
+
# @param word [String] The misspelled word
|
|
124
|
+
# @param context [Models::Context, nil] Context for context-aware suggestions
|
|
125
|
+
# @return [Array<Models::Suggestion>] Suggested corrections
|
|
126
|
+
def suggest_corrections(word, context: nil)
|
|
127
|
+
return [] if word.nil? || word.empty?
|
|
128
|
+
|
|
129
|
+
neighbors = @model.nearest_neighbors(word, k: @max_suggestions * 3)
|
|
130
|
+
neighbors = edit_distance_fallback(word) if neighbors.empty?
|
|
131
|
+
|
|
132
|
+
# Filter by minimum similarity
|
|
133
|
+
neighbors = neighbors.select { |n| n.similarity >= @min_similarity }
|
|
134
|
+
|
|
135
|
+
# If we have context, rank by contextual relevance
|
|
136
|
+
if context.is_a?(Kotoshu::Models::Context)
|
|
137
|
+
neighbors = rank_by_context(neighbors, context)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Convert to Suggestions
|
|
141
|
+
neighbors.first(@max_suggestions).map do |neighbor|
|
|
142
|
+
Models::Suggestion.new(
|
|
143
|
+
neighbor.word,
|
|
144
|
+
confidence: neighbor.similarity,
|
|
145
|
+
source: :semantic,
|
|
146
|
+
metadata: {
|
|
147
|
+
distance: neighbor.distance,
|
|
148
|
+
similarity: neighbor.similarity
|
|
149
|
+
}
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Check if a word is valid (exists in vocabulary).
|
|
155
|
+
#
|
|
156
|
+
# @param word [String] The word to check
|
|
157
|
+
# @return [Boolean] True if word is valid
|
|
158
|
+
def valid_word?(word)
|
|
159
|
+
return false if word.nil? || word.empty?
|
|
160
|
+
|
|
161
|
+
# Skip numbers
|
|
162
|
+
return true if /^\d+$/.match?(word)
|
|
163
|
+
|
|
164
|
+
# Skip single characters (likely abbreviations)
|
|
165
|
+
return true if word.length == 1
|
|
166
|
+
|
|
167
|
+
# Check if word exists in model vocabulary
|
|
168
|
+
@model.has_word?(word)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Calculate confidence score for suggestions.
|
|
172
|
+
#
|
|
173
|
+
# @param suggestions [Array<Models::Suggestion>] List of suggestions
|
|
174
|
+
# @return [Float] Confidence score (0.0 to 1.0)
|
|
175
|
+
def calculate_confidence(suggestions)
|
|
176
|
+
return 0.0 unless suggestions&.any?
|
|
177
|
+
|
|
178
|
+
# Confidence is based on top suggestion quality
|
|
179
|
+
top = suggestions.first
|
|
180
|
+
|
|
181
|
+
# High confidence: top suggestion > 0.85 similarity
|
|
182
|
+
return 1.0 if top.confidence > HIGH_CONFIDENCE_THRESHOLD
|
|
183
|
+
|
|
184
|
+
# Medium confidence: top suggestion > 0.70 similarity
|
|
185
|
+
return 0.7 if top.confidence > MEDIUM_CONFIDENCE_THRESHOLD
|
|
186
|
+
|
|
187
|
+
# Low confidence: top suggestion < 0.70
|
|
188
|
+
0.5
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
private
|
|
192
|
+
|
|
193
|
+
# Fallback for OOV words: walk the model's vocabulary and return
|
|
194
|
+
# candidates within edit distance 2. Scored by 1/(1+distance) so
|
|
195
|
+
# distance-1 matches rank above distance-2 matches. Mirrors the
|
|
196
|
+
# shape {EmbeddingModel#nearest_neighbors} returns
|
|
197
|
+
# (+NearestNeighbor+ instances with +similarity+ in [0, 1]).
|
|
198
|
+
#
|
|
199
|
+
# @param word [String] OOV query word
|
|
200
|
+
# @return [Array<Models::NearestNeighbor>]
|
|
201
|
+
def edit_distance_fallback(word)
|
|
202
|
+
vocab = @model.vocabulary
|
|
203
|
+
return [] unless vocab&.any?
|
|
204
|
+
|
|
205
|
+
downcased = word.downcase
|
|
206
|
+
candidates = vocab.each_with_object([]) do |candidate, acc|
|
|
207
|
+
next if candidate == word || candidate.downcase == downcased
|
|
208
|
+
|
|
209
|
+
distance = levenshtein(downcased, candidate.downcase)
|
|
210
|
+
next unless distance.positive? && distance <= 2
|
|
211
|
+
|
|
212
|
+
acc << Models::NearestNeighbor.new(
|
|
213
|
+
word: candidate,
|
|
214
|
+
similarity: 1.0 / (1.0 + distance),
|
|
215
|
+
distance: distance,
|
|
216
|
+
embedding: nil
|
|
217
|
+
)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
candidates.sort.first(@max_suggestions * 3)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Two-row Levenshtein edit distance.
|
|
224
|
+
def levenshtein(a, b)
|
|
225
|
+
return b.length if a.empty?
|
|
226
|
+
return a.length if b.empty?
|
|
227
|
+
|
|
228
|
+
prev = (0..b.length).to_a
|
|
229
|
+
a.each_char.with_index do |achar, i|
|
|
230
|
+
curr = [i + 1]
|
|
231
|
+
b.each_char.with_index do |bchar, j|
|
|
232
|
+
cost = achar == bchar ? 0 : 1
|
|
233
|
+
curr << [curr[j] + 1, prev[j + 1] + 1, prev[j] + cost].min
|
|
234
|
+
end
|
|
235
|
+
prev = curr
|
|
236
|
+
end
|
|
237
|
+
prev.last
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Tokenize text into [word, offset_in_text] pairs. The offset
|
|
241
|
+
# is the 0-based character position of the word's first char
|
|
242
|
+
# within +text+. Used by {#analyze} to compute flattened offsets
|
|
243
|
+
# for source-range resolution.
|
|
244
|
+
#
|
|
245
|
+
# @param text [String]
|
|
246
|
+
# @return [Array<Array(String, Integer)>]
|
|
247
|
+
def tokenize_with_offsets(text)
|
|
248
|
+
return [] unless text
|
|
249
|
+
|
|
250
|
+
text.downcase.scan(/[a-z]+(?:['’-][a-z]+)*/i).map do |match|
|
|
251
|
+
char_offset = Regexp.last_match.offset(0).first
|
|
252
|
+
[match.downcase, char_offset]
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Tokenize text into words. Kept for backward compat with specs.
|
|
257
|
+
#
|
|
258
|
+
# @param text [String] Text to tokenize
|
|
259
|
+
# @return [Array<String>] Words
|
|
260
|
+
def tokenize_words(text)
|
|
261
|
+
tokenize_with_offsets(text).map(&:first)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Build a {Models::Context} around a flattened-text range.
|
|
265
|
+
# Slices up to 32 chars before and after the [start, end) range,
|
|
266
|
+
# rounding at whitespace so the context reads naturally.
|
|
267
|
+
#
|
|
268
|
+
# @param flattened [String]
|
|
269
|
+
# @param flattened_start [Integer]
|
|
270
|
+
# @param flattened_end [Integer]
|
|
271
|
+
# @return [Models::Context, nil] nil when the slice is empty
|
|
272
|
+
def build_context(flattened, flattened_start, flattened_end)
|
|
273
|
+
return nil if flattened.nil? || flattened.empty?
|
|
274
|
+
|
|
275
|
+
window = 32
|
|
276
|
+
before_start = [flattened_start - window, 0].max
|
|
277
|
+
after_end = [flattened_end + window, flattened.length].min
|
|
278
|
+
before = flattened[before_start...flattened_start] || ""
|
|
279
|
+
current = flattened[flattened_start...flattened_end] || ""
|
|
280
|
+
after = flattened[flattened_end...after_end] || ""
|
|
281
|
+
return nil if current.empty?
|
|
282
|
+
|
|
283
|
+
Models::Context.new(
|
|
284
|
+
before: before,
|
|
285
|
+
current: current,
|
|
286
|
+
after: after,
|
|
287
|
+
location: nil
|
|
288
|
+
)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Classify error type based on word and suggestions.
|
|
292
|
+
#
|
|
293
|
+
# @param word [String] The error word
|
|
294
|
+
# @param suggestions [Array<Models::Suggestion>] Suggestions
|
|
295
|
+
# @param _context [Models::Context, nil] Context (unused)
|
|
296
|
+
# @return [Symbol] Error type
|
|
297
|
+
def classify_error(word, suggestions, _context)
|
|
298
|
+
return :orthographic if suggestions && suggestions.empty?
|
|
299
|
+
|
|
300
|
+
top_suggestion = suggestions.first
|
|
301
|
+
|
|
302
|
+
# Check if it's a capitalization error
|
|
303
|
+
if word.downcase == top_suggestion.word.downcase
|
|
304
|
+
return :capitalization
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Check if it's a diacritic/accent error
|
|
308
|
+
if similar_without_diacritics?(word, top_suggestion.word)
|
|
309
|
+
return :orthographic
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Check if it's a word choice error (semantic similarity but different word)
|
|
313
|
+
if suggestions.first&.source == "semantic"
|
|
314
|
+
return :word_choice
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Default to orthographic (spelling)
|
|
318
|
+
:orthographic
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# Check if two words are similar ignoring diacritics.
|
|
322
|
+
#
|
|
323
|
+
# @param word1 [String] First word
|
|
324
|
+
# @param word2 [String] Second word
|
|
325
|
+
# @return [Boolean] True if similar without diacritics
|
|
326
|
+
def similar_without_diacritics?(word1, word2)
|
|
327
|
+
# Remove diacritics and compare
|
|
328
|
+
normalize_diacritics(word1) == normalize_diacritics(word2)
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
# Normalize diacritics from a word.
|
|
332
|
+
#
|
|
333
|
+
# @param word [String] Word with diacritics
|
|
334
|
+
# @return [String] Word without diacritics
|
|
335
|
+
def normalize_diacritics(word)
|
|
336
|
+
# Simple normalization (transliterate to ASCII)
|
|
337
|
+
word.encode('ASCII', fallback: ->(c) {
|
|
338
|
+
if c == 'ä'
|
|
339
|
+
'ae'
|
|
340
|
+
elsif c == 'ö'
|
|
341
|
+
'oe'
|
|
342
|
+
elsif c == 'ü'
|
|
343
|
+
'ue'
|
|
344
|
+
else
|
|
345
|
+
c == 'ß' ? 'ss' : c
|
|
346
|
+
end
|
|
347
|
+
})
|
|
348
|
+
.downcase
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Rank neighbors by contextual relevance.
|
|
352
|
+
#
|
|
353
|
+
# @param neighbors [Array<Models::NearestNeighbor>] Neighbors to rank
|
|
354
|
+
# @param context [Models::Context] Context for ranking
|
|
355
|
+
# @return [Array<Models::NearestNeighbor>] Ranked neighbors
|
|
356
|
+
def rank_by_context(neighbors, context)
|
|
357
|
+
# Get surrounding words
|
|
358
|
+
surrounding = context.surrounding_words(3)
|
|
359
|
+
return neighbors unless surrounding&.any?
|
|
360
|
+
|
|
361
|
+
# Boost neighbors that appear in similar context
|
|
362
|
+
# In full implementation, would use more sophisticated context modeling
|
|
363
|
+
neighbors.map do |neighbor|
|
|
364
|
+
boost = context_boost(neighbor.word, surrounding)
|
|
365
|
+
# Create boosted neighbor (create new object to avoid mutation)
|
|
366
|
+
boosted_similarity = [neighbor.similarity + boost, 1.0].min
|
|
367
|
+
Models::NearestNeighbor.new(
|
|
368
|
+
word: neighbor.word,
|
|
369
|
+
similarity: boosted_similarity,
|
|
370
|
+
embedding: neighbor.embedding
|
|
371
|
+
)
|
|
372
|
+
end.sort.reverse
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# Calculate context boost for a word.
|
|
376
|
+
#
|
|
377
|
+
# @param word [String] Word to boost
|
|
378
|
+
# @param surrounding [Array<String>] Surrounding words
|
|
379
|
+
# @return [Float] Boost amount (0.0 to 0.1)
|
|
380
|
+
def context_boost(word, surrounding)
|
|
381
|
+
return 0.0 unless surrounding&.any?
|
|
382
|
+
|
|
383
|
+
# Simple boost: if word is semantically similar to surrounding words
|
|
384
|
+
surrounding.reduce(0.0) do |boost, surrounding_word|
|
|
385
|
+
sim = @model.similarity(word, surrounding_word)
|
|
386
|
+
boost + ((sim || 0.0) * 0.02) # Small boost for each similar word
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# Generate unique error ID.
|
|
391
|
+
#
|
|
392
|
+
# @param word [String] The error word
|
|
393
|
+
# @param source_range [Kotoshu::Documents::SourceRange, nil]
|
|
394
|
+
# @return [String] Truncated SHA-256 hex
|
|
395
|
+
def generate_error_id(word, source_range)
|
|
396
|
+
key = if source_range
|
|
397
|
+
"#{word}-#{source_range.start.offset}-#{source_range.end.offset}"
|
|
398
|
+
else
|
|
399
|
+
"word:#{word}"
|
|
400
|
+
end
|
|
401
|
+
Digest::SHA256.hexdigest(key)[0...16]
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
end
|
|
405
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Baseline
|
|
5
|
+
# Outcome of applying a {Store} to one check result. Plain data
|
|
6
|
+
# (Struct) — not a serialized model.
|
|
7
|
+
#
|
|
8
|
+
# - +result+ — the new {Models::Result::DocumentResult}: errors
|
|
9
|
+
# are the ones NOT covered by the baseline; baseline-suppressed
|
|
10
|
+
# errors moved into `suppressed_errors` (marked with
|
|
11
|
+
# `suppressed_by: "baseline"`), joining any inline-suppressed
|
|
12
|
+
# entries produced by the checker.
|
|
13
|
+
# - +suppressed_count+ — how many errors the baseline absorbed
|
|
14
|
+
# - +stale_count+ — baseline entries for the checked file whose
|
|
15
|
+
# errors no longer exist
|
|
16
|
+
Application = Struct.new(:result, :suppressed_count, :stale_count, keyword_init: true)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Baseline
|
|
7
|
+
# One baseline record: +count+ occurrences of +word+ in +file+ are
|
|
8
|
+
# known debt. +line+ is the first occurrence at the time the
|
|
9
|
+
# baseline was written — informational only, matching is
|
|
10
|
+
# count-based (stable across reformatting).
|
|
11
|
+
#
|
|
12
|
+
# Serialized via lutaml-model; the canonical JSON shape is
|
|
13
|
+
# `{"file": ..., "line": ..., "word": ..., "count": ...}`.
|
|
14
|
+
class Entry < Lutaml::Model::Serializable
|
|
15
|
+
attribute :file, :string
|
|
16
|
+
attribute :line, :integer, default: 1
|
|
17
|
+
attribute :word, :string, default: ""
|
|
18
|
+
attribute :count, :integer, default: 1
|
|
19
|
+
|
|
20
|
+
# Accept framework-private keywords (e.g. +lutaml_register+)
|
|
21
|
+
# from lutaml-model so from_hash round-trips without lutaml
|
|
22
|
+
# having to poke ivars directly.
|
|
23
|
+
def initialize(file: nil, line: 1, word: "", count: 1, **kwargs)
|
|
24
|
+
super(file: file, line: line, word: word.to_s, count: count, **kwargs)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Entries are keyed by (file, word); counts merge on key.
|
|
28
|
+
def key
|
|
29
|
+
[file.to_s, word.to_s]
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
require "set"
|
|
5
|
+
|
|
6
|
+
module Kotoshu
|
|
7
|
+
module Baseline
|
|
8
|
+
# A baseline file: canonical JSON `{version, entries: [{file,
|
|
9
|
+
# line, word, count}]}`. Serialized via lutaml-model.
|
|
10
|
+
#
|
|
11
|
+
# Semantics when applied to a check result (plan 82):
|
|
12
|
+
# - an error passes when the baseline has budget left for it
|
|
13
|
+
# (same file + word, occurrence count not exceeded);
|
|
14
|
+
# - errors beyond the budget, and unknown words, fail;
|
|
15
|
+
# - a summary reports how many entries are stale (the file no
|
|
16
|
+
# longer has the error), so debt shrinks visibly.
|
|
17
|
+
class Store < Lutaml::Model::Serializable
|
|
18
|
+
DEFAULT_FILENAME = ".kotoshu-baseline.json"
|
|
19
|
+
CURRENT_VERSION = 1
|
|
20
|
+
|
|
21
|
+
attribute :version, :integer, default: CURRENT_VERSION
|
|
22
|
+
attribute :entries, Entry, collection: true
|
|
23
|
+
|
|
24
|
+
# Accept framework-private keywords from lutaml-model.
|
|
25
|
+
def initialize(version: CURRENT_VERSION, entries: [], **kwargs)
|
|
26
|
+
super
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
# Read and parse a baseline file.
|
|
31
|
+
#
|
|
32
|
+
# @param path [String]
|
|
33
|
+
# @return [Store]
|
|
34
|
+
# @raise [Errno::ENOENT, Lutaml::Model::InvalidValueError] when
|
|
35
|
+
# missing or malformed — callers translate to usage errors
|
|
36
|
+
def load(path)
|
|
37
|
+
from_json(File.read(path))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Build a Store from check results. +checks+ maps each checked
|
|
41
|
+
# file path to a `[DocumentResult, source_text]` pair (the
|
|
42
|
+
# text supplies line numbers). Entries are canonicalized: one
|
|
43
|
+
# row per (file, word), count = occurrences, line = line of
|
|
44
|
+
# the first occurrence, rows sorted by (file, word).
|
|
45
|
+
#
|
|
46
|
+
# @param checks [Hash<String => Array(Models::Result::DocumentResult, String)>]
|
|
47
|
+
# @return [Store]
|
|
48
|
+
def from_checks(checks)
|
|
49
|
+
entries = checks.flat_map do |file, (result, text)|
|
|
50
|
+
result.errors.group_by(&:word).map do |word, word_errors|
|
|
51
|
+
first = word_errors.min_by { |e| e.position || 0 }
|
|
52
|
+
Entry.new(
|
|
53
|
+
file: file,
|
|
54
|
+
line: Documents::SourcePosition.line_for_offset(text || "", first.position || 0),
|
|
55
|
+
word: word,
|
|
56
|
+
count: word_errors.size
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
new(entries: canonical_entries(entries))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Collapse entries to one row per (file, word) with summed
|
|
64
|
+
# counts, in canonical order.
|
|
65
|
+
def canonical_entries(entries)
|
|
66
|
+
entries.group_by(&:key).map do |(file, word), group|
|
|
67
|
+
first_line = group.map(&:line).compact.min || 1
|
|
68
|
+
Entry.new(file: file, line: first_line, word: word,
|
|
69
|
+
count: group.sum(&:count))
|
|
70
|
+
end.sort_by { |e| [e.file.to_s, e.word.to_s] }
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Write the baseline to +path+ (pretty JSON).
|
|
75
|
+
#
|
|
76
|
+
# @return [self]
|
|
77
|
+
def save(path)
|
|
78
|
+
File.write(path, to_json)
|
|
79
|
+
self
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# A decrementing budget view of +file+'s entries for the check
|
|
83
|
+
# path (plan 116): the callable answers false for the first
|
|
84
|
+
# +count+ occurrences of each entry word — exactly the
|
|
85
|
+
# occurrences {#apply} will absorb, whose suggestions nobody
|
|
86
|
+
# reads — and true for anything beyond, so new errors keep
|
|
87
|
+
# their suggestions. Occurrence order is check order, the same
|
|
88
|
+
# position order #apply absorbs in.
|
|
89
|
+
#
|
|
90
|
+
# @param file [String] path the text being checked came from
|
|
91
|
+
# @return [#call] word -> boolean
|
|
92
|
+
def suggestions_filter_for(file)
|
|
93
|
+
budget = Hash.new(0)
|
|
94
|
+
entries_for(file).each { |entry| budget[entry.word] += entry.count }
|
|
95
|
+
lambda do |word|
|
|
96
|
+
if budget[word].positive?
|
|
97
|
+
budget[word] -= 1
|
|
98
|
+
false
|
|
99
|
+
else
|
|
100
|
+
true
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Entries recorded for +file+. Paths compare canonically — a
|
|
106
|
+
# baseline recorded as `docs/a.md` matches a check run over
|
|
107
|
+
# `./docs/a.md` and vice versa — so walking `.` and explicit
|
|
108
|
+
# paths share one baseline file.
|
|
109
|
+
def entries_for(file)
|
|
110
|
+
target = self.class.canonical_path(file)
|
|
111
|
+
entries.select { |entry| self.class.canonical_path(entry.file) == target }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# @return [String] path with a leading `./` stripped and
|
|
115
|
+
# duplicate separators collapsed; absolute paths unchanged.
|
|
116
|
+
def self.canonical_path(path)
|
|
117
|
+
path.to_s.delete_prefix('./').gsub(%r{//+}, "/")
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Apply the baseline to +result+ (checked +file+): errors with
|
|
121
|
+
# baseline budget pass (moved to `suppressed_errors`, marked
|
|
122
|
+
# `suppressed_by: "baseline"`); the rest keep failing.
|
|
123
|
+
# Inline-suppressed errors were already filtered upstream and
|
|
124
|
+
# do not consume baseline budget.
|
|
125
|
+
#
|
|
126
|
+
# @param result [Models::Result::DocumentResult]
|
|
127
|
+
# @param file [String] path the result was checked against
|
|
128
|
+
# @return [Application]
|
|
129
|
+
def apply(result, file:)
|
|
130
|
+
budget = Hash.new(0)
|
|
131
|
+
entries_for(file).each { |entry| budget[entry.word] += entry.count }
|
|
132
|
+
|
|
133
|
+
occurrences = Hash.new { |hash, key| hash[key] = [] }
|
|
134
|
+
result.errors.each { |error| occurrences[error.word] << error }
|
|
135
|
+
|
|
136
|
+
absorbed = []
|
|
137
|
+
occurrences.each_value do |word_errors|
|
|
138
|
+
remaining = budget[word_errors.first.word]
|
|
139
|
+
next if remaining <= 0
|
|
140
|
+
|
|
141
|
+
in_order = word_errors.sort_by { |error| error.position || 0 }
|
|
142
|
+
absorbed.concat(in_order.first(remaining))
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
stale = entries_for(file).count { |entry| occurrences[entry.word].empty? }
|
|
146
|
+
absorbed_ids = absorbed.map(&:object_id).to_set
|
|
147
|
+
|
|
148
|
+
kept = result.errors.reject { |error| absorbed_ids.include?(error.object_id) }
|
|
149
|
+
suppressed = result.suppressed_errors + absorbed.map do |error|
|
|
150
|
+
Models::Result::WordResult.new(
|
|
151
|
+
word: error.word,
|
|
152
|
+
correct: false,
|
|
153
|
+
suggestions: error.suggestions.to_a,
|
|
154
|
+
position: error.position,
|
|
155
|
+
metadata: error.metadata,
|
|
156
|
+
suppressed: true,
|
|
157
|
+
suppressed_by: Models::Result::WordResult::SUPPRESSED_BY_BASELINE
|
|
158
|
+
)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
Application.new(
|
|
162
|
+
result: Models::Result::DocumentResult.new(
|
|
163
|
+
file: self.class.canonical_path(result.file),
|
|
164
|
+
errors: kept,
|
|
165
|
+
suppressed_errors: suppressed,
|
|
166
|
+
word_count: result.word_count,
|
|
167
|
+
metadata: result.metadata
|
|
168
|
+
),
|
|
169
|
+
suppressed_count: absorbed.size,
|
|
170
|
+
stale_count: stale
|
|
171
|
+
)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# CI baselines (plan 82, Track B).
|
|
5
|
+
#
|
|
6
|
+
# `kotoshu check --baseline FILE` freezes existing spelling debt so
|
|
7
|
+
# CI fails only on NEW errors: errors covered by the baseline pass,
|
|
8
|
+
# anything else fails, and stale entries (whose errors no longer
|
|
9
|
+
# exist) are reported so debt shrinks visibly.
|
|
10
|
+
#
|
|
11
|
+
# Matching is count-based, not position-based — an entry records how
|
|
12
|
+
# often a word is misspelled in a file (`count`), which survives
|
|
13
|
+
# reformatting that moves lines around. `line` is informational
|
|
14
|
+
# (the first occurrence when the baseline was written).
|
|
15
|
+
module Baseline
|
|
16
|
+
autoload :Application, "kotoshu/baseline/application"
|
|
17
|
+
autoload :Entry, "kotoshu/baseline/entry"
|
|
18
|
+
autoload :Store, "kotoshu/baseline/store"
|
|
19
|
+
end
|
|
20
|
+
end
|