kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Embeddings module for FastText ONNX integration.
|
|
4
|
+
#
|
|
5
|
+
# Provides semantic spell checking using FastText word embeddings converted
|
|
6
|
+
# to ONNX format. Loading this file is cheap — every class below is
|
|
7
|
+
# autoloaded on first reference.
|
|
8
|
+
#
|
|
9
|
+
# @example Simple usage (recommended)
|
|
10
|
+
# pipeline = Kotoshu::Embeddings.from_cache(language: 'en')
|
|
11
|
+
# neighbors = pipeline.find_nearest('semantic', k: 5)
|
|
12
|
+
#
|
|
13
|
+
# @example Advanced usage
|
|
14
|
+
# vocab = Kotoshu::Embeddings::Vocabulary.from_file('vocab.json')
|
|
15
|
+
# model = Kotoshu::Embeddings::OnnxRuntimeModel.from_file('model.onnx')
|
|
16
|
+
# engine = Kotoshu::Embeddings::SimilarityEngine.new(pre_normalize: true)
|
|
17
|
+
#
|
|
18
|
+
module Kotoshu
|
|
19
|
+
module Embeddings
|
|
20
|
+
DEFAULT_DIMENSION = 300
|
|
21
|
+
MAX_VOCABULARY_SIZE = 100_000
|
|
22
|
+
VERSION = '2.0.0'
|
|
23
|
+
|
|
24
|
+
autoload :Protocol, "kotoshu/embeddings/protocol"
|
|
25
|
+
autoload :ProtocolError, "kotoshu/embeddings/protocol"
|
|
26
|
+
autoload :EmbeddingModelProtocol, "kotoshu/embeddings/protocol"
|
|
27
|
+
autoload :SimilarityEngineProtocol, "kotoshu/embeddings/protocol"
|
|
28
|
+
autoload :VocabularyProtocol, "kotoshu/embeddings/protocol"
|
|
29
|
+
autoload :LruCache, "kotoshu/embeddings/lru_cache"
|
|
30
|
+
autoload :Vocabulary, "kotoshu/embeddings/vocabulary"
|
|
31
|
+
autoload :OnnxRuntimeModel, "kotoshu/embeddings/onnx_runtime_model"
|
|
32
|
+
autoload :SimilarityEngine, "kotoshu/embeddings/similarity_engine"
|
|
33
|
+
autoload :Search, "kotoshu/embeddings/search"
|
|
34
|
+
autoload :SimilaritySearch, "kotoshu/embeddings/similarity_search"
|
|
35
|
+
autoload :EmbeddingPipeline, "kotoshu/embeddings/embedding_pipeline"
|
|
36
|
+
autoload :Registry, "kotoshu/embeddings/registry"
|
|
37
|
+
|
|
38
|
+
# Create an EmbeddingPipeline from cache
|
|
39
|
+
#
|
|
40
|
+
# @param language [String] ISO 639-1 language code
|
|
41
|
+
# @param preload [Boolean] Preload embeddings into memory
|
|
42
|
+
# @param index [:exact, :auto] Search index type
|
|
43
|
+
# @return [EmbeddingPipeline]
|
|
44
|
+
#
|
|
45
|
+
def self.from_cache(language:, preload: false, index: :exact)
|
|
46
|
+
EmbeddingPipeline.from_cache(language: language, preload: preload,
|
|
47
|
+
index: index)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Check if a language is supported
|
|
51
|
+
#
|
|
52
|
+
# @param language [String] ISO 639-1 language code
|
|
53
|
+
# @return [Boolean]
|
|
54
|
+
#
|
|
55
|
+
def self.language_supported?(language)
|
|
56
|
+
cache = Kotoshu::Cache::ModelCache.new
|
|
57
|
+
cache.available_models_for(language.to_sym).include?(:onnx)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# List all supported languages
|
|
61
|
+
#
|
|
62
|
+
# @return [Array<String>]
|
|
63
|
+
#
|
|
64
|
+
def self.supported_languages
|
|
65
|
+
cache = Kotoshu::Cache::ModelCache.new
|
|
66
|
+
cache.all_available_models[:onnx].keys.map(&:to_s)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Create a custom embedding pipeline
|
|
70
|
+
#
|
|
71
|
+
# @param vocabulary [Vocabulary] Vocabulary instance
|
|
72
|
+
# @param model [EmbeddingModel] Model instance
|
|
73
|
+
# @param preload [Boolean] Preload embeddings
|
|
74
|
+
# @param pre_normalize [Boolean] Pre-normalize vectors
|
|
75
|
+
# @return [EmbeddingPipeline]
|
|
76
|
+
#
|
|
77
|
+
def self.create_pipeline(vocabulary:, model:, preload: false,
|
|
78
|
+
pre_normalize: false)
|
|
79
|
+
EmbeddingPipeline.new(
|
|
80
|
+
vocabulary: vocabulary,
|
|
81
|
+
model: model,
|
|
82
|
+
preload: preload,
|
|
83
|
+
pre_normalize: pre_normalize,
|
|
84
|
+
)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
class Spellchecker
|
|
5
|
+
# Fluent checker for chainable configuration.
|
|
6
|
+
#
|
|
7
|
+
# Experimental and not public API: outside the 1.0 stability
|
|
8
|
+
# freeze, may change or be removed without a deprecation cycle.
|
|
9
|
+
#
|
|
10
|
+
# Provides a convenient API for spell checking with method chaining.
|
|
11
|
+
#
|
|
12
|
+
# @example Basic usage
|
|
13
|
+
# result = Kotoshu.fluent.check("Hello wrold")
|
|
14
|
+
#
|
|
15
|
+
# @example With options
|
|
16
|
+
# Kotoshu.fluent
|
|
17
|
+
# .ignore_words(/https?:\/\/\S+/)
|
|
18
|
+
# .max_suggestions(5)
|
|
19
|
+
# .check("Hello wrold")
|
|
20
|
+
class FluentChecker
|
|
21
|
+
# @return [Spellchecker] The underlying spellchecker
|
|
22
|
+
attr_reader :spellchecker
|
|
23
|
+
|
|
24
|
+
# @return [Hash] Configuration options
|
|
25
|
+
attr_reader :options
|
|
26
|
+
|
|
27
|
+
# Create a new fluent checker.
|
|
28
|
+
#
|
|
29
|
+
# @param spellchecker [Spellchecker] The underlying spellchecker
|
|
30
|
+
# @param options [Hash] Configuration options
|
|
31
|
+
def initialize(spellchecker:, options: {})
|
|
32
|
+
@spellchecker = spellchecker
|
|
33
|
+
@options = options
|
|
34
|
+
@progress_callback = nil
|
|
35
|
+
@error_callback = nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Check text for spelling errors.
|
|
39
|
+
#
|
|
40
|
+
# @param text [String] Text to check
|
|
41
|
+
# @return [Models::Result::DocumentResult] Check result
|
|
42
|
+
def check(text)
|
|
43
|
+
@spellchecker.check(text)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Ignore words matching pattern.
|
|
47
|
+
#
|
|
48
|
+
# @param pattern [Regexp] Pattern to ignore
|
|
49
|
+
# @return [FluentChecker] Self for chaining
|
|
50
|
+
#
|
|
51
|
+
# @example
|
|
52
|
+
# fluent.ignore_words(/https?:\/\/\S+/)
|
|
53
|
+
def ignore_words(pattern)
|
|
54
|
+
@options[:ignore_patterns] ||= []
|
|
55
|
+
@options[:ignore_patterns] << pattern
|
|
56
|
+
self
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Set maximum suggestions.
|
|
60
|
+
#
|
|
61
|
+
# @param max [Integer] Maximum suggestions
|
|
62
|
+
# @return [FluentChecker] Self for chaining
|
|
63
|
+
def max_suggestions(max)
|
|
64
|
+
@options[:max_suggestions] = max
|
|
65
|
+
self
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Set progress callback.
|
|
69
|
+
#
|
|
70
|
+
# @param block [Proc] Callback proc
|
|
71
|
+
# @return [FluentChecker] Self for chaining
|
|
72
|
+
def on_progress(&block)
|
|
73
|
+
@progress_callback = block
|
|
74
|
+
self
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Set error callback.
|
|
78
|
+
#
|
|
79
|
+
# @param block [Proc] Callback proc
|
|
80
|
+
# @return [FluentChecker] Self for chaining
|
|
81
|
+
def on_error(&block)
|
|
82
|
+
@error_callback = block
|
|
83
|
+
self
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Get the result.
|
|
87
|
+
#
|
|
88
|
+
# @return [Models::Result::ResultDocumentResult] Check result
|
|
89
|
+
def result
|
|
90
|
+
check(@text)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Base class for pattern matchers.
|
|
7
|
+
#
|
|
8
|
+
# Pattern matchers check token sequences against specific patterns
|
|
9
|
+
# defined in YAML configuration.
|
|
10
|
+
class BaseMatcher
|
|
11
|
+
def initialize(pattern, exceptions = {})
|
|
12
|
+
@pattern = pattern
|
|
13
|
+
@exceptions = exceptions
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Match tokens against the pattern.
|
|
17
|
+
#
|
|
18
|
+
# @param _tokens [Array<Hash>] Array of token hashes (unused)
|
|
19
|
+
# @param _rule [Rule] The rule being checked (unused)
|
|
20
|
+
# @return [Array<Hash>] Array of error hashes
|
|
21
|
+
def match(_tokens, _rule)
|
|
22
|
+
[]
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
protected
|
|
26
|
+
|
|
27
|
+
# Extract target tokens from context specification.
|
|
28
|
+
#
|
|
29
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
30
|
+
# @param context_spec [Hash] Context specification from pattern
|
|
31
|
+
# @return [Array<Hash>] Array of matching tokens with their indices
|
|
32
|
+
def extract_tokens_from_context(tokens, context_spec)
|
|
33
|
+
result = []
|
|
34
|
+
context_spec.each do |spec|
|
|
35
|
+
if spec['target_token']
|
|
36
|
+
tokens.each_with_index do |token, idx|
|
|
37
|
+
if token[:token]&.downcase == spec['target_token']
|
|
38
|
+
result << { token: token, index: idx }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
result
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for double negative rules.
|
|
7
|
+
#
|
|
8
|
+
# Detects when multiple negative words appear within a certain
|
|
9
|
+
# distance, with support for declared exception phrases. The
|
|
10
|
+
# canonical exception is the "not only...but also" correlative
|
|
11
|
+
# conjunction: both "not" and "also" are flagged as negatives,
|
|
12
|
+
# but the construction is grammatical and must not fire.
|
|
13
|
+
#
|
|
14
|
+
# Exception phrases use the literal +...+ separator to indicate
|
|
15
|
+
# arbitrary text between the prefix and suffix:
|
|
16
|
+
#
|
|
17
|
+
# "not only...but also"
|
|
18
|
+
#
|
|
19
|
+
# means "not only" at some position, then any text, then "but
|
|
20
|
+
# also". Every negative inside such a region is suppressed.
|
|
21
|
+
class DoubleNegativeMatcher < BaseMatcher
|
|
22
|
+
# Match tokens against the double negative pattern.
|
|
23
|
+
#
|
|
24
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
25
|
+
# @param rule [Rule] The rule being checked
|
|
26
|
+
# @return [Array<Hash>] Array of error hashes
|
|
27
|
+
def match(tokens, rule)
|
|
28
|
+
errors = []
|
|
29
|
+
exceptions = rule.exceptions || {}
|
|
30
|
+
exception_phrases = exceptions['phrases'] || []
|
|
31
|
+
|
|
32
|
+
conditions = @pattern['conditions'] || []
|
|
33
|
+
distance_condition = conditions.find { |c| c['type'] == 'distance_check' }
|
|
34
|
+
max_distance = distance_condition&.dig('max_distance') || 15
|
|
35
|
+
|
|
36
|
+
regions = exception_regions(tokens, exception_phrases)
|
|
37
|
+
|
|
38
|
+
negative_indices = tokens.each_with_index.filter_map do |token, idx|
|
|
39
|
+
word = token[:token]&.downcase
|
|
40
|
+
next unless negative?(word)
|
|
41
|
+
next if regions.any? { |range| range.include?(idx) }
|
|
42
|
+
|
|
43
|
+
idx
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
negative_indices.each_cons(2) do |idx1, idx2|
|
|
47
|
+
pos1 = tokens[idx1][:position]
|
|
48
|
+
pos2 = tokens[idx2][:position]
|
|
49
|
+
distance = pos2 - pos1
|
|
50
|
+
next if distance > max_distance
|
|
51
|
+
|
|
52
|
+
error = build_error(tokens, idx1, idx2, rule)
|
|
53
|
+
errors << error if error
|
|
54
|
+
end
|
|
55
|
+
errors
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
# Check if a word is a negative.
|
|
61
|
+
#
|
|
62
|
+
# @param word [String] The word to check
|
|
63
|
+
# @return [Boolean] True if the word is a negative
|
|
64
|
+
def negative?(word)
|
|
65
|
+
return false if word.nil? || word.empty?
|
|
66
|
+
|
|
67
|
+
negatives = %w[not no neither nobody never nothing nowhere hardly barely scarcely]
|
|
68
|
+
return true if negatives.include?(word)
|
|
69
|
+
return true if word.end_with?("n't")
|
|
70
|
+
|
|
71
|
+
false
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Locate every (start...end) region of the token stream that
|
|
75
|
+
# matches one of the declared +exception_phrases+. Each phrase
|
|
76
|
+
# is "<prefix>...<suffix>"; a region runs from the start of the
|
|
77
|
+
# prefix match through the end of the corresponding suffix
|
|
78
|
+
# match. If the suffix never matches, the region is not
|
|
79
|
+
# recorded (the idiom was never closed).
|
|
80
|
+
#
|
|
81
|
+
# @param tokens [Array<Hash>]
|
|
82
|
+
# @param exception_phrases [Array<String>]
|
|
83
|
+
# @return [Array<Range>] Index ranges to suppress
|
|
84
|
+
def exception_regions(tokens, exception_phrases)
|
|
85
|
+
return [] if exception_phrases.empty?
|
|
86
|
+
|
|
87
|
+
exception_phrases.flat_map do |phrase|
|
|
88
|
+
phrase_regions(tokens, phrase)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Find every region for a single phrase.
|
|
93
|
+
#
|
|
94
|
+
# Two phrase shapes are supported:
|
|
95
|
+
#
|
|
96
|
+
# - "prefix...suffix" — correlative / open-ended idiom. Region
|
|
97
|
+
# runs from the start of the prefix match through the end of
|
|
98
|
+
# the next matching suffix. If the suffix never matches, the
|
|
99
|
+
# region is not recorded (idiom was never closed).
|
|
100
|
+
# - "literal phrase" (no "...") — fixed n-gram. Region is the
|
|
101
|
+
# exact span of the match.
|
|
102
|
+
def phrase_regions(tokens, phrase)
|
|
103
|
+
if phrase.include?("...")
|
|
104
|
+
split_phrase_regions(tokens, phrase)
|
|
105
|
+
else
|
|
106
|
+
literal_phrase_regions(tokens, phrase)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# "prefix...suffix" form.
|
|
111
|
+
def split_phrase_regions(tokens, phrase)
|
|
112
|
+
prefix, suffix = phrase.split("...", 2)
|
|
113
|
+
prefix_tokens = prefix.to_s.split.map(&:downcase)
|
|
114
|
+
suffix_tokens = suffix.to_s.split.map(&:downcase)
|
|
115
|
+
return [] if prefix_tokens.empty? || suffix_tokens.empty?
|
|
116
|
+
|
|
117
|
+
regions = []
|
|
118
|
+
tokens.each_index do |start_idx|
|
|
119
|
+
next unless tokens_match?(tokens, start_idx, prefix_tokens)
|
|
120
|
+
|
|
121
|
+
suffix_search_from = start_idx + prefix_tokens.length
|
|
122
|
+
suffix_idx = (suffix_search_from...tokens.length).find do |i|
|
|
123
|
+
tokens_match?(tokens, i, suffix_tokens)
|
|
124
|
+
end
|
|
125
|
+
next unless suffix_idx
|
|
126
|
+
|
|
127
|
+
regions << (start_idx...(suffix_idx + suffix_tokens.length))
|
|
128
|
+
end
|
|
129
|
+
regions
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Literal n-gram form (no "...").
|
|
133
|
+
def literal_phrase_regions(tokens, phrase)
|
|
134
|
+
pattern = phrase.split.map(&:downcase)
|
|
135
|
+
return [] if pattern.empty?
|
|
136
|
+
|
|
137
|
+
regions = []
|
|
138
|
+
tokens.each_index do |start_idx|
|
|
139
|
+
next unless tokens_match?(tokens, start_idx, pattern)
|
|
140
|
+
|
|
141
|
+
regions << (start_idx...(start_idx + pattern.length))
|
|
142
|
+
end
|
|
143
|
+
regions
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# True if tokens[idx, idx + pattern.length] matches +pattern+
|
|
147
|
+
# (case-insensitive, token-by-token).
|
|
148
|
+
def tokens_match?(tokens, idx, pattern)
|
|
149
|
+
return false if idx.negative?
|
|
150
|
+
return false if idx + pattern.length > tokens.length
|
|
151
|
+
|
|
152
|
+
pattern.each_with_index.all? do |word, offset|
|
|
153
|
+
tokens[idx + offset][:token]&.downcase == word
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Build an error hash.
|
|
158
|
+
#
|
|
159
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
160
|
+
# @param idx1 [Integer] First negative index
|
|
161
|
+
# @param idx2 [Integer] Second negative index
|
|
162
|
+
# @param rule [Rule] The rule being checked
|
|
163
|
+
# @return [Hash] Error hash
|
|
164
|
+
def build_error(tokens, idx1, idx2, rule)
|
|
165
|
+
words = tokens[idx1..idx2].map { |t| t[:token] }.join(' ')
|
|
166
|
+
|
|
167
|
+
{
|
|
168
|
+
rule_id: rule.id,
|
|
169
|
+
position: tokens[idx1][:position],
|
|
170
|
+
message: rule.message,
|
|
171
|
+
suggestion: rule.suggestion,
|
|
172
|
+
context: words,
|
|
173
|
+
suggestions: rule.suggestion ? [rule.suggestion] : []
|
|
174
|
+
}
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for literal multi-word phrase confusions.
|
|
7
|
+
#
|
|
8
|
+
# Catches phrases where every word is individually valid but the
|
|
9
|
+
# combination is a common error — typically phonetic confusions
|
|
10
|
+
# like "could of" (should be "could have"). The spelling checker
|
|
11
|
+
# passes these because each word is in the dictionary; only a
|
|
12
|
+
# phrase-level grammar check catches them.
|
|
13
|
+
#
|
|
14
|
+
# The rule config declares the wrong phrase and its replacement:
|
|
15
|
+
#
|
|
16
|
+
# conditions:
|
|
17
|
+
# - type: phrase_check
|
|
18
|
+
# wrong_phrase: "could of"
|
|
19
|
+
# suggestion: "could have"
|
|
20
|
+
#
|
|
21
|
+
# The matcher scans the token stream for consecutive tokens whose
|
|
22
|
+
# downcased text matches +wrong_phrase+ (split on whitespace)
|
|
23
|
+
# and emits one error per match.
|
|
24
|
+
class PhraseMatcher < BaseMatcher
|
|
25
|
+
# Match tokens against the phrase-confusion pattern.
|
|
26
|
+
#
|
|
27
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
28
|
+
# @param rule [Rule] The rule being checked
|
|
29
|
+
# @return [Array<Hash>] Array of error hashes
|
|
30
|
+
def match(tokens, rule)
|
|
31
|
+
wrong_phrase = phrase_condition&.dig('wrong_phrase')
|
|
32
|
+
suggestion = phrase_condition&.dig('suggestion')
|
|
33
|
+
return [] unless wrong_phrase && suggestion
|
|
34
|
+
|
|
35
|
+
wrong_tokens = wrong_phrase.downcase.split
|
|
36
|
+
return [] if wrong_tokens.empty?
|
|
37
|
+
|
|
38
|
+
matches = []
|
|
39
|
+
tokens.each_with_index do |start_token, start_idx|
|
|
40
|
+
next unless start_token[:token]&.downcase == wrong_tokens.first
|
|
41
|
+
|
|
42
|
+
match_idx = find_match(tokens, start_idx, wrong_tokens)
|
|
43
|
+
next unless match_idx
|
|
44
|
+
|
|
45
|
+
matches << build_error(tokens, match_idx, wrong_tokens.length,
|
|
46
|
+
wrong_phrase, suggestion, rule)
|
|
47
|
+
end
|
|
48
|
+
matches
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def phrase_condition
|
|
54
|
+
@phrase_condition ||= @pattern['conditions']&.find do |c|
|
|
55
|
+
c['type'] == 'phrase_check'
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Walk forward from +start_idx+ and confirm the next
|
|
60
|
+
# +expected.length+ tokens match +expected+ (case-insensitive).
|
|
61
|
+
# Returns the start_idx on full match, nil otherwise.
|
|
62
|
+
def find_match(tokens, start_idx, expected)
|
|
63
|
+
expected.each_with_index do |word, offset|
|
|
64
|
+
tok = tokens[start_idx + offset]
|
|
65
|
+
return nil unless tok && tok[:token]&.downcase == word
|
|
66
|
+
end
|
|
67
|
+
start_idx
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def build_error(tokens, start_idx, length, wrong_phrase, suggestion, rule)
|
|
71
|
+
end_idx = start_idx + length - 1
|
|
72
|
+
words = tokens[start_idx..end_idx].map { |t| t[:token] }.join(' ')
|
|
73
|
+
{
|
|
74
|
+
rule_id: rule.id,
|
|
75
|
+
position: tokens[start_idx][:position],
|
|
76
|
+
message: rule.message,
|
|
77
|
+
suggestion: suggestion,
|
|
78
|
+
context: words,
|
|
79
|
+
suggestions: [suggestion],
|
|
80
|
+
wrong_phrase: wrong_phrase
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for there/their/they're confusion rules.
|
|
7
|
+
#
|
|
8
|
+
# This matcher detects when "there" is used where "their"
|
|
9
|
+
# (possessive) is intended.
|
|
10
|
+
class PossessiveContextMatcher < BaseMatcher
|
|
11
|
+
# Match tokens against the there/their pattern.
|
|
12
|
+
#
|
|
13
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
14
|
+
# @param rule [Rule] The rule being checked
|
|
15
|
+
# @return [Array<Hash>] Array of error hashes
|
|
16
|
+
def match(tokens, rule)
|
|
17
|
+
errors = []
|
|
18
|
+
exceptions = rule.exceptions || {}
|
|
19
|
+
|
|
20
|
+
location_indicators = exceptions['location_indicators'] || {}
|
|
21
|
+
location_verbs = location_indicators['verbs'] || []
|
|
22
|
+
possessive_nouns = location_indicators['possessive_nouns'] || []
|
|
23
|
+
|
|
24
|
+
tokens.each_with_index do |token, idx|
|
|
25
|
+
word = token[:token]&.downcase
|
|
26
|
+
next unless word == 'there'
|
|
27
|
+
|
|
28
|
+
next_token = tokens[idx + 1]
|
|
29
|
+
next unless next_token
|
|
30
|
+
|
|
31
|
+
next_word = next_token[:token]&.downcase
|
|
32
|
+
|
|
33
|
+
# Skip if followed by verb (location/existence context)
|
|
34
|
+
next if location_verbs.include?(next_word)
|
|
35
|
+
|
|
36
|
+
uses_their = false
|
|
37
|
+
|
|
38
|
+
# Check POS tags first
|
|
39
|
+
next_pos = next_token[:pos_tag]
|
|
40
|
+
if next_pos && ['NOUN', 'NOUN_PROPER', 'ADJ'].include?(next_pos)
|
|
41
|
+
uses_their = true
|
|
42
|
+
# Fallback to word list
|
|
43
|
+
elsif possessive_nouns.include?(next_word)
|
|
44
|
+
uses_their = true
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
if uses_their
|
|
48
|
+
errors << build_error(token, next_token, rule)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
errors
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
# Build an error hash.
|
|
57
|
+
#
|
|
58
|
+
# @param token [Hash] The token with "there"
|
|
59
|
+
# @param next_token [Hash] The next token
|
|
60
|
+
# @param rule [Rule] The rule being checked
|
|
61
|
+
# @return [Hash] Error hash
|
|
62
|
+
def build_error(token, next_token, rule)
|
|
63
|
+
{
|
|
64
|
+
rule_id: rule.id,
|
|
65
|
+
position: token[:position],
|
|
66
|
+
message: rule.message,
|
|
67
|
+
suggestion: rule.suggestion,
|
|
68
|
+
context: "\"#{token[:token]} #{next_token[:token]}\"",
|
|
69
|
+
suggestions: [rule.suggestion]
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for possessive-vs-contraction confusion rules.
|
|
7
|
+
#
|
|
8
|
+
# Detects when a possessive form ("its", "your", "whose") is
|
|
9
|
+
# used where the corresponding contraction ("it's", "you're",
|
|
10
|
+
# "who's") is intended — or vice versa. The trigger is the
|
|
11
|
+
# following token's POS tag or word form.
|
|
12
|
+
#
|
|
13
|
+
# Rule shape:
|
|
14
|
+
#
|
|
15
|
+
# patterns:
|
|
16
|
+
# - context:
|
|
17
|
+
# target_token: "its"
|
|
18
|
+
# trigger_when_followed_by:
|
|
19
|
+
# tags: [ADJ, ADV]
|
|
20
|
+
# words: [been, going, getting]
|
|
21
|
+
# conditions:
|
|
22
|
+
# - type: possessive_contraction_check
|
|
23
|
+
#
|
|
24
|
+
# When the target token is followed by a token whose POS tag is
|
|
25
|
+
# in +tags+ OR whose lowercased form is in +words+, emit an
|
|
26
|
+
# error with +rule.suggestion+ as the replacement.
|
|
27
|
+
#
|
|
28
|
+
# Examples:
|
|
29
|
+
# "its cold" → suggest "it's" (cold is ADJ)
|
|
30
|
+
# "your ready" → suggest "you're"
|
|
31
|
+
# "whose coming" → suggest "who's"
|
|
32
|
+
class PossessiveContractionMatcher < BaseMatcher
|
|
33
|
+
def match(tokens, rule)
|
|
34
|
+
errors = []
|
|
35
|
+
target = pattern_target_token
|
|
36
|
+
triggers = pattern_triggers
|
|
37
|
+
|
|
38
|
+
tokens.each_with_index do |token, idx|
|
|
39
|
+
next unless matches_target?(token, target)
|
|
40
|
+
|
|
41
|
+
next_token = tokens[idx + 1]
|
|
42
|
+
next unless next_token
|
|
43
|
+
next unless triggers_match?(next_token, triggers)
|
|
44
|
+
|
|
45
|
+
errors << build_error(token, next_token, rule)
|
|
46
|
+
end
|
|
47
|
+
errors
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
protected
|
|
51
|
+
|
|
52
|
+
def pattern_target_token
|
|
53
|
+
@pattern.dig('context', 'target_token')
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def pattern_triggers
|
|
57
|
+
@pattern.dig('context', 'trigger_when_followed_by') || {}
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def matches_target?(token, target)
|
|
61
|
+
target && token[:token]&.downcase == target.downcase
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def triggers_match?(next_token, triggers)
|
|
65
|
+
trigger_tags = triggers['tags'] || []
|
|
66
|
+
trigger_words = (triggers['words'] || []).map(&:downcase)
|
|
67
|
+
|
|
68
|
+
tag = next_token[:pos_tag]
|
|
69
|
+
word = next_token[:token]&.downcase
|
|
70
|
+
|
|
71
|
+
trigger_tags.include?(tag) || trigger_words.include?(word)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def build_error(token, next_token, rule)
|
|
75
|
+
{
|
|
76
|
+
rule_id: rule.id,
|
|
77
|
+
position: token[:position],
|
|
78
|
+
message: rule.message,
|
|
79
|
+
suggestion: rule.suggestion,
|
|
80
|
+
context: "\"#{token[:token]} #{next_token[:token]}\"",
|
|
81
|
+
suggestions: [rule.suggestion]
|
|
82
|
+
}
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|