kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Core
|
|
5
|
+
# Indexed dictionary for efficient word lookup with multiple indexes.
|
|
6
|
+
# This is MORE model-driven than Spylls which uses simple hash indices.
|
|
7
|
+
#
|
|
8
|
+
# This is a proper domain model with rich behavior including:
|
|
9
|
+
# - Multiple indexes (case-sensitive, case-insensitive, prefix, suffix)
|
|
10
|
+
# - Rich query methods
|
|
11
|
+
# - Index management
|
|
12
|
+
# - Domain-specific behavior
|
|
13
|
+
class IndexedDictionary
|
|
14
|
+
attr_reader :words, :size
|
|
15
|
+
|
|
16
|
+
# @param words [Array<String>] Initial words to add
|
|
17
|
+
def initialize(words = [])
|
|
18
|
+
@words = []
|
|
19
|
+
@indexes = {
|
|
20
|
+
exact: {}, # case_sensitive: word => [positions]
|
|
21
|
+
lowercase: {}, # case_insensitive: word.downcase => [positions]
|
|
22
|
+
prefix: {}, # prefix => [words]
|
|
23
|
+
suffix: {}, # suffix => [words]
|
|
24
|
+
length: {}, # length => [words]
|
|
25
|
+
flag: {} # flag => [words] (future: for Hunspell)
|
|
26
|
+
}
|
|
27
|
+
@size = 0
|
|
28
|
+
|
|
29
|
+
words.each { |word| add_word(word) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Add a word to the dictionary with optional metadata.
|
|
33
|
+
#
|
|
34
|
+
# @param word [String] The word to add
|
|
35
|
+
# @param metadata [Hash] Optional metadata associated with the word
|
|
36
|
+
# @return [IndexedDictionary] Self for chaining
|
|
37
|
+
def add_word(word, metadata = {})
|
|
38
|
+
# Store the word with its index and metadata
|
|
39
|
+
entry = { word: word, index: @size, metadata: metadata }
|
|
40
|
+
@words << entry
|
|
41
|
+
@size += 1
|
|
42
|
+
|
|
43
|
+
# Update exact index (case-sensitive)
|
|
44
|
+
@indexes[:exact][word] ||= []
|
|
45
|
+
@indexes[:exact][word] << (@size - 1)
|
|
46
|
+
|
|
47
|
+
# Update lowercase index (case-insensitive)
|
|
48
|
+
lower = word.downcase
|
|
49
|
+
@indexes[:lowercase][lower] ||= []
|
|
50
|
+
@indexes[:lowercase][lower] << (@size - 1)
|
|
51
|
+
|
|
52
|
+
# Update prefix indexes (for prefix searching)
|
|
53
|
+
(1...word.length).each do |i|
|
|
54
|
+
prefix = word[0...i]
|
|
55
|
+
@indexes[:prefix][prefix] ||= []
|
|
56
|
+
@indexes[:prefix][prefix] << word
|
|
57
|
+
|
|
58
|
+
# Update suffix indexes (for suffix searching)
|
|
59
|
+
suffix = word[i..]
|
|
60
|
+
@indexes[:suffix][suffix] ||= []
|
|
61
|
+
@indexes[:suffix][suffix] << word
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Update length index (for length-range queries — used by
|
|
65
|
+
# edit-distance strategies to prune the dictionary by length).
|
|
66
|
+
(@indexes[:length][word.length] ||= []) << word
|
|
67
|
+
|
|
68
|
+
self
|
|
69
|
+
end
|
|
70
|
+
alias << add_word
|
|
71
|
+
|
|
72
|
+
# Add multiple words.
|
|
73
|
+
#
|
|
74
|
+
# @param new_words [Array<String>] Words to add
|
|
75
|
+
# @return [IndexedDictionary] Self for chaining
|
|
76
|
+
def add_words(new_words)
|
|
77
|
+
new_words.each { |word| add_word(word) }
|
|
78
|
+
self
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Check if a word exists (case-sensitive).
|
|
82
|
+
#
|
|
83
|
+
# @param word [String] The word to check
|
|
84
|
+
# @return [Boolean] True if word exists
|
|
85
|
+
def has_word?(word)
|
|
86
|
+
@indexes[:exact].key?(word)
|
|
87
|
+
end
|
|
88
|
+
alias include? has_word?
|
|
89
|
+
alias contains? has_word?
|
|
90
|
+
|
|
91
|
+
# Check if a word exists (case-insensitive).
|
|
92
|
+
#
|
|
93
|
+
# @param word [String] The word to check
|
|
94
|
+
# @return [Boolean] True if word exists (any case)
|
|
95
|
+
def has_word_ignorecase?(word)
|
|
96
|
+
@indexes[:lowercase].key?(word.downcase)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Look up a word (case-sensitive).
|
|
100
|
+
#
|
|
101
|
+
# @param word [String] The word to look up
|
|
102
|
+
# @return [Hash, nil] Word entry or nil
|
|
103
|
+
def lookup(word)
|
|
104
|
+
indices = @indexes[:exact][word]
|
|
105
|
+
return nil if indices.nil? || indices.empty?
|
|
106
|
+
|
|
107
|
+
@words[indices.first]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Look up a word (case-insensitive).
|
|
111
|
+
#
|
|
112
|
+
# @param word [String] The word to look up
|
|
113
|
+
# @return [Hash, nil] Word entry or nil
|
|
114
|
+
def lookup_ignorecase(word)
|
|
115
|
+
indices = @indexes[:lowercase][word.downcase]
|
|
116
|
+
return nil if indices.nil? || indices.empty?
|
|
117
|
+
|
|
118
|
+
@words[indices.first]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Find all words with a given prefix.
|
|
122
|
+
#
|
|
123
|
+
# @param prefix [String] The prefix to match
|
|
124
|
+
# @param ignore_case [Boolean] Whether to ignore case
|
|
125
|
+
# @return [Array<String>] Words with the prefix
|
|
126
|
+
def find_by_prefix(prefix, ignore_case: false)
|
|
127
|
+
if ignore_case
|
|
128
|
+
prefix_lower = prefix.downcase
|
|
129
|
+
all_words.select { |w| w.downcase.start_with?(prefix_lower) }
|
|
130
|
+
else
|
|
131
|
+
@indexes[:prefix].fetch(prefix, []).dup
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Find all words with a given suffix.
|
|
136
|
+
#
|
|
137
|
+
# @param suffix [String] The suffix to match
|
|
138
|
+
# @param ignore_case [Boolean] Whether to ignore case
|
|
139
|
+
# @return [Array<String>] Words with the suffix
|
|
140
|
+
def find_by_suffix(suffix, ignore_case: false)
|
|
141
|
+
if ignore_case
|
|
142
|
+
suffix_lower = suffix.downcase
|
|
143
|
+
all_words.select { |w| w.downcase.end_with?(suffix_lower) }
|
|
144
|
+
else
|
|
145
|
+
@indexes[:suffix].fetch(suffix, []).dup
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Find words matching a pattern.
|
|
150
|
+
#
|
|
151
|
+
# @param pattern [Regexp] The pattern to match
|
|
152
|
+
# @return [Array<String>] Matching words
|
|
153
|
+
def find_by_pattern(pattern)
|
|
154
|
+
all_words.select { |w| w.match?(pattern) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Find words of a specific length.
|
|
158
|
+
#
|
|
159
|
+
# @param length [Integer] The exact length
|
|
160
|
+
# @return [Array<String>] Words of the given length
|
|
161
|
+
def find_by_length(length)
|
|
162
|
+
(@indexes[:length][length] || []).dup
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Find words within a length range.
|
|
166
|
+
#
|
|
167
|
+
# @param min_length [Integer] Minimum length
|
|
168
|
+
# @param max_length [Integer] Maximum length
|
|
169
|
+
# @return [Array<String>] Words within the length range
|
|
170
|
+
def find_by_length_range(min_length:, max_length:)
|
|
171
|
+
(min_length..max_length).flat_map { |len| @indexes[:length][len] || [] }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Get all words in the dictionary.
|
|
175
|
+
#
|
|
176
|
+
# @return [Array<String>] All words
|
|
177
|
+
def all_words
|
|
178
|
+
@words.map { |entry| entry[:word] }
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Get random words from the dictionary.
|
|
182
|
+
#
|
|
183
|
+
# @param count [Integer] Number of random words
|
|
184
|
+
# @return [Array<String>] Random words
|
|
185
|
+
def random_words(count: 1)
|
|
186
|
+
return [] if @words.empty?
|
|
187
|
+
|
|
188
|
+
indices = (0...@size).to_a.sample(count)
|
|
189
|
+
indices.map { |i| @words[i][:word] }
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Get words starting with each letter (A-Z).
|
|
193
|
+
#
|
|
194
|
+
# @return [Hash] Hash of letter => word count
|
|
195
|
+
def count_by_first_letter
|
|
196
|
+
result = Hash.new(0)
|
|
197
|
+
all_words.each do |word|
|
|
198
|
+
next if word.empty?
|
|
199
|
+
|
|
200
|
+
letter = word[0].upcase
|
|
201
|
+
result[letter] += 1
|
|
202
|
+
end
|
|
203
|
+
result
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Get word length distribution.
|
|
207
|
+
#
|
|
208
|
+
# @return [Hash] Hash of length => count
|
|
209
|
+
def count_by_length
|
|
210
|
+
result = Hash.new(0)
|
|
211
|
+
all_words.each { |word| result[word.length] += 1 }
|
|
212
|
+
result
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Check if the dictionary is empty.
|
|
216
|
+
#
|
|
217
|
+
# @return [Boolean] True if empty
|
|
218
|
+
def empty?
|
|
219
|
+
@size.zero?
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Iterate over all words.
|
|
223
|
+
#
|
|
224
|
+
# @yield [word] Each word
|
|
225
|
+
# @return [Enumerator] Enumerator if no block given
|
|
226
|
+
def each_word
|
|
227
|
+
return enum_for(:each_word) unless block_given?
|
|
228
|
+
|
|
229
|
+
@words.each { |entry| yield entry[:word] }
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Iterate over all words with indices.
|
|
233
|
+
#
|
|
234
|
+
# @yield [word, index] Each word and its index
|
|
235
|
+
# @return [Enumerator] Enumerator if no block given
|
|
236
|
+
def each_with_index
|
|
237
|
+
return enum_for(:each_with_index) unless block_given?
|
|
238
|
+
|
|
239
|
+
@words.each { |entry| yield entry[:word], entry[:index] }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Build a Trie from the dictionary words.
|
|
243
|
+
#
|
|
244
|
+
# @return [Trie] New trie containing all words
|
|
245
|
+
def to_trie
|
|
246
|
+
Trie::Builder.from_array(all_words)
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Get statistics about the dictionary.
|
|
250
|
+
#
|
|
251
|
+
# @return [Hash] Statistics
|
|
252
|
+
def statistics
|
|
253
|
+
lengths = all_words.map(&:length)
|
|
254
|
+
|
|
255
|
+
{
|
|
256
|
+
total_words: @size,
|
|
257
|
+
unique_words: all_words.uniq.size,
|
|
258
|
+
min_length: lengths.min || 0,
|
|
259
|
+
max_length: lengths.max || 0,
|
|
260
|
+
avg_length: lengths.empty? ? 0 : (lengths.sum.to_f / lengths.size).round(2),
|
|
261
|
+
count_by_first_letter: count_by_first_letter,
|
|
262
|
+
count_by_length: count_by_length
|
|
263
|
+
}
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# Convert to string.
|
|
267
|
+
#
|
|
268
|
+
# @return [String] String representation
|
|
269
|
+
def to_s
|
|
270
|
+
"IndexedDictionary(size: #{@size})"
|
|
271
|
+
end
|
|
272
|
+
alias inspect to_s
|
|
273
|
+
|
|
274
|
+
# Create indexed dictionary from a file.
|
|
275
|
+
#
|
|
276
|
+
# @param path [String] Path to word list file
|
|
277
|
+
# @return [IndexedDictionary] New dictionary
|
|
278
|
+
def self.from_file(path)
|
|
279
|
+
words = File.foreach(path, chomp: true).reject { |l| l.empty? || l.start_with?("#") }
|
|
280
|
+
new(words)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Create indexed dictionary from a Trie.
|
|
284
|
+
#
|
|
285
|
+
# @param trie [Trie] The trie to convert
|
|
286
|
+
# @return [IndexedDictionary] New dictionary
|
|
287
|
+
def self.from_trie(trie)
|
|
288
|
+
words = trie.all_words
|
|
289
|
+
new(words)
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Affix rule model for Hunspell-style affix processing.
|
|
6
|
+
#
|
|
7
|
+
# Affix rules define how prefixes and suffixes can be added or removed
|
|
8
|
+
# from words to generate morphological variants.
|
|
9
|
+
#
|
|
10
|
+
# This is a value object that represents a single affix rule.
|
|
11
|
+
#
|
|
12
|
+
# @note This class is immutable and frozen on initialization.
|
|
13
|
+
#
|
|
14
|
+
# @example Creating a prefix rule
|
|
15
|
+
# rule = Models::AffixRule.new(
|
|
16
|
+
# type: :prefix,
|
|
17
|
+
# flag: "A",
|
|
18
|
+
# strip: "",
|
|
19
|
+
# add: "re",
|
|
20
|
+
# condition: "."
|
|
21
|
+
# )
|
|
22
|
+
# rule.prefix? # => true
|
|
23
|
+
# rule.suffix? # => false
|
|
24
|
+
class AffixRule
|
|
25
|
+
# @return [Symbol] The affix type (:prefix or :suffix)
|
|
26
|
+
attr_reader :type
|
|
27
|
+
|
|
28
|
+
# @return [String] The flag character identifying this rule
|
|
29
|
+
attr_reader :flag
|
|
30
|
+
|
|
31
|
+
# @return [String] Characters to strip from the word
|
|
32
|
+
attr_reader :strip
|
|
33
|
+
|
|
34
|
+
# @return [String] Characters to add to the word
|
|
35
|
+
attr_reader :add
|
|
36
|
+
|
|
37
|
+
# @return [String, Regexp] Condition for applying this rule
|
|
38
|
+
attr_reader :condition
|
|
39
|
+
|
|
40
|
+
# @return [Boolean] Whether this is a cross-product rule
|
|
41
|
+
attr_reader :cross_product
|
|
42
|
+
|
|
43
|
+
# Affix rule types.
|
|
44
|
+
TYPES = {
|
|
45
|
+
prefix: "PFX",
|
|
46
|
+
suffix: "SFX"
|
|
47
|
+
}.freeze
|
|
48
|
+
|
|
49
|
+
# Create a new AffixRule.
|
|
50
|
+
#
|
|
51
|
+
# @param type [Symbol] The affix type (:prefix or :suffix)
|
|
52
|
+
# @param flag [String] The flag character
|
|
53
|
+
# @param strip [String] Characters to strip
|
|
54
|
+
# @param add [String] Characters to add
|
|
55
|
+
# @param condition [String, Regexp] Condition for applying
|
|
56
|
+
# @param cross_product [Boolean] Whether this is cross-product
|
|
57
|
+
def initialize(type:, flag:, strip:, add:, condition: ".", cross_product: false)
|
|
58
|
+
raise ArgumentError, "Invalid type: #{type}" unless %i[prefix suffix].include?(type)
|
|
59
|
+
raise ArgumentError, "Flag cannot be empty" if flag.nil? || flag.empty?
|
|
60
|
+
|
|
61
|
+
@type = type
|
|
62
|
+
@flag = flag.dup.freeze
|
|
63
|
+
@strip = strip.dup.freeze
|
|
64
|
+
@add = add.dup.freeze
|
|
65
|
+
@condition = condition.is_a?(Regexp) ? condition : compile_condition(condition)
|
|
66
|
+
@cross_product = cross_product
|
|
67
|
+
|
|
68
|
+
freeze
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Check if this is a prefix rule.
|
|
72
|
+
#
|
|
73
|
+
# @return [Boolean] True if prefix
|
|
74
|
+
def prefix?
|
|
75
|
+
@type == :prefix
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Check if this is a suffix rule.
|
|
79
|
+
#
|
|
80
|
+
# @return [Boolean] True if suffix
|
|
81
|
+
def suffix?
|
|
82
|
+
@type == :suffix
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Check if this rule can be applied to a word.
|
|
86
|
+
#
|
|
87
|
+
# @param word [String] The word to check
|
|
88
|
+
# @return [Boolean] True if the rule applies
|
|
89
|
+
def applies_to?(word)
|
|
90
|
+
return false if word.nil? || word.empty?
|
|
91
|
+
|
|
92
|
+
word.match?(@condition)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Apply this rule to a word.
|
|
96
|
+
#
|
|
97
|
+
# @param word [String] The word to modify
|
|
98
|
+
# @return [String, nil] The modified word, or nil if rule doesn't apply
|
|
99
|
+
def apply(word)
|
|
100
|
+
return nil unless applies_to?(word)
|
|
101
|
+
|
|
102
|
+
if prefix?
|
|
103
|
+
# Strip from beginning, add prefix
|
|
104
|
+
word.start_with?(@strip) ? @add + word[@strip.length..] : nil
|
|
105
|
+
else
|
|
106
|
+
# Strip from end, add suffix
|
|
107
|
+
word.end_with?(@strip) ? word[0...-@strip.length] + @add : nil
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Remove this affix from a word (reverse operation).
|
|
112
|
+
#
|
|
113
|
+
# Unlike #apply, the condition is checked against the candidate base
|
|
114
|
+
# form (the result of stripping the affix), not the input. This makes
|
|
115
|
+
# #remove the proper inverse of #apply: every successful apply can be
|
|
116
|
+
# undone by a matching remove.
|
|
117
|
+
#
|
|
118
|
+
# @param word [String] The word to modify
|
|
119
|
+
# @return [String, nil] The stripped word, or nil if affix doesn't match
|
|
120
|
+
def remove(word)
|
|
121
|
+
return nil if word.nil? || word.empty?
|
|
122
|
+
|
|
123
|
+
candidate =
|
|
124
|
+
if prefix?
|
|
125
|
+
word.start_with?(@add) ? @strip + word[@add.length..] : nil
|
|
126
|
+
else
|
|
127
|
+
word.end_with?(@add) ? word[0...-@add.length] + @strip : nil
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
return nil if candidate.nil?
|
|
131
|
+
return nil unless candidate.match?(@condition)
|
|
132
|
+
|
|
133
|
+
candidate
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Get the Hunspell representation.
|
|
137
|
+
#
|
|
138
|
+
# @return [String] The affix line for Hunspell format
|
|
139
|
+
def to_hunspell
|
|
140
|
+
type_code = TYPES[@type]
|
|
141
|
+
cross = @cross_product ? "Y" : "N"
|
|
142
|
+
"#{type_code} #{@flag} #{cross} #{@strip.empty? ? '0' : @strip} " \
|
|
143
|
+
"#{@add} #{@condition.is_a?(Regexp) ? condition_to_s : @condition}"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Check equality based on all attributes.
|
|
147
|
+
#
|
|
148
|
+
# @param other [AffixRule] The other rule
|
|
149
|
+
# @return [Boolean] True if equal
|
|
150
|
+
def ==(other)
|
|
151
|
+
return false unless other.is_a?(AffixRule)
|
|
152
|
+
|
|
153
|
+
@type == other.type &&
|
|
154
|
+
@flag == other.flag &&
|
|
155
|
+
@strip == other.strip &&
|
|
156
|
+
@add == other.add &&
|
|
157
|
+
@condition == other.condition &&
|
|
158
|
+
@cross_product == other.cross_product
|
|
159
|
+
end
|
|
160
|
+
alias eql? ==
|
|
161
|
+
|
|
162
|
+
# Hash based on all attributes.
|
|
163
|
+
#
|
|
164
|
+
# @return [Integer] Hash code
|
|
165
|
+
def hash
|
|
166
|
+
[@type, @flag, @strip, @add, @cross_product].hash
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Compare rules by flag.
|
|
170
|
+
#
|
|
171
|
+
# @param other [AffixRule] The other rule
|
|
172
|
+
# @return [Integer] Comparison result
|
|
173
|
+
def <=>(other)
|
|
174
|
+
return nil unless other.is_a?(AffixRule)
|
|
175
|
+
|
|
176
|
+
@flag <=> other.flag
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
private
|
|
180
|
+
|
|
181
|
+
# Compile condition string to regex.
|
|
182
|
+
#
|
|
183
|
+
# Hunspell uses '.' for match-all, '[...]' for character classes,
|
|
184
|
+
# and '^[...]' for negated classes. The compiled regex is anchored:
|
|
185
|
+
# suffix conditions anchor to end-of-string, prefix to start.
|
|
186
|
+
#
|
|
187
|
+
# @param condition [String] The condition string
|
|
188
|
+
# @return [Regexp] The compiled regex
|
|
189
|
+
def compile_condition(condition)
|
|
190
|
+
return // if condition == "."
|
|
191
|
+
|
|
192
|
+
regex_str = condition.dup
|
|
193
|
+
|
|
194
|
+
# Convert [...] to Ruby character class
|
|
195
|
+
regex_str = regex_str.gsub(/\[([^\]]+)\]/, "(?:\\1)")
|
|
196
|
+
|
|
197
|
+
# Convert ^[...] to negative lookahead
|
|
198
|
+
# Convert ^ to negative lookahead for single character
|
|
199
|
+
regex_str = regex_str.gsub("\\^(\\w)", "(?!\\1).")
|
|
200
|
+
|
|
201
|
+
# Anchor to end for suffix, beginning for prefix.
|
|
202
|
+
# The anchors themselves are NOT escaped — Ruby's Regexp needs
|
|
203
|
+
# the bare ^/$ metacharacters to mean start/end of line.
|
|
204
|
+
if @type == :suffix
|
|
205
|
+
Regexp.new("#{regex_str}$")
|
|
206
|
+
else
|
|
207
|
+
Regexp.new("^#{regex_str}")
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Convert regex condition back to a Hunspell condition string.
|
|
212
|
+
#
|
|
213
|
+
# @return [String] The condition string
|
|
214
|
+
def condition_to_s
|
|
215
|
+
source = @condition.source
|
|
216
|
+
return "." if source.empty?
|
|
217
|
+
|
|
218
|
+
# Strip the anchors added by compile_condition. Using a regex
|
|
219
|
+
# pattern with escaped metacharacters so gsub matches the literal
|
|
220
|
+
# ^ and $ characters, not the zero-width anchors.
|
|
221
|
+
source = source.gsub('^', "").gsub('$', "")
|
|
222
|
+
|
|
223
|
+
# Convert negative lookaheads back
|
|
224
|
+
source = source.gsub(/\(\?!([^)]+)\)\./, "^\\1")
|
|
225
|
+
|
|
226
|
+
# Convert non-capturing groups back to character classes
|
|
227
|
+
source.gsub('(?:', "[").gsub(')', "]")
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Create an affix rule from a Hunspell affix line.
|
|
231
|
+
#
|
|
232
|
+
# @param line [String] The affix line
|
|
233
|
+
# @param type [Symbol] The rule type (:prefix or :suffix)
|
|
234
|
+
# @return [AffixRule] New affix rule
|
|
235
|
+
#
|
|
236
|
+
# @example Parsing a Hunspell prefix rule
|
|
237
|
+
# AffixRule.from_hunspell("PFX A Y 1 re .", :prefix)
|
|
238
|
+
#
|
|
239
|
+
# @example Parsing a Hunspell suffix rule
|
|
240
|
+
# AffixRule.from_hunspell("SFX V N 2 ive e", :suffix)
|
|
241
|
+
def self.from_hunspell(line, type)
|
|
242
|
+
parts = line.split
|
|
243
|
+
return nil if parts.length < 5
|
|
244
|
+
|
|
245
|
+
flag = parts[1]
|
|
246
|
+
cross_product = parts[2] == "Y"
|
|
247
|
+
strip = parts[3] == "0" ? "" : parts[3]
|
|
248
|
+
add = parts[4]
|
|
249
|
+
condition = parts[5] || "."
|
|
250
|
+
|
|
251
|
+
new(
|
|
252
|
+
type: type,
|
|
253
|
+
flag: flag,
|
|
254
|
+
strip: strip,
|
|
255
|
+
add: add,
|
|
256
|
+
condition: condition,
|
|
257
|
+
cross_product: cross_product
|
|
258
|
+
)
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Models
|
|
7
|
+
module Result
|
|
8
|
+
# Result object for checking a document or file.
|
|
9
|
+
#
|
|
10
|
+
# Serialized via lutaml-model. Holds an Array<WordResult> in +errors+.
|
|
11
|
+
class DocumentResult < Lutaml::Model::Serializable
|
|
12
|
+
attribute :file, :string
|
|
13
|
+
attribute :word_count, :integer, default: 0
|
|
14
|
+
attribute :errors, WordResult, collection: true
|
|
15
|
+
# Errors absorbed by inline ignore directives or a baseline
|
|
16
|
+
# (plan 82); each entry is a real misspelling marked
|
|
17
|
+
# `suppressed: true` with `suppressed_by` naming what absorbed it.
|
|
18
|
+
attribute :suppressed_errors, WordResult, collection: true
|
|
19
|
+
attribute :metadata, :hash, default: {}
|
|
20
|
+
|
|
21
|
+
# Accept framework-private keywords (e.g. +lutaml_register+)
|
|
22
|
+
# from lutaml-model's +from_hash+ so the round-trip works
|
|
23
|
+
# without lutaml having to poke ivars directly.
|
|
24
|
+
def initialize(file: nil, errors: [], word_count: 0, metadata: {}, suppressed_errors: [], **kwargs)
|
|
25
|
+
super(
|
|
26
|
+
file: file,
|
|
27
|
+
word_count: word_count,
|
|
28
|
+
errors: errors,
|
|
29
|
+
suppressed_errors: suppressed_errors,
|
|
30
|
+
metadata: metadata,
|
|
31
|
+
**kwargs
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def success?
|
|
36
|
+
Array(errors).empty?
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def failed?
|
|
40
|
+
!success?
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def error_count
|
|
44
|
+
Array(errors).size
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def suppressed_count
|
|
48
|
+
Array(suppressed_errors).size
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def unique_error_count
|
|
52
|
+
errors.map(&:word).uniq.size
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def has_error_for?(word)
|
|
56
|
+
errors.any? { |e| e.word == word }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def errors_for(word)
|
|
60
|
+
errors.select { |e| e.word == word }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def each_error(&)
|
|
64
|
+
return enum_for(:each_error) unless block_given?
|
|
65
|
+
|
|
66
|
+
errors.each(&)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def each_unique_error(&)
|
|
70
|
+
return enum_for(:each_unique_error) unless block_given?
|
|
71
|
+
|
|
72
|
+
errors.group_by(&:word).each(&)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def first_errors(n = 10)
|
|
76
|
+
errors.first(n)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def error_summary
|
|
80
|
+
summary = Hash.new(0)
|
|
81
|
+
each_error { |error| summary[error.word] += 1 }
|
|
82
|
+
summary
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def ==(other)
|
|
86
|
+
return false unless other.is_a?(DocumentResult)
|
|
87
|
+
|
|
88
|
+
file == other.file && errors == other.errors && suppressed_errors == other.suppressed_errors
|
|
89
|
+
end
|
|
90
|
+
alias eql? ==
|
|
91
|
+
|
|
92
|
+
def hash
|
|
93
|
+
[file, errors, suppressed_errors].hash
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def to_s
|
|
97
|
+
if success?
|
|
98
|
+
if file
|
|
99
|
+
"File '#{file}': No spelling errors found (#{word_count} words checked)"
|
|
100
|
+
else
|
|
101
|
+
"No spelling errors found (#{word_count} words checked)"
|
|
102
|
+
end
|
|
103
|
+
else
|
|
104
|
+
prefix = file ? "File '#{file}':" : ""
|
|
105
|
+
"#{prefix} #{error_count} spelling error(s) found " \
|
|
106
|
+
"(#{unique_error_count} unique) in #{word_count} words"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
alias inspect to_s
|
|
110
|
+
|
|
111
|
+
def self.success(file: nil, word_count: 0)
|
|
112
|
+
new(file: file, errors: [], word_count: word_count)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def self.failure(file: nil, errors: [], word_count: 0)
|
|
116
|
+
new(file: file, errors: errors, word_count: word_count)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def self.merge(results)
|
|
120
|
+
return new if results.empty?
|
|
121
|
+
|
|
122
|
+
new(
|
|
123
|
+
file: nil,
|
|
124
|
+
errors: results.flat_map(&:errors),
|
|
125
|
+
suppressed_errors: results.flat_map(&:suppressed_errors),
|
|
126
|
+
word_count: results.sum(&:word_count)
|
|
127
|
+
)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|