kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "thor"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cli
|
|
7
|
+
# Catalog of commands the completion scripts know about, plus pure
|
|
8
|
+
# template builders that render shell-specific scripts from the
|
|
9
|
+
# catalog. Kept in a single module so the data and the rendering
|
|
10
|
+
# logic live next to the Thor subcommand that exposes them.
|
|
11
|
+
module Completions
|
|
12
|
+
# Describes a single CLI command for completion purposes.
|
|
13
|
+
Command = Struct.new(:name, :description, keyword_init: true)
|
|
14
|
+
|
|
15
|
+
# Top-level commands the bash/zsh/fish scripts will offer. Kept as
|
|
16
|
+
# a static catalog — descriptions are user-facing, so they are
|
|
17
|
+
# curated here rather than introspected from Thor (whose
|
|
18
|
+
# `description` field is often terse).
|
|
19
|
+
COMMANDS = [
|
|
20
|
+
Command.new(name: "check", description: "Check spelling in files, directories, or stdin"),
|
|
21
|
+
Command.new(name: "setup", description: "Set up languages (download or register local files)"),
|
|
22
|
+
Command.new(name: "status", description: "Show setup, cache, and runtime status"),
|
|
23
|
+
Command.new(name: "dict", description: "Dictionary operations"),
|
|
24
|
+
Command.new(name: "cache", description: "Cache management"),
|
|
25
|
+
Command.new(name: "completions", description: "Emit shell completion scripts"),
|
|
26
|
+
Command.new(name: "version", description: "Show version information"),
|
|
27
|
+
Command.new(name: "fetch", description: "Alias for `setup` (deprecated)")
|
|
28
|
+
].freeze
|
|
29
|
+
|
|
30
|
+
# Commands whose first positional argument is a language code.
|
|
31
|
+
LANGUAGE_ARGUMENT_COMMANDS = %w[setup fetch].freeze
|
|
32
|
+
|
|
33
|
+
# Pure template builders. Each takes the catalog and renders a
|
|
34
|
+
# shell-specific completion script. They have no IO dependencies
|
|
35
|
+
# and are unit-tested in isolation.
|
|
36
|
+
module ScriptBuilders
|
|
37
|
+
module_function
|
|
38
|
+
|
|
39
|
+
# Indent every line of +text+ by +n+ spaces. Used to align
|
|
40
|
+
# interpolated blocks within their surrounding heredoc body.
|
|
41
|
+
def indent(text, n)
|
|
42
|
+
pad = " " * n
|
|
43
|
+
text.lines.map { |line| "#{pad}#{line}" }.join
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
module Bash
|
|
47
|
+
module_function
|
|
48
|
+
|
|
49
|
+
def build(commands, language_argument_commands)
|
|
50
|
+
word_list = commands.map(&:name).join(" ")
|
|
51
|
+
lang_branch = language_branch(language_argument_commands)
|
|
52
|
+
<<~BASH
|
|
53
|
+
# kotoshu bash completion — install with:
|
|
54
|
+
# kotoshu completions bash > /etc/bash_completion.d/kotoshu
|
|
55
|
+
# or for a single user:
|
|
56
|
+
# kotoshu completions bash > ~/.local/share/bash-completion/completions/kotoshu
|
|
57
|
+
_kotoshu_completions() {
|
|
58
|
+
local cur prev
|
|
59
|
+
cur="${COMP_WORDS[COMP_CWORD]}"
|
|
60
|
+
prev="${COMP_WORDS[COMP_CWORD-1]}"
|
|
61
|
+
|
|
62
|
+
if [ "$COMP_CWORD" -eq 1 ]; then
|
|
63
|
+
COMPREPLY=( $(compgen -W "#{word_list}" -- "$cur") )
|
|
64
|
+
return 0
|
|
65
|
+
fi
|
|
66
|
+
|
|
67
|
+
#{ScriptBuilders.indent(lang_branch, 2)}
|
|
68
|
+
|
|
69
|
+
return 0
|
|
70
|
+
}
|
|
71
|
+
complete -F _kotoshu_completions kotoshu
|
|
72
|
+
BASH
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def language_branch(language_argument_commands)
|
|
76
|
+
return "# (no language-argument commands registered)" if language_argument_commands.empty?
|
|
77
|
+
|
|
78
|
+
matcher = language_argument_commands.join("|")
|
|
79
|
+
<<~BRANCH
|
|
80
|
+
case "$prev" in
|
|
81
|
+
#{matcher})
|
|
82
|
+
COMPREPLY=( $(compgen -W "$(kotoshu completions languages 2>/dev/null)" -- "$cur") )
|
|
83
|
+
return 0
|
|
84
|
+
;;
|
|
85
|
+
esac
|
|
86
|
+
BRANCH
|
|
87
|
+
end
|
|
88
|
+
private_class_method :language_branch
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
module Zsh
|
|
92
|
+
module_function
|
|
93
|
+
|
|
94
|
+
def build(commands, language_argument_commands)
|
|
95
|
+
describe_block = commands.map { |c| "'#{c.name}:#{escape(c.description)}'" }.join(" ")
|
|
96
|
+
lang_case = language_case(language_argument_commands)
|
|
97
|
+
<<~ZSH
|
|
98
|
+
#compdef kotoshu
|
|
99
|
+
# kotoshu zsh completion — install with:
|
|
100
|
+
# kotoshu completions zsh > "${fpath[1]}/_kotoshu"
|
|
101
|
+
|
|
102
|
+
_kotoshu() {
|
|
103
|
+
local -a commands
|
|
104
|
+
commands=(
|
|
105
|
+
#{describe_block}
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
if (( CURRENT == 2 )); then
|
|
109
|
+
_describe 'command' commands
|
|
110
|
+
return
|
|
111
|
+
fi
|
|
112
|
+
|
|
113
|
+
#{ScriptBuilders.indent(lang_case, 2)}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
_kotoshu "$@"
|
|
117
|
+
ZSH
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def language_case(language_argument_commands)
|
|
121
|
+
return "# (no language-argument commands registered)" if language_argument_commands.empty?
|
|
122
|
+
|
|
123
|
+
matcher = language_argument_commands.join("|")
|
|
124
|
+
<<~CASE
|
|
125
|
+
case "$words[2]" in
|
|
126
|
+
#{matcher})
|
|
127
|
+
local -a langs
|
|
128
|
+
langs=("${(@f)$(kotoshu completions languages 2>/dev/null)}")
|
|
129
|
+
_describe 'language' langs
|
|
130
|
+
;;
|
|
131
|
+
esac
|
|
132
|
+
CASE
|
|
133
|
+
end
|
|
134
|
+
private_class_method :language_case
|
|
135
|
+
|
|
136
|
+
def escape(text)
|
|
137
|
+
text.gsub("'", "''")
|
|
138
|
+
end
|
|
139
|
+
private_class_method :escape
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
module Fish
|
|
143
|
+
module_function
|
|
144
|
+
|
|
145
|
+
def build(commands, language_argument_commands)
|
|
146
|
+
command_lines = commands.map do |c|
|
|
147
|
+
"complete -c kotoshu -n \"__fish_use_subcommand\" -a \"#{c.name}\" -d \"#{escape(c.description)}\""
|
|
148
|
+
end.join("\n")
|
|
149
|
+
language_lines = language_argument_commands.map do |cmd|
|
|
150
|
+
matcher = "__fish_seen_subcommand_from #{cmd}"
|
|
151
|
+
"complete -c kotoshu -n \"#{matcher}\" -a \"(kotoshu completions languages 2>/dev/null)\""
|
|
152
|
+
end.join("\n")
|
|
153
|
+
<<~FISH
|
|
154
|
+
# kotoshu fish completion — install with:
|
|
155
|
+
# kotoshu completions fish > ~/.config/fish/completions/kotoshu.fish
|
|
156
|
+
#{command_lines}
|
|
157
|
+
#{language_lines}
|
|
158
|
+
FISH
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def escape(text)
|
|
162
|
+
text.gsub('\\', '\\\\\\\\').gsub('"', '\\"')
|
|
163
|
+
end
|
|
164
|
+
private_class_method :escape
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Thor subcommand wired as `kotoshu completions <shell>`.
|
|
170
|
+
#
|
|
171
|
+
# Emits shell completion scripts for bash, zsh, and fish, plus a
|
|
172
|
+
# `languages` helper that prints supported language codes one per
|
|
173
|
+
# line. The shell scripts shell out to `kotoshu completions
|
|
174
|
+
# languages` for dynamic language completion, so newly registered
|
|
175
|
+
# languages appear without regenerating the script.
|
|
176
|
+
class CompletionsCommand < Thor
|
|
177
|
+
desc "bash", "Emit a bash completion script for the kotoshu CLI"
|
|
178
|
+
def bash
|
|
179
|
+
$stdout.puts Completions::ScriptBuilders::Bash.build(
|
|
180
|
+
Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
|
|
181
|
+
)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
desc "zsh", "Emit a zsh completion script for the kotoshu CLI"
|
|
185
|
+
def zsh
|
|
186
|
+
$stdout.puts Completions::ScriptBuilders::Zsh.build(
|
|
187
|
+
Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
|
|
188
|
+
)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
desc "fish", "Emit a fish completion script for the kotoshu CLI"
|
|
192
|
+
def fish
|
|
193
|
+
$stdout.puts Completions::ScriptBuilders::Fish.build(
|
|
194
|
+
Completions::COMMANDS, Completions::LANGUAGE_ARGUMENT_COMMANDS
|
|
195
|
+
)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
desc "languages", "Emit supported language codes, one per line"
|
|
199
|
+
long_desc <<~DESC
|
|
200
|
+
Prints every language code registered with
|
|
201
|
+
`Kotoshu::Language::Registry`, one per line. Used by the bash,
|
|
202
|
+
zsh, and fish completion scripts to populate the language list
|
|
203
|
+
dynamically.
|
|
204
|
+
DESC
|
|
205
|
+
def languages
|
|
206
|
+
Kotoshu::Language::Registry.supported_codes.each { |c| $stdout.puts c }
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "thor"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Cli
|
|
7
|
+
# Dictionary command class.
|
|
8
|
+
#
|
|
9
|
+
# @example
|
|
10
|
+
# kotoshu dict list
|
|
11
|
+
# kotoshu dict info en-US
|
|
12
|
+
class DictCommand < Thor
|
|
13
|
+
desc "list", "List available dictionaries"
|
|
14
|
+
# Print the available dictionary types (one line each).
|
|
15
|
+
#
|
|
16
|
+
# @return [void]
|
|
17
|
+
def list
|
|
18
|
+
puts "Available dictionary types:"
|
|
19
|
+
puts " - unix_words: Unix system dictionary"
|
|
20
|
+
puts " - plain_text: Plain text word list"
|
|
21
|
+
puts " - custom: Custom in-memory dictionary"
|
|
22
|
+
puts " - hunspell: Hunspell (.dic/.aff)"
|
|
23
|
+
puts " - cspell: CSpell (.txt/.trie)"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
desc "info TYPE", "Show information about a dictionary type"
|
|
27
|
+
# Print a short description of one dictionary type, or a hint to
|
|
28
|
+
# run `kotoshu dict list` for an unknown type.
|
|
29
|
+
#
|
|
30
|
+
# @param type [String, Symbol] One of unix_words, plain_text,
|
|
31
|
+
# custom, hunspell, cspell
|
|
32
|
+
# @return [void]
|
|
33
|
+
def info(type)
|
|
34
|
+
case type.to_sym
|
|
35
|
+
when :unix_words
|
|
36
|
+
puts "UnixWords Dictionary:"
|
|
37
|
+
puts " Reads from Unix system dictionary files"
|
|
38
|
+
puts " Default paths:"
|
|
39
|
+
puts " - /usr/share/dict/words"
|
|
40
|
+
puts " - /usr/share/dict/web2"
|
|
41
|
+
puts " - /usr/share/dict/american-english"
|
|
42
|
+
when :plain_text
|
|
43
|
+
puts "PlainText Dictionary:"
|
|
44
|
+
puts " Reads from plain text word lists"
|
|
45
|
+
puts " One word per line, # comments supported"
|
|
46
|
+
when :custom
|
|
47
|
+
puts "Custom Dictionary:"
|
|
48
|
+
puts " In-memory dictionary for user-defined words"
|
|
49
|
+
when :hunspell
|
|
50
|
+
puts "Hunspell Dictionary:"
|
|
51
|
+
puts " Reads Hunspell .dic and .aff files"
|
|
52
|
+
puts " Supports morphological affix rules"
|
|
53
|
+
when :cspell
|
|
54
|
+
puts "CSpell Dictionary:"
|
|
55
|
+
puts " Reads CSpell .txt or .trie files"
|
|
56
|
+
puts " Uses trie data structure for fast lookups"
|
|
57
|
+
else
|
|
58
|
+
puts "Unknown dictionary type: #{type}"
|
|
59
|
+
puts "Run 'kotoshu dict list' for available types"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Cli
|
|
5
|
+
# Runs the check pipeline over a set of files and directories
|
|
6
|
+
# (plan 88): the CLI file mode, one target at a time, sharing one
|
|
7
|
+
# language resolution, baseline, and per-file output.
|
|
8
|
+
#
|
|
9
|
+
# Directories are expanded by DirectoryWalker; explicit file
|
|
10
|
+
# targets are checked as given. A baseline (when provided) is
|
|
11
|
+
# applied per file, exactly as single-file check does.
|
|
12
|
+
class DirectoryCheck
|
|
13
|
+
# Outcome of checking one file: the checked path, the check
|
|
14
|
+
# result, and the baseline application (nil without a
|
|
15
|
+
# baseline).
|
|
16
|
+
FileOutcome = Struct.new(:path, :result, :application, keyword_init: true)
|
|
17
|
+
|
|
18
|
+
attr_reader :outcomes
|
|
19
|
+
|
|
20
|
+
# @param roots [Array<String>] files and directories to check
|
|
21
|
+
# @param check [#call] callable taking the file text and an
|
|
22
|
+
# optional suggestions_filter: keyword and returning a
|
|
23
|
+
# Models::Result::DocumentResult (the CLI file-mode
|
|
24
|
+
# pipeline, including inline suppressions)
|
|
25
|
+
# @param baseline_path [String, nil] --baseline file
|
|
26
|
+
# @param include_globs [Array<String>] --include globs
|
|
27
|
+
# @param exclude_globs [Array<String>] --exclude globs
|
|
28
|
+
def initialize(roots, check:, baseline_path: nil, include_globs: [], exclude_globs: [])
|
|
29
|
+
@roots = Array(roots)
|
|
30
|
+
@check = check
|
|
31
|
+
@baseline_path = baseline_path
|
|
32
|
+
@walker = DirectoryWalker.new(include_globs: include_globs, exclude_globs: exclude_globs)
|
|
33
|
+
@outcomes = []
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Check every selected file.
|
|
37
|
+
#
|
|
38
|
+
# @return [Array<FileOutcome>] one outcome per checked file, in
|
|
39
|
+
# walk order
|
|
40
|
+
def run
|
|
41
|
+
@outcomes = target_files.map { |path| check_file(path) }
|
|
42
|
+
@outcomes
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The number of files checked.
|
|
46
|
+
#
|
|
47
|
+
# @return [Integer]
|
|
48
|
+
def file_count
|
|
49
|
+
@outcomes.size
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Total errors across all files, after baseline application.
|
|
53
|
+
#
|
|
54
|
+
# @return [Integer]
|
|
55
|
+
def error_count
|
|
56
|
+
@outcomes.sum { |outcome| outcome.result.error_count }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Total words across all files.
|
|
60
|
+
#
|
|
61
|
+
# @return [Integer]
|
|
62
|
+
def word_count
|
|
63
|
+
@outcomes.sum { |outcome| outcome.result.word_count }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Whether any file failed.
|
|
67
|
+
#
|
|
68
|
+
# @return [Boolean]
|
|
69
|
+
def failed?
|
|
70
|
+
@outcomes.any? { |outcome| outcome.result.failed? }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Baseline-suppressed error count across all files.
|
|
74
|
+
#
|
|
75
|
+
# @return [Integer]
|
|
76
|
+
def suppressed_count
|
|
77
|
+
applications.sum(&:suppressed_count)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Stale baseline entry count across all files.
|
|
81
|
+
#
|
|
82
|
+
# @return [Integer]
|
|
83
|
+
def stale_count
|
|
84
|
+
applications.sum(&:stale_count)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
# The applications of the baseline across outcomes.
|
|
90
|
+
#
|
|
91
|
+
# @return [Array[Baseline::Store::Application>]
|
|
92
|
+
def applications
|
|
93
|
+
@outcomes.filter_map(&:application)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Expand roots into a deduplicated file list: explicit files
|
|
97
|
+
# first (in argument order), then walked directory files.
|
|
98
|
+
#
|
|
99
|
+
# @return [Array<String>] file paths
|
|
100
|
+
def target_files
|
|
101
|
+
files = @roots.select { |root| File.file?(root) }
|
|
102
|
+
@roots.select { |root| File.directory?(root) }.each do |dir|
|
|
103
|
+
files.concat(@walker.files(dir))
|
|
104
|
+
end
|
|
105
|
+
files.each_with_object([]) do |path, unique|
|
|
106
|
+
unique << path unless unique.include?(path)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Check one file and apply the baseline to its result.
|
|
111
|
+
#
|
|
112
|
+
# @param path [String] file path
|
|
113
|
+
# @return [FileOutcome]
|
|
114
|
+
def check_file(path)
|
|
115
|
+
text = File.read(path, encoding: Kotoshu.configuration.encoding).scrub
|
|
116
|
+
# Baseline-covered occurrences skip the suggestion sweep
|
|
117
|
+
# (plan 116): the filter answers false for exactly the
|
|
118
|
+
# occurrences apply_baseline will absorb.
|
|
119
|
+
filter = @baseline_path ? baseline_store.suggestions_filter_for(path) : nil
|
|
120
|
+
result = @check.call(text, suggestions_filter: filter)
|
|
121
|
+
application = apply_baseline(result, path)
|
|
122
|
+
result = application.result if application
|
|
123
|
+
FileOutcome.new(path: path, result: result, application: application)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Apply the baseline to one file result, mirroring the CLI
|
|
127
|
+
# file-mode semantics.
|
|
128
|
+
#
|
|
129
|
+
# @param result [Models::Result::DocumentResult] check result
|
|
130
|
+
# @param path [String] file path
|
|
131
|
+
# @return [Baseline::Store::Application, nil]
|
|
132
|
+
def apply_baseline(result, path)
|
|
133
|
+
return nil unless @baseline_path
|
|
134
|
+
|
|
135
|
+
baseline_store.apply(result, file: path)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Load the baseline store once.
|
|
139
|
+
#
|
|
140
|
+
# @return [Baseline::Store]
|
|
141
|
+
def baseline_store
|
|
142
|
+
@baseline_store ||= Baseline::Store.load(@baseline_path)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Cli
|
|
5
|
+
# Walks directory trees and selects the text files to check
|
|
6
|
+
# (plan 88).
|
|
7
|
+
#
|
|
8
|
+
# Selection rules:
|
|
9
|
+
# - known text extensions by default: md markdown asciidoc adoc
|
|
10
|
+
# txt rst mdx
|
|
11
|
+
# - --include globs replace the extension default; --exclude
|
|
12
|
+
# globs always win
|
|
13
|
+
# - globs without a slash match the file basename; globs with a
|
|
14
|
+
# slash match the path relative to the checked root, with *
|
|
15
|
+
# and ? never crossing / and ** crossing directories
|
|
16
|
+
# - hidden files and directories (dot-prefixed) are skipped, as
|
|
17
|
+
# are .git, node_modules, vendor and target directories
|
|
18
|
+
# - .gitignore and .ignore files are honored (see IgnoreMatcher
|
|
19
|
+
# for the supported subset); nested ignore files only apply to
|
|
20
|
+
# their own subtree
|
|
21
|
+
class DirectoryWalker
|
|
22
|
+
TEXT_EXTENSIONS = %w[.md .markdown .asciidoc .adoc .txt .rst .mdx].freeze
|
|
23
|
+
DEFAULT_SKIP_DIRS = %w[.git node_modules vendor target].freeze
|
|
24
|
+
|
|
25
|
+
# @param include_globs [Array<String>] --include globs
|
|
26
|
+
# @param exclude_globs [Array<String>] --exclude globs
|
|
27
|
+
def initialize(include_globs: [], exclude_globs: [])
|
|
28
|
+
@include_globs = Array(include_globs)
|
|
29
|
+
@exclude_globs = Array(exclude_globs)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Collect the checkable files under one root, sorted
|
|
33
|
+
# deterministically.
|
|
34
|
+
#
|
|
35
|
+
# @param root [String] directory to walk
|
|
36
|
+
# @return [Array<String>] file paths rooted at +root+
|
|
37
|
+
def files(root)
|
|
38
|
+
walk(root, [], [])
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# Recursively collect files, carrying the ignore matchers of
|
|
44
|
+
# every ancestor directory paired with their depth below the
|
|
45
|
+
# root, so each matcher sees paths relative to its own
|
|
46
|
+
# directory (gitignore scoping).
|
|
47
|
+
#
|
|
48
|
+
# Selection and ignore matching use forward-slash relative
|
|
49
|
+
# paths (the gitignore/glob convention), independent of the
|
|
50
|
+
# platforms path separator; returned file paths are
|
|
51
|
+
# platform-native.
|
|
52
|
+
#
|
|
53
|
+
# @param directory [String] current absolute directory
|
|
54
|
+
# @param relative [Array<String>] path segments below the root
|
|
55
|
+
# @param matchers [Array<[IgnoreMatcher, Integer]>] ancestor
|
|
56
|
+
# matchers with their depths
|
|
57
|
+
# @return [Array<String>] collected file paths
|
|
58
|
+
def walk(directory, relative, matchers)
|
|
59
|
+
if IgnoreMatcher.present_in?(directory)
|
|
60
|
+
matchers = matchers + [[IgnoreMatcher.load_directory(directory), relative.length]]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
files = []
|
|
64
|
+
Dir.children(directory).sort.each do |entry|
|
|
65
|
+
path = File.join(directory, entry)
|
|
66
|
+
entry_relative = relative + [entry]
|
|
67
|
+
next if hidden?(entry)
|
|
68
|
+
next if ignored_by_any?(matchers, entry_relative, File.directory?(path))
|
|
69
|
+
|
|
70
|
+
if File.directory?(path)
|
|
71
|
+
next if DEFAULT_SKIP_DIRS.include?(entry)
|
|
72
|
+
|
|
73
|
+
files.concat(walk(path, entry_relative, matchers))
|
|
74
|
+
elsif selected?(entry, entry_relative.join("/"))
|
|
75
|
+
files << path
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
files
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Whether a path is ignored by any scoped matcher.
|
|
82
|
+
#
|
|
83
|
+
# @param matchers [Array<[IgnoreMatcher, Integer]>] matchers in
|
|
84
|
+
# scope with their depths
|
|
85
|
+
# @param entry_relative [Array<String>] path segments below root
|
|
86
|
+
# @param directory [Boolean] whether the entry is a directory
|
|
87
|
+
# @return [Boolean]
|
|
88
|
+
def ignored_by_any?(matchers, entry_relative, directory)
|
|
89
|
+
matchers.any? do |matcher, depth|
|
|
90
|
+
scoped = entry_relative[depth..].join("/")
|
|
91
|
+
matcher.ignored?(scoped, directory: directory)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Whether an entry name is hidden.
|
|
96
|
+
#
|
|
97
|
+
# @param entry [String] directory entry name
|
|
98
|
+
# @return [Boolean]
|
|
99
|
+
def hidden?(entry)
|
|
100
|
+
entry.start_with?(".")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Whether a file is selected for checking.
|
|
104
|
+
#
|
|
105
|
+
# @param basename [String] file name
|
|
106
|
+
# @param relative [String] path relative to the root
|
|
107
|
+
# @return [Boolean]
|
|
108
|
+
def selected?(basename, relative)
|
|
109
|
+
return false if globbed?(@exclude_globs, basename, relative)
|
|
110
|
+
return globbed?(@include_globs, basename, relative) unless @include_globs.empty?
|
|
111
|
+
|
|
112
|
+
TEXT_EXTENSIONS.include?(File.extname(basename))
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Whether any glob matches the file.
|
|
116
|
+
#
|
|
117
|
+
# @param globs [Array<String>] glob patterns
|
|
118
|
+
# @param basename [String] file name
|
|
119
|
+
# @param relative [String] path relative to the root
|
|
120
|
+
# @return [Boolean]
|
|
121
|
+
def globbed?(globs, basename, relative)
|
|
122
|
+
globs.any? do |glob|
|
|
123
|
+
if glob.include?("/")
|
|
124
|
+
File.fnmatch(glob, relative, File::FNM_PATHNAME)
|
|
125
|
+
else
|
|
126
|
+
File.fnmatch(glob, basename)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|