kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
# Immutable value object: a point in the original (markup-bearing)
|
|
6
|
+
# source.
|
|
7
|
+
#
|
|
8
|
+
# Carries both a 0-based character offset (useful for tools that
|
|
9
|
+
# want to slice the source by range) and a 1-based line/column pair
|
|
10
|
+
# (useful for editor highlighting and human-readable messages).
|
|
11
|
+
SourcePosition = Struct.new(:offset, :line, :column, keyword_init: true) do
|
|
12
|
+
include Comparable
|
|
13
|
+
|
|
14
|
+
def initialize(offset:, line:, column:)
|
|
15
|
+
raise ArgumentError, "offset must be >= 0" if offset.negative?
|
|
16
|
+
raise ArgumentError, "line must be >= 1" if line < 1
|
|
17
|
+
raise ArgumentError, "column must be >= 1" if column < 1
|
|
18
|
+
|
|
19
|
+
super
|
|
20
|
+
freeze
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Lexicographic order by (offset, line, column). Matches the
|
|
24
|
+
# natural order of positions when scanning the source left to
|
|
25
|
+
# right, top to bottom.
|
|
26
|
+
def <=>(other)
|
|
27
|
+
return nil unless other.is_a?(SourcePosition)
|
|
28
|
+
|
|
29
|
+
[offset, line, column] <=> [other.offset, other.line, other.column]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# 1-based line number of the character at +offset+ in +text+.
|
|
33
|
+
# Offsets past the end clamp to the last line; a nil offset
|
|
34
|
+
# counts as the first line.
|
|
35
|
+
def self.line_for_offset(text, offset)
|
|
36
|
+
clamped = [offset.to_i, 0].max.clamp(0, [text.length - 1, 0].max)
|
|
37
|
+
text[0, clamped].count("\n") + 1
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def to_s
|
|
41
|
+
"line #{line}, column #{column} (offset #{offset})"
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "forwardable"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Documents
|
|
7
|
+
# Immutable value object: a span in the original source. The
|
|
8
|
+
# +start+ position is inclusive; the +end+ position is exclusive
|
|
9
|
+
# (matches Ruby's range convention and editor selection APIs).
|
|
10
|
+
#
|
|
11
|
+
# SourceRange is the canonical "where" that an error report
|
|
12
|
+
# carries. The checker resolves it from the {Document} before
|
|
13
|
+
# constructing an error so callers (plugins, editors, the CLI)
|
|
14
|
+
# never have to know about flattened vs source offsets.
|
|
15
|
+
class SourceRange
|
|
16
|
+
include Comparable
|
|
17
|
+
extend Forwardable
|
|
18
|
+
|
|
19
|
+
attr_reader :start, :end
|
|
20
|
+
|
|
21
|
+
def_delegators :@start, :offset, :line, :column
|
|
22
|
+
|
|
23
|
+
# @param start_pos [SourcePosition] inclusive start
|
|
24
|
+
# @param end_pos [SourcePosition] exclusive end
|
|
25
|
+
def initialize(start_pos:, end_pos:)
|
|
26
|
+
raise TypeError, "start must be a SourcePosition" unless start_pos.is_a?(SourcePosition)
|
|
27
|
+
raise TypeError, "end must be a SourcePosition" unless end_pos.is_a?(SourcePosition)
|
|
28
|
+
raise ArgumentError, "end must be >= start" if end_pos < start_pos
|
|
29
|
+
|
|
30
|
+
@start = start_pos
|
|
31
|
+
@end = end_pos
|
|
32
|
+
freeze
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Order by start position; ties broken by end position.
|
|
36
|
+
def <=>(other)
|
|
37
|
+
return nil unless other.is_a?(SourceRange)
|
|
38
|
+
|
|
39
|
+
cmp = @start <=> other.start
|
|
40
|
+
cmp.zero? ? (@end <=> other.end) : cmp
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# True if +pos+ falls inside this range. End is exclusive.
|
|
44
|
+
def contains?(pos)
|
|
45
|
+
pos.is_a?(SourcePosition) && pos >= @start && pos < @end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Length in characters of the source span. Uses the offset
|
|
49
|
+
# difference, which is the editor-friendly interpretation: a
|
|
50
|
+
# range from offset 7 to offset 15 covers 8 characters.
|
|
51
|
+
def length
|
|
52
|
+
@end.offset - @start.offset
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# True when this range covers zero characters (start == end).
|
|
56
|
+
def empty?
|
|
57
|
+
@start == @end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Combine +other+ with self into a range that spans both.
|
|
61
|
+
# @param other [SourceRange]
|
|
62
|
+
# @return [SourceRange]
|
|
63
|
+
def union(other)
|
|
64
|
+
raise TypeError, "other must be a SourceRange" unless other.is_a?(SourceRange)
|
|
65
|
+
|
|
66
|
+
min_start = [@start, other.start].min
|
|
67
|
+
max_end = [@end, other.end].max
|
|
68
|
+
self.class.new(start_pos: min_start, end_pos: max_end)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def to_s
|
|
72
|
+
"#{@start}..#{@end}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
module Suppressions
|
|
6
|
+
# Line-oriented scanner for inline ignore directives.
|
|
7
|
+
#
|
|
8
|
+
# ONE shared implementation of the directive semantics; the only
|
|
9
|
+
# per-format knowledge is how to find a comment on a line:
|
|
10
|
+
#
|
|
11
|
+
# - :plain — a line whose trimmed content IS the directive
|
|
12
|
+
# (whole-line only; trailing directives would be ambiguous in
|
|
13
|
+
# prose)
|
|
14
|
+
# - :markdown — HTML comments `<!-- ... -->`, whole-line or
|
|
15
|
+
# trailing
|
|
16
|
+
# - :asciidoc — `//` line comments, whole-line or trailing
|
|
17
|
+
# (the marker must be at line start or preceded by whitespace,
|
|
18
|
+
# so URLs like https://example.com are not comments)
|
|
19
|
+
# - :auto — all of the above. The Spellchecker facade uses this
|
|
20
|
+
# profile because it checks raw text without a parser.
|
|
21
|
+
#
|
|
22
|
+
# Inside a comment the directive must appear at the very start:
|
|
23
|
+
# prose that merely mentions `kotoshu:disable-line` is not a
|
|
24
|
+
# directive.
|
|
25
|
+
#
|
|
26
|
+
# Word lists are only meaningful on `disable-next-line` (the one
|
|
27
|
+
# form the plan defines them for); extra words after the other
|
|
28
|
+
# directives are ignored.
|
|
29
|
+
class Scanner
|
|
30
|
+
# A directive line: `kotoshu:<verb>` at the very start of a
|
|
31
|
+
# comment body, capturing the verb and any trailing word list.
|
|
32
|
+
DIRECTIVE_PATTERN =
|
|
33
|
+
/\Akotoshu:(disable-line|disable-next-line|disable-file|enable-file)\b[ \t]*(.*)\z/
|
|
34
|
+
|
|
35
|
+
# Matches an HTML comment body (`<!-- ... -->`; Markdown profile).
|
|
36
|
+
HTML_COMMENT_BODIES = /<!--\s*(.*?)\s*-->/
|
|
37
|
+
|
|
38
|
+
# Matches an AsciiDoc `//` line comment body. The marker must
|
|
39
|
+
# start the line or follow whitespace so URLs are not comments.
|
|
40
|
+
ASCII_DOC_COMMENT_BODY = /(?:\A|[ \t])\/\/[ \t]?(?<body>.*)\Z/
|
|
41
|
+
|
|
42
|
+
# Which comment kinds each format profile recognizes: :bare
|
|
43
|
+
# (whole-line directives), :html_comment (Markdown),
|
|
44
|
+
# :line_comment (AsciiDoc). :auto enables all three — the
|
|
45
|
+
# facade default, since it checks raw text without a parser.
|
|
46
|
+
PROFILE = {
|
|
47
|
+
plain: %i[bare],
|
|
48
|
+
markdown: %i[html_comment],
|
|
49
|
+
asciidoc: %i[line_comment],
|
|
50
|
+
auto: %i[bare html_comment line_comment]
|
|
51
|
+
}.freeze
|
|
52
|
+
|
|
53
|
+
EMPTY = [].freeze
|
|
54
|
+
private_constant :EMPTY
|
|
55
|
+
|
|
56
|
+
class << self
|
|
57
|
+
# Scan +source+ and return the suppressions its directives
|
|
58
|
+
# produce, in source order.
|
|
59
|
+
#
|
|
60
|
+
# @param source [String, nil]
|
|
61
|
+
# @param format [Symbol] :plain, :markdown, :asciidoc, :auto
|
|
62
|
+
# (unknown symbols fall back to :auto)
|
|
63
|
+
# @return [Array<Suppression>] frozen
|
|
64
|
+
def scan(source, format: :auto)
|
|
65
|
+
return EMPTY if source.nil? || source.empty?
|
|
66
|
+
|
|
67
|
+
extractors = PROFILE.fetch(format.to_sym, PROFILE[:auto])
|
|
68
|
+
suppressions = []
|
|
69
|
+
open_blocks = []
|
|
70
|
+
last_line = 1
|
|
71
|
+
|
|
72
|
+
source.each_line.with_index(1) do |line, lineno|
|
|
73
|
+
last_line = lineno
|
|
74
|
+
directives_on(line, extractors).each do |action, words|
|
|
75
|
+
case action
|
|
76
|
+
when :disable_line
|
|
77
|
+
suppressions << Suppression.new(
|
|
78
|
+
kind: :line, directive_line: lineno,
|
|
79
|
+
start_line: lineno, end_line: lineno
|
|
80
|
+
)
|
|
81
|
+
when :disable_next_line
|
|
82
|
+
suppressions << Suppression.new(
|
|
83
|
+
kind: :next_line, directive_line: lineno,
|
|
84
|
+
start_line: lineno + 1, end_line: lineno + 1,
|
|
85
|
+
words: words
|
|
86
|
+
)
|
|
87
|
+
when :disable_file
|
|
88
|
+
open_blocks << lineno
|
|
89
|
+
when :enable_file
|
|
90
|
+
close_block(open_blocks.pop, lineno, suppressions)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
open_blocks.each { |start| push_file_block(start, last_line, suppressions) }
|
|
96
|
+
suppressions.freeze
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# All directives found on +line+, in order, as
|
|
102
|
+
# [action_symbol, word_list] pairs.
|
|
103
|
+
def directives_on(line, extractors)
|
|
104
|
+
directives = []
|
|
105
|
+
extractors.each do |extractor|
|
|
106
|
+
comment_bodies(line, extractor).each do |body|
|
|
107
|
+
match = body.match(DIRECTIVE_PATTERN)
|
|
108
|
+
next unless match
|
|
109
|
+
|
|
110
|
+
action = match[1].tr("-", "_").to_sym
|
|
111
|
+
words = action == :disable_next_line ? match[2].split : []
|
|
112
|
+
directives << [action, words]
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
directives
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Comment bodies on +line+ for one extractor kind.
|
|
119
|
+
def comment_bodies(line, extractor)
|
|
120
|
+
case extractor
|
|
121
|
+
when :bare
|
|
122
|
+
[line.strip]
|
|
123
|
+
when :html_comment
|
|
124
|
+
line.scan(HTML_COMMENT_BODIES).flatten
|
|
125
|
+
when :line_comment
|
|
126
|
+
match = line.match(ASCII_DOC_COMMENT_BODY)
|
|
127
|
+
match ? [match[:body]] : []
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def close_block(start_line, enable_line, suppressions)
|
|
132
|
+
return unless start_line
|
|
133
|
+
|
|
134
|
+
# The enable-file line itself belongs to the block:
|
|
135
|
+
# directive text is never spellchecked.
|
|
136
|
+
push_file_block(start_line, [enable_line, start_line].max, suppressions)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def push_file_block(start_line, end_line, suppressions)
|
|
140
|
+
suppressions << Suppression.new(
|
|
141
|
+
kind: :file, directive_line: start_line,
|
|
142
|
+
start_line: start_line, end_line: end_line
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
module Suppressions
|
|
6
|
+
# One inline ignore directive, resolved to the 1-based source
|
|
7
|
+
# lines it covers. A plain value object — never serialized.
|
|
8
|
+
#
|
|
9
|
+
# Kinds:
|
|
10
|
+
# - +:line+ — `kotoshu:disable-line`, covers its own line
|
|
11
|
+
# - +:next_line+ — `kotoshu:disable-next-line [WORDS]`, covers
|
|
12
|
+
# the following line (only the listed words when a word list
|
|
13
|
+
# is present)
|
|
14
|
+
# - +:file+ — a `kotoshu:disable-file` / `kotoshu:enable-file`
|
|
15
|
+
# block, covers the whole span (to EOF when never re-enabled)
|
|
16
|
+
#
|
|
17
|
+
# The directive line itself is always suppressed for every
|
|
18
|
+
# word: the directive text is an instruction, not prose to
|
|
19
|
+
# check.
|
|
20
|
+
class Suppression
|
|
21
|
+
KINDS = %i[line next_line file].freeze
|
|
22
|
+
|
|
23
|
+
attr_reader :kind, :words, :directive_line, :start_line, :end_line
|
|
24
|
+
|
|
25
|
+
# @param kind [Symbol] one of KINDS
|
|
26
|
+
# @param directive_line [Integer] 1-based line the directive is on
|
|
27
|
+
# @param start_line [Integer] first covered line (inclusive)
|
|
28
|
+
# @param end_line [Integer] last covered line (inclusive)
|
|
29
|
+
# @param words [Array<String>, nil] word list (downcased);
|
|
30
|
+
# empty means "every word"
|
|
31
|
+
def initialize(kind:, directive_line:, start_line:, end_line:, words: [])
|
|
32
|
+
raise ArgumentError, "unknown kind: #{kind.inspect}" unless KINDS.include?(kind)
|
|
33
|
+
raise ArgumentError, "directive_line must be >= 1" if directive_line < 1
|
|
34
|
+
raise ArgumentError, "start_line must be >= 1" if start_line < 1
|
|
35
|
+
raise ArgumentError, "end_line must cover start_line" if end_line < start_line
|
|
36
|
+
|
|
37
|
+
@kind = kind
|
|
38
|
+
@directive_line = directive_line
|
|
39
|
+
@start_line = start_line
|
|
40
|
+
@end_line = end_line
|
|
41
|
+
@words = Array(words).map { |w| w.to_s.downcase }.reject(&:empty?).freeze
|
|
42
|
+
freeze
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# A `kotoshu:disable-file` block.
|
|
46
|
+
def file_block?
|
|
47
|
+
kind == :file
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# True when this suppression covers +line+ at all (word scope
|
|
51
|
+
# not considered).
|
|
52
|
+
def covers_line?(line)
|
|
53
|
+
line >= start_line && line <= end_line
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# True when only specifically listed words are suppressed.
|
|
57
|
+
def word_scoped?
|
|
58
|
+
!words.empty?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# True when +word+ reported on 1-based +line+ is suppressed.
|
|
62
|
+
# Passing +word: nil+ asks "could any word on this line be
|
|
63
|
+
# suppressed?" — true when the line is the directive line or
|
|
64
|
+
# the suppression is not word-scoped.
|
|
65
|
+
def applies_to?(line, word: nil)
|
|
66
|
+
return true if line == directive_line
|
|
67
|
+
return false unless covers_line?(line)
|
|
68
|
+
return true if words.empty?
|
|
69
|
+
|
|
70
|
+
word.nil? || words.include?(word.to_s.downcase)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
# Inline ignore directives (plan 82, Track A).
|
|
6
|
+
#
|
|
7
|
+
# A user can mark intentionally-"wrong" words (product names, code
|
|
8
|
+
# identifiers in prose, dialect) as ignored directly in the
|
|
9
|
+
# document, instead of polluting the personal dictionary:
|
|
10
|
+
#
|
|
11
|
+
# kotoshu:disable-line suppress everything on this line
|
|
12
|
+
# kotoshu:disable-next-line suppress the following line
|
|
13
|
+
# kotoshu:disable-next-line foo bar suppress only these words
|
|
14
|
+
# kotoshu:disable-file / enable-file block suppression
|
|
15
|
+
#
|
|
16
|
+
# Each document format recognizes the directive inside its own
|
|
17
|
+
# comment syntax: HTML comments in Markdown
|
|
18
|
+
# (`<!-- kotoshu:disable-next-line -->`), `//` line comments in
|
|
19
|
+
# AsciiDoc, and bare lines in plain text. Trailing directives are
|
|
20
|
+
# supported in Markdown and AsciiDoc.
|
|
21
|
+
#
|
|
22
|
+
# ONE shared scanner implements the semantics; only comment
|
|
23
|
+
# detection is per-format (see {Scanner}). Documents attach the
|
|
24
|
+
# resulting {Suppression} records during parse, and the
|
|
25
|
+
# Spellchecker facade filters suppressed words out of its results
|
|
26
|
+
# (they surface via `DocumentResult#suppressed_errors`).
|
|
27
|
+
module Suppressions
|
|
28
|
+
autoload :Suppression, "kotoshu/documents/suppressions/suppression"
|
|
29
|
+
autoload :Scanner, "kotoshu/documents/suppressions/scanner"
|
|
30
|
+
|
|
31
|
+
# Scan +source+ for ignore directives using the comment syntax
|
|
32
|
+
# of +format+ (:plain, :markdown, :asciidoc, or :auto for all
|
|
33
|
+
# three — the facade default, since it sees raw text without a
|
|
34
|
+
# parser).
|
|
35
|
+
#
|
|
36
|
+
# @param source [String, nil]
|
|
37
|
+
# @param format [Symbol]
|
|
38
|
+
# @return [Array<Suppression>] frozen, possibly empty
|
|
39
|
+
def self.scan(source, format: :auto)
|
|
40
|
+
Scanner.scan(source, format: format)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Documents
|
|
5
|
+
# A contiguous run of flattened text plus its source position.
|
|
6
|
+
#
|
|
7
|
+
# The checker scans the concatenation of every text node's +text+
|
|
8
|
+
# (the document's flattened form). Each node carries the {SourceRange}
|
|
9
|
+
# it occupies in the original markup-bearing source, plus its
|
|
10
|
+
# +flattened_offset+ (where the node starts in the concatenated
|
|
11
|
+
# flattened text) so the document can map errors back to source.
|
|
12
|
+
#
|
|
13
|
+
# +format+ and +metadata+ are escape hatches for plugins that want
|
|
14
|
+
# to attach structural context (e.g. `format: :heading,
|
|
15
|
+
# metadata: { level: 2 }` for an h2). Kotoshu's core checker
|
|
16
|
+
# ignores them; consumers (editors, reporters) can use them to
|
|
17
|
+
# surface context-aware UI.
|
|
18
|
+
TextNode = Struct.new(:text, :source_range, :flattened_offset, :format, :metadata,
|
|
19
|
+
keyword_init: true) do
|
|
20
|
+
def initialize(text:, source_range:, flattened_offset:, format: :plain, metadata: {})
|
|
21
|
+
raise TypeError, "source_range must be a SourceRange" unless source_range.is_a?(SourceRange)
|
|
22
|
+
raise ArgumentError, "flattened_offset must be >= 0" if flattened_offset.negative?
|
|
23
|
+
|
|
24
|
+
super
|
|
25
|
+
freeze
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Length of the flattened text in this node.
|
|
29
|
+
def length
|
|
30
|
+
text.length
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Flattened-text range covered by this node: [flattened_offset,
|
|
34
|
+
# flattened_offset + text.length).
|
|
35
|
+
def flattened_range
|
|
36
|
+
flattened_offset...(flattened_offset + text.length)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# True when +offset+ falls within this node's flattened range.
|
|
40
|
+
def contains_flattened?(offset)
|
|
41
|
+
flattened_range.include?(offset)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def to_s
|
|
45
|
+
"#{format}(#{text.inspect} @ #{source_range})"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Document model for structure-aware spell and grammar checking.
|
|
5
|
+
#
|
|
6
|
+
# The Document abstraction pairs the *flattened* text a checker scans
|
|
7
|
+
# with the *source* positions of each text run, so that errors can be
|
|
8
|
+
# reported against the original markup-bearing source rather than the
|
|
9
|
+
# stripped text. The canonical example is an AsciiDoc or Markdown
|
|
10
|
+
# sentence like `I'm an **friend** of Tom`: the checker sees the
|
|
11
|
+
# flattened `I'm an friend of Tom` and flags "an friend", but the
|
|
12
|
+
# error report needs to point at the original `an **friend**` range
|
|
13
|
+
# so an editor can highlight what the user actually wrote.
|
|
14
|
+
#
|
|
15
|
+
# Kotoshu ships only the value-object layer and a trivial
|
|
16
|
+
# {PlainTextDocument}. Format-specific parsers (Markdown, AsciiDoc,
|
|
17
|
+
# reStructuredText, etc.) live in plugins — kotoshu never owns
|
|
18
|
+
# document parsing (see the `kotoshu-document-plugin-boundary` design
|
|
19
|
+
# memory). Plugins register a parser class via {Documents.register}
|
|
20
|
+
# and produce {Document} instances whose {TextNode}s carry proper
|
|
21
|
+
# {SourceRange}s.
|
|
22
|
+
module Documents
|
|
23
|
+
autoload :SourcePosition, "kotoshu/documents/source_position"
|
|
24
|
+
autoload :SourceRange, "kotoshu/documents/source_range"
|
|
25
|
+
autoload :TextNode, "kotoshu/documents/text_node"
|
|
26
|
+
autoload :Document, "kotoshu/documents/document"
|
|
27
|
+
autoload :PlainTextDocument, "kotoshu/documents/plain_text_document"
|
|
28
|
+
autoload :Suppressions, "kotoshu/documents/suppressions"
|
|
29
|
+
|
|
30
|
+
@parsers = {}
|
|
31
|
+
@discovered_formats = []
|
|
32
|
+
@discovered_plugins_loaded = false
|
|
33
|
+
|
|
34
|
+
class << self
|
|
35
|
+
# Register a document parser for a format symbol.
|
|
36
|
+
#
|
|
37
|
+
# The parser class must respond to:
|
|
38
|
+
#
|
|
39
|
+
# .from_string(text, language_code:) -> Documents::Document
|
|
40
|
+
#
|
|
41
|
+
# Optionally also:
|
|
42
|
+
#
|
|
43
|
+
# .from_file(path, language_code:) -> Documents::Document
|
|
44
|
+
#
|
|
45
|
+
# ==== Parser contract ====
|
|
46
|
+
#
|
|
47
|
+
# The returned {Document}'s +text_nodes+ must each carry a
|
|
48
|
+
# {Documents::SourceRange} that points at the original
|
|
49
|
+
# markup-bearing source for that node. This is the structure-aware
|
|
50
|
+
# contract — the analyzer reads the document's flattened text
|
|
51
|
+
# to run spelling/grammar checks, but every error is reported
|
|
52
|
+
# against the original source so an editor or plugin can
|
|
53
|
+
# highlight the user's actual markup, not the stripped text.
|
|
54
|
+
#
|
|
55
|
+
# Example: for the source
|
|
56
|
+
#
|
|
57
|
+
# "I'm an **friend** of Tom"
|
|
58
|
+
#
|
|
59
|
+
# the parser produces TextNodes whose SourceRanges cover the
|
|
60
|
+
# original range, not the flattened "I'm an friend of Tom".
|
|
61
|
+
# A grammar error "an friend" → "a friend" then carries a
|
|
62
|
+
# SourceRange pointing at "an **friend**" (the bold span plus the
|
|
63
|
+
# preceding "an ").
|
|
64
|
+
#
|
|
65
|
+
# ==== Format symbol ====
|
|
66
|
+
#
|
|
67
|
+
# +format+ is the canonical symbol callers pass to {.parse}. The
|
|
68
|
+
# gem ships +:plain+ (via {PlainTextDocument}). Plugins
|
|
69
|
+
# commonly add +:markdown+, +:asciidoc+, +:rst+, +:latex+, etc.
|
|
70
|
+
# Format symbols are global — two plugins registering the same
|
|
71
|
+
# symbol is a name clash; the last one wins.
|
|
72
|
+
#
|
|
73
|
+
# ==== Discovery ====
|
|
74
|
+
#
|
|
75
|
+
# Plugins ship a file under +kotoshu_plugin/document/*.rb+ in
|
|
76
|
+
# their gem. The first lookup of {.parser_for} / {.parse} walks
|
|
77
|
+
# +Gem.find_files+ for those files and requires each one. Each
|
|
78
|
+
# file's body is expected to call {.register} for the formats
|
|
79
|
+
# it provides. See {.discovered_plugin_files} and
|
|
80
|
+
# {.discovered_formats}.
|
|
81
|
+
#
|
|
82
|
+
# @param format [Symbol] e.g. :plain, :markdown, :asciidoc
|
|
83
|
+
# @param parser_class [Class] responds to .from_string
|
|
84
|
+
# @return [void]
|
|
85
|
+
def register(format, parser_class)
|
|
86
|
+
@parsers[format.to_sym] = parser_class
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Look up the registered parser for a format.
|
|
90
|
+
#
|
|
91
|
+
# @param format [Symbol]
|
|
92
|
+
# @return [Class, nil]
|
|
93
|
+
def parser_for(format)
|
|
94
|
+
ensure_plugins_discovered!
|
|
95
|
+
@parsers[format.to_sym]
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# List every registered format symbol.
|
|
99
|
+
#
|
|
100
|
+
# @return [Array<Symbol>]
|
|
101
|
+
def registered_formats
|
|
102
|
+
ensure_plugins_discovered!
|
|
103
|
+
@parsers.keys
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Parse a source string with the registered parser for +format+,
|
|
107
|
+
# falling back to {PlainTextDocument} when no parser is
|
|
108
|
+
# registered. Never raises — the fallback is intentional so
|
|
109
|
+
# callers can pass arbitrary format hints without first checking
|
|
110
|
+
# the registry.
|
|
111
|
+
#
|
|
112
|
+
# @param source [String]
|
|
113
|
+
# @param format [Symbol]
|
|
114
|
+
# @param language_code [String, nil]
|
|
115
|
+
# @return [Document]
|
|
116
|
+
def parse(source, format:, language_code: nil)
|
|
117
|
+
ensure_plugins_discovered!
|
|
118
|
+
parser = parser_for(format) || PlainTextDocument
|
|
119
|
+
parser.from_string(source, language_code: language_code)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# List every plugin file found on the load path via
|
|
123
|
+
# +Gem.find_files("kotoshu_plugin/document/*.rb")+. Each such
|
|
124
|
+
# file, when required, is expected to call {register} for the
|
|
125
|
+
# formats it provides. The list is the raw file paths; not
|
|
126
|
+
# loaded into the registry yet.
|
|
127
|
+
#
|
|
128
|
+
# @return [Array<String>]
|
|
129
|
+
def discovered_plugin_files
|
|
130
|
+
Gem.find_files("kotoshu_plugin/document/*.rb").sort
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# List every format symbol that came from an auto-discovered
|
|
134
|
+
# plugin (vs. one registered explicitly at runtime). Useful for
|
|
135
|
+
# diagnostics and "which plugin registered this format?" UX.
|
|
136
|
+
#
|
|
137
|
+
# @return [Array<Symbol>]
|
|
138
|
+
def discovered_formats
|
|
139
|
+
ensure_plugins_discovered!
|
|
140
|
+
@discovered_formats.dup.freeze
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Clear every registration. Test-only — production code should
|
|
144
|
+
# register once at load time. Also resets the discovery flag so
|
|
145
|
+
# the next lookup re-runs Gem.find_files.
|
|
146
|
+
#
|
|
147
|
+
# @return [void]
|
|
148
|
+
def reset!
|
|
149
|
+
@parsers.clear
|
|
150
|
+
@discovered_formats.clear
|
|
151
|
+
@discovered_plugins_loaded = false
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
|
|
156
|
+
# Walk Gem.find_files for plugin files at most once per
|
|
157
|
+
# process. Each file is +require+d; the file's body is expected
|
|
158
|
+
# to call {register}. Idempotent — re-entry is a no-op once the
|
|
159
|
+
# flag is set.
|
|
160
|
+
#
|
|
161
|
+
# The set of formats registered by each plugin file is captured
|
|
162
|
+
# by snapshotting the registry before and after the require.
|
|
163
|
+
def ensure_plugins_discovered!
|
|
164
|
+
return if @discovered_plugins_loaded
|
|
165
|
+
|
|
166
|
+
@discovered_plugins_loaded = true
|
|
167
|
+
before = @parsers.keys
|
|
168
|
+
discovered_plugin_files.each do |path|
|
|
169
|
+
require path
|
|
170
|
+
rescue StandardError, LoadError => e
|
|
171
|
+
warn "kotoshu: failed to load document plugin #{path}: #{e.message}"
|
|
172
|
+
end
|
|
173
|
+
after = @parsers.keys
|
|
174
|
+
@discovered_formats.concat(after - before)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|