kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for sentence-start capitalization rules.
|
|
7
|
+
#
|
|
8
|
+
# English (and most Latin-script languages) capitalizes the
|
|
9
|
+
# first word of every sentence. This matcher walks the token
|
|
10
|
+
# stream tracking sentence boundaries and flags lowercase
|
|
11
|
+
# sentence starts.
|
|
12
|
+
#
|
|
13
|
+
# A sentence starts:
|
|
14
|
+
# - at the beginning of the stream, OR
|
|
15
|
+
# - immediately after a token whose final character is a
|
|
16
|
+
# sentence-ending punctuation mark (. ! ?).
|
|
17
|
+
#
|
|
18
|
+
# The matcher only considers alphabetic tokens. Numbers,
|
|
19
|
+
# symbols, and whitespace don't reset the sentence-start flag.
|
|
20
|
+
#
|
|
21
|
+
# Rule shape:
|
|
22
|
+
#
|
|
23
|
+
# patterns:
|
|
24
|
+
# - conditions:
|
|
25
|
+
# - type: sentence_start_check
|
|
26
|
+
#
|
|
27
|
+
class SentenceStartMatcher < BaseMatcher
|
|
28
|
+
SENTENCE_ENDERS = %w[. ! ?].freeze
|
|
29
|
+
|
|
30
|
+
def match(tokens, rule)
|
|
31
|
+
errors = []
|
|
32
|
+
expecting_capital = true
|
|
33
|
+
|
|
34
|
+
tokens.each do |token|
|
|
35
|
+
word = token[:token]
|
|
36
|
+
next if word.nil? || word.empty?
|
|
37
|
+
|
|
38
|
+
if sentence_start_word?(word, expecting_capital)
|
|
39
|
+
errors << build_error(token, rule)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Any non-empty token resets the sentence-start expectation.
|
|
43
|
+
# Only sentence-ending punctuation re-arms it (below).
|
|
44
|
+
expecting_capital = false
|
|
45
|
+
expecting_capital = true if ends_sentence?(word)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
errors
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
protected
|
|
52
|
+
|
|
53
|
+
def sentence_start_word?(word, expecting_capital)
|
|
54
|
+
expecting_capital && alphabetic_word?(word) && first_letter_lowercase?(word)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def alphabetic_word?(word)
|
|
58
|
+
!word.nil? && !word.empty? && word[0].match?(/[[:alpha:]]/)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def first_letter_lowercase?(word)
|
|
62
|
+
first = word[0]
|
|
63
|
+
first == first.downcase && first != first.upcase
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def ends_sentence?(word)
|
|
67
|
+
SENTENCE_ENDERS.include?(word[-1])
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def build_error(token, rule)
|
|
71
|
+
capitalized = token[:token].sub(/\A(\w)/) { $1.upcase }
|
|
72
|
+
{
|
|
73
|
+
rule_id: rule.id,
|
|
74
|
+
position: token[:position],
|
|
75
|
+
message: rule.message,
|
|
76
|
+
suggestion: capitalized,
|
|
77
|
+
context: "\"#{token[:token]}\"",
|
|
78
|
+
suggestions: [capitalized]
|
|
79
|
+
}
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for a/an article usage rules.
|
|
7
|
+
#
|
|
8
|
+
# This matcher checks if "a" or "an" is used correctly before
|
|
9
|
+
# vowel and consonant sounds.
|
|
10
|
+
class VowelSoundMatcher < BaseMatcher
|
|
11
|
+
VOWEL_SOUNDS = %w[a e i o u].freeze
|
|
12
|
+
|
|
13
|
+
# Match tokens against the a/an pattern.
|
|
14
|
+
#
|
|
15
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
16
|
+
# @param rule [Rule] The rule being checked
|
|
17
|
+
# @return [Array<Hash>] Array of error hashes
|
|
18
|
+
def match(tokens, rule)
|
|
19
|
+
errors = []
|
|
20
|
+
tokens.each_cons(2) do |prev_token, current_token|
|
|
21
|
+
prev_word = prev_token[:token]&.downcase
|
|
22
|
+
next unless %w[a an].include?(prev_word)
|
|
23
|
+
next unless prev_token[:pos_tag] == 'DET' || prev_token[:pos_tag].nil?
|
|
24
|
+
|
|
25
|
+
next_word = current_token[:token]
|
|
26
|
+
next if next_word.nil? || next_word.empty?
|
|
27
|
+
|
|
28
|
+
expected = article_for(next_word, rule)
|
|
29
|
+
if prev_word != expected
|
|
30
|
+
errors << build_error(prev_token, current_token, expected, rule)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
errors
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
# Determine the correct article for a word.
|
|
39
|
+
#
|
|
40
|
+
# @param word [String] The word to check
|
|
41
|
+
# @param rule [Rule] The rule with exceptions
|
|
42
|
+
# @return [String] "a" or "an"
|
|
43
|
+
def article_for(word, rule)
|
|
44
|
+
word_downcase = word.downcase
|
|
45
|
+
exceptions = rule.exceptions || {}
|
|
46
|
+
|
|
47
|
+
consonant_exceptions = exceptions['consonant_sound_exceptions'] || []
|
|
48
|
+
return 'a' if consonant_exceptions.include?(word_downcase)
|
|
49
|
+
|
|
50
|
+
silent_exceptions = exceptions['silent_consonant_exceptions'] || []
|
|
51
|
+
return 'an' if silent_exceptions.include?(word_downcase)
|
|
52
|
+
|
|
53
|
+
first_char = word_downcase[0]
|
|
54
|
+
VOWEL_SOUNDS.include?(first_char) ? 'an' : 'a'
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Build an error hash.
|
|
58
|
+
#
|
|
59
|
+
# @param prev_token [Hash] The previous token (article)
|
|
60
|
+
# @param current_token [Hash] The current token (word)
|
|
61
|
+
# @param expected [String] The expected article
|
|
62
|
+
# @param rule [Rule] The rule being checked
|
|
63
|
+
# @return [Hash] Error hash
|
|
64
|
+
def build_error(prev_token, current_token, expected, rule)
|
|
65
|
+
prev_word = prev_token[:token]
|
|
66
|
+
next_word = current_token[:token]
|
|
67
|
+
message = rule.message.gsub('{expected}', expected).gsub('{word}', next_word)
|
|
68
|
+
|
|
69
|
+
{
|
|
70
|
+
rule_id: rule.id,
|
|
71
|
+
position: prev_token[:position],
|
|
72
|
+
message: message,
|
|
73
|
+
suggestion: expected,
|
|
74
|
+
context: "#{prev_word} #{next_word}",
|
|
75
|
+
suggestions: [expected]
|
|
76
|
+
}
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
module PatternMatchers
|
|
6
|
+
# Matcher for word-list corrections.
|
|
7
|
+
#
|
|
8
|
+
# Flags any token whose lowercase form is a key in the rule's
|
|
9
|
+
# +corrections+ map. Useful for proper-noun capitalization
|
|
10
|
+
# (monday → Monday, english → English) where a single rule
|
|
11
|
+
# covers dozens of word pairs.
|
|
12
|
+
#
|
|
13
|
+
# Rule shape:
|
|
14
|
+
#
|
|
15
|
+
# patterns:
|
|
16
|
+
# - conditions:
|
|
17
|
+
# - type: word_list_check
|
|
18
|
+
# corrections:
|
|
19
|
+
# monday: Monday
|
|
20
|
+
# tuesday: Tuesday
|
|
21
|
+
# english: English
|
|
22
|
+
#
|
|
23
|
+
# The matcher only fires when the actual token text differs from
|
|
24
|
+
# the mapped correction (so "Monday" doesn't get flagged).
|
|
25
|
+
#
|
|
26
|
+
class WordListMatcher < BaseMatcher
|
|
27
|
+
def match(tokens, rule)
|
|
28
|
+
corrections = word_list_condition&.dig('corrections') || {}
|
|
29
|
+
return [] if corrections.empty?
|
|
30
|
+
|
|
31
|
+
errors = []
|
|
32
|
+
tokens.each do |token|
|
|
33
|
+
word = token[:token]
|
|
34
|
+
next unless word
|
|
35
|
+
|
|
36
|
+
correction = corrections[word.downcase]
|
|
37
|
+
next unless correction
|
|
38
|
+
next if word == correction # already correct
|
|
39
|
+
|
|
40
|
+
errors << build_error(token, correction, rule)
|
|
41
|
+
end
|
|
42
|
+
errors
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def word_list_condition
|
|
48
|
+
@word_list_condition ||= @pattern['conditions']&.find do |c|
|
|
49
|
+
c['type'] == 'word_list_check'
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def build_error(token, correction, rule)
|
|
54
|
+
{
|
|
55
|
+
rule_id: rule.id,
|
|
56
|
+
position: token[:position],
|
|
57
|
+
message: rule.message,
|
|
58
|
+
suggestion: correction,
|
|
59
|
+
context: "\"#{token[:token]}\"",
|
|
60
|
+
suggestions: [correction]
|
|
61
|
+
}
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
# Base class for grammar rules.
|
|
6
|
+
#
|
|
7
|
+
# All grammar rules inherit from this class and implement
|
|
8
|
+
# the #check method to validate tokens.
|
|
9
|
+
class Rule
|
|
10
|
+
attr_reader :id, :name, :category, :severity, :description,
|
|
11
|
+
:exceptions, :message, :suggestion
|
|
12
|
+
|
|
13
|
+
def initialize(id:, name:, category:, severity:, description:,
|
|
14
|
+
patterns:, message:, suggestion:, exceptions: {})
|
|
15
|
+
@id = id
|
|
16
|
+
@name = name
|
|
17
|
+
@category = category
|
|
18
|
+
@severity = severity
|
|
19
|
+
@description = description
|
|
20
|
+
@patterns = patterns
|
|
21
|
+
@exceptions = exceptions
|
|
22
|
+
@message = message
|
|
23
|
+
@suggestion = suggestion
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Factory method to create Rule from YAML configuration.
|
|
27
|
+
#
|
|
28
|
+
# @param config [Hash] YAML configuration hash
|
|
29
|
+
# @return [Rule] A new rule instance
|
|
30
|
+
def self.from_yaml(config)
|
|
31
|
+
new(
|
|
32
|
+
id: config['id'],
|
|
33
|
+
name: config['name'],
|
|
34
|
+
category: config['category'],
|
|
35
|
+
severity: config['severity'],
|
|
36
|
+
description: config['description'],
|
|
37
|
+
patterns: config['patterns'],
|
|
38
|
+
exceptions: config['exceptions'] || {},
|
|
39
|
+
message: config['message'],
|
|
40
|
+
suggestion: config['suggestion']
|
|
41
|
+
)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Check tokens against this rule.
|
|
45
|
+
#
|
|
46
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
47
|
+
# @return [Array<Hash>] Array of error hashes
|
|
48
|
+
def check(tokens)
|
|
49
|
+
errors = []
|
|
50
|
+
@patterns.each do |pattern|
|
|
51
|
+
pattern_errors = check_pattern(tokens, pattern)
|
|
52
|
+
errors.concat(pattern_errors)
|
|
53
|
+
end
|
|
54
|
+
errors
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Check a single pattern against tokens.
|
|
60
|
+
#
|
|
61
|
+
# @param tokens [Array<Hash>] Array of token hashes
|
|
62
|
+
# @param pattern [Hash] Pattern configuration hash
|
|
63
|
+
# @return [Array<Hash>] Array of error hashes
|
|
64
|
+
def check_pattern(tokens, pattern)
|
|
65
|
+
matcher = create_matcher(pattern)
|
|
66
|
+
matcher.match(tokens, self)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Create appropriate pattern matcher based on pattern type.
|
|
70
|
+
#
|
|
71
|
+
# @param pattern [Hash] Pattern configuration hash
|
|
72
|
+
# @return [PatternMatchers::BaseMatcher] A pattern matcher instance
|
|
73
|
+
def create_matcher(pattern)
|
|
74
|
+
conditions = pattern['conditions'] || []
|
|
75
|
+
return PatternMatchers::BaseMatcher.new(pattern) if conditions.empty?
|
|
76
|
+
|
|
77
|
+
condition_types = conditions.map { |c| c['type'] }
|
|
78
|
+
|
|
79
|
+
if condition_types.include?('vowel_check')
|
|
80
|
+
PatternMatchers::VowelSoundMatcher.new(pattern, exceptions)
|
|
81
|
+
elsif condition_types.include?('context_check')
|
|
82
|
+
PatternMatchers::PossessiveContextMatcher.new(pattern, exceptions)
|
|
83
|
+
elsif condition_types.include?('possessive_contraction_check')
|
|
84
|
+
PatternMatchers::PossessiveContractionMatcher.new(pattern, exceptions)
|
|
85
|
+
elsif condition_types.include?('distance_check')
|
|
86
|
+
PatternMatchers::DoubleNegativeMatcher.new(pattern, exceptions)
|
|
87
|
+
elsif condition_types.include?('phrase_check')
|
|
88
|
+
PatternMatchers::PhraseMatcher.new(pattern, exceptions)
|
|
89
|
+
elsif condition_types.include?('sentence_start_check')
|
|
90
|
+
PatternMatchers::SentenceStartMatcher.new(pattern, exceptions)
|
|
91
|
+
elsif condition_types.include?('word_list_check')
|
|
92
|
+
PatternMatchers::WordListMatcher.new(pattern, exceptions)
|
|
93
|
+
else
|
|
94
|
+
PatternMatchers::BaseMatcher.new(pattern)
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Grammar
|
|
5
|
+
# Engine for loading and executing grammar rules from YAML configuration.
|
|
6
|
+
#
|
|
7
|
+
# This implements configuration-driven design where all linguistic data
|
|
8
|
+
# (rules, patterns, exceptions) is stored in YAML files, not hardcoded.
|
|
9
|
+
#
|
|
10
|
+
# @example Loading rules for English
|
|
11
|
+
# engine = RuleEngine.new(language: 'en')
|
|
12
|
+
# errors = engine.check(tokens)
|
|
13
|
+
#
|
|
14
|
+
class RuleEngine
|
|
15
|
+
attr_reader :language, :rules
|
|
16
|
+
|
|
17
|
+
# Create a new rule engine for a language.
|
|
18
|
+
#
|
|
19
|
+
# @param language [String] Language code (e.g., 'en', 'de', 'fr')
|
|
20
|
+
# @param rules_path [String, nil] Optional custom path to grammar rules
|
|
21
|
+
# @param dictionaries_path [String, nil] Optional custom path to dictionaries directory
|
|
22
|
+
def initialize(language:, rules_path: nil, dictionaries_path: nil)
|
|
23
|
+
@language = language
|
|
24
|
+
@rules_path = rules_path || default_rules_path(dictionaries_path)
|
|
25
|
+
@loader = RuleLoader.new(@rules_path)
|
|
26
|
+
@rules = @loader.load_rules
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Check tokens against all loaded rules.
|
|
30
|
+
#
|
|
31
|
+
# @param tokens [Array<Hash>] Array of token hashes with :token, :pos_tag, :position keys
|
|
32
|
+
# @return [Array<Hash>] Array of error hashes
|
|
33
|
+
def check(tokens)
|
|
34
|
+
errors = []
|
|
35
|
+
@rules.each do |rule|
|
|
36
|
+
rule_errors = rule.check(tokens)
|
|
37
|
+
errors.concat(rule_errors)
|
|
38
|
+
end
|
|
39
|
+
errors
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Get list of rule IDs.
|
|
43
|
+
#
|
|
44
|
+
# @return [Array<String>] Array of rule IDs
|
|
45
|
+
def rule_names
|
|
46
|
+
@rules.map(&:id)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Get a specific rule by ID.
|
|
50
|
+
#
|
|
51
|
+
# @param id [String] Rule ID
|
|
52
|
+
# @return [Rule, nil] The rule if found, nil otherwise
|
|
53
|
+
def get_rule(id)
|
|
54
|
+
@rules.find { |r| r.id == id }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Check if a rule exists.
|
|
58
|
+
#
|
|
59
|
+
# @param id [String] Rule ID
|
|
60
|
+
# @return [Boolean] True if rule exists
|
|
61
|
+
def rule_exists?(id)
|
|
62
|
+
@rules.any? { |r| r.id == id }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# Resolve the directory holding <language>/grammar/rules.yaml.
|
|
68
|
+
#
|
|
69
|
+
# Priority:
|
|
70
|
+
# 1. Explicit `dictionaries_path:` constructor arg
|
|
71
|
+
# 2. KOTOSHU_DICTIONARIES_PATH env var
|
|
72
|
+
# 3. Configuration#dictionaries_path (when set)
|
|
73
|
+
# 4. Gem-bundled data/grammar (always available)
|
|
74
|
+
#
|
|
75
|
+
# @param dictionaries_path [String, nil] Caller-supplied override
|
|
76
|
+
# @return [String] Path joined with language + "grammar"
|
|
77
|
+
def default_rules_path(dictionaries_path = nil)
|
|
78
|
+
base = dictionaries_path || resolve_dictionaries_base
|
|
79
|
+
File.join(base, @language, "grammar")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def resolve_dictionaries_base
|
|
83
|
+
ENV["KOTOSHU_DICTIONARIES_PATH"] ||
|
|
84
|
+
Configuration.instance.dictionaries_path ||
|
|
85
|
+
gem_bundled_data_path
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Path to the gem's own data/ directory. Always available after
|
|
89
|
+
# `gem install kotoshu`, so grammar rules work without external
|
|
90
|
+
# dependencies or sibling repos checked out.
|
|
91
|
+
def gem_bundled_data_path
|
|
92
|
+
File.expand_path("../../../data", __dir__)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'yaml'
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Grammar
|
|
7
|
+
# Loads grammar rules from YAML configuration files.
|
|
8
|
+
#
|
|
9
|
+
# This class reads rule definitions from YAML files in the
|
|
10
|
+
# dictionaries/{language}/grammar/ directory.
|
|
11
|
+
class RuleLoader
|
|
12
|
+
def initialize(rules_path)
|
|
13
|
+
@rules_path = rules_path
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Load all rules from the rules.yaml file.
|
|
17
|
+
#
|
|
18
|
+
# @return [Array<Rule>] Array of rule instances
|
|
19
|
+
def load_rules
|
|
20
|
+
rules_file = File.join(@rules_path, 'rules.yaml')
|
|
21
|
+
return [] unless File.exist?(rules_file)
|
|
22
|
+
|
|
23
|
+
config = YAML.load_file(rules_file)
|
|
24
|
+
return [] unless config && config['rules']
|
|
25
|
+
|
|
26
|
+
config['rules'].map { |rule_config| Rule.from_yaml(rule_config) }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Grammar rules infrastructure for Kotoshu.
|
|
5
|
+
#
|
|
6
|
+
# This module provides configuration-driven grammar checking
|
|
7
|
+
# where all linguistic data is stored in YAML files.
|
|
8
|
+
module Grammar
|
|
9
|
+
autoload :Rule, "kotoshu/grammar/rule"
|
|
10
|
+
autoload :RuleEngine, "kotoshu/grammar/rule_engine"
|
|
11
|
+
autoload :RuleLoader, "kotoshu/grammar/rule_loader"
|
|
12
|
+
|
|
13
|
+
module PatternMatchers
|
|
14
|
+
autoload :BaseMatcher, "kotoshu/grammar/pattern_matchers/base_matcher"
|
|
15
|
+
autoload :VowelSoundMatcher, "kotoshu/grammar/pattern_matchers/vowel_sound_matcher"
|
|
16
|
+
autoload :PossessiveContextMatcher, "kotoshu/grammar/pattern_matchers/possessive_context_matcher"
|
|
17
|
+
autoload :PossessiveContractionMatcher, "kotoshu/grammar/pattern_matchers/possessive_contraction_matcher"
|
|
18
|
+
autoload :DoubleNegativeMatcher, "kotoshu/grammar/pattern_matchers/double_negative_matcher"
|
|
19
|
+
autoload :PhraseMatcher, "kotoshu/grammar/pattern_matchers/phrase_matcher"
|
|
20
|
+
autoload :SentenceStartMatcher, "kotoshu/grammar/pattern_matchers/sentence_start_matcher"
|
|
21
|
+
autoload :WordListMatcher, "kotoshu/grammar/pattern_matchers/word_list_matcher"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
|
|
7
|
+
module Kotoshu
|
|
8
|
+
module Integrity
|
|
9
|
+
# Append-only JSON audit log of every resource download.
|
|
10
|
+
#
|
|
11
|
+
# Each entry is one JSON object per line, written to +audit.log+
|
|
12
|
+
# under the configured Kotoshu home directory. The log is consulted
|
|
13
|
+
# by users when investigating "what did Kotoshu fetch?" and by CI
|
|
14
|
+
# for reproducibility audits.
|
|
15
|
+
#
|
|
16
|
+
# Statuses:
|
|
17
|
+
# "verified" — content matched manifest entry's SHA-256
|
|
18
|
+
# "unverified" — no manifest entry available; bytes trusted as-is
|
|
19
|
+
# "mismatch" — SHA-256 mismatch (also raises IntegrityError)
|
|
20
|
+
# "missing" — attempted download failed (network, 404, etc.)
|
|
21
|
+
#
|
|
22
|
+
# The log is opened, appended, and closed per entry — no long-lived
|
|
23
|
+
# file handle. Writes are line-buffered and fsync'd so the record
|
|
24
|
+
# survives a crash mid-batch.
|
|
25
|
+
#
|
|
26
|
+
# == Rotation
|
|
27
|
+
#
|
|
28
|
+
# When the current file exceeds +rotation_policy.max_bytes+, the
|
|
29
|
+
# policy produces a rename plan (see {RotationPolicy#plan_for}) and
|
|
30
|
+
# the log shifts the existing rotations up by one slot before
|
|
31
|
+
# writing the new entry to a fresh current file. The total on-disk
|
|
32
|
+
# footprint is bounded at +max_bytes * (rotations + 1)+.
|
|
33
|
+
#
|
|
34
|
+
# Rotation happens under an exclusive flock on a sibling lockfile
|
|
35
|
+
# (+audit.log.lock+) — not on the log itself — because the log path
|
|
36
|
+
# moves during rotation and the lock would otherwise travel with it.
|
|
37
|
+
class AuditLog
|
|
38
|
+
# Default location: $XDG_DATA_HOME/kotoshu/audit.log
|
|
39
|
+
# (~/.local/share/kotoshu/audit.log), or $KOTOSHU_AUDIT_LOG.
|
|
40
|
+
def self.default_path
|
|
41
|
+
Kotoshu::Paths.audit_log_path
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
attr_reader :path, :rotation_policy
|
|
45
|
+
|
|
46
|
+
# @param path [String] Override the log file location.
|
|
47
|
+
# @param rotation_policy [RotationPolicy, nil] When nil, no
|
|
48
|
+
# rotation is performed (the log grows unbounded). Pass a
|
|
49
|
+
# +RotationPolicy+ instance to enable bounded rotation.
|
|
50
|
+
def initialize(path: self.class.default_path, rotation_policy: nil)
|
|
51
|
+
@path = path
|
|
52
|
+
@rotation_policy = rotation_policy
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Record one download attempt. Returns the written entry hash.
|
|
56
|
+
#
|
|
57
|
+
# @param url [String] Source URL
|
|
58
|
+
# @param size [Integer, nil] Bytes downloaded (nil on missing)
|
|
59
|
+
# @param sha256 [String, nil] Computed SHA-256 of bytes (nil on missing)
|
|
60
|
+
# @param manifest_sha256 [String, nil] Expected SHA-256 from manifest
|
|
61
|
+
# @param status [String] One of: verified, unverified, mismatch, missing
|
|
62
|
+
# @param resource_id [String, nil] Caller-supplied resource identifier
|
|
63
|
+
def record(url:, status:, size: nil, sha256: nil,
|
|
64
|
+
manifest_sha256: nil, resource_id: nil)
|
|
65
|
+
entry = {
|
|
66
|
+
timestamp: Time.now.utc.iso8601,
|
|
67
|
+
url: url,
|
|
68
|
+
resource_id: resource_id,
|
|
69
|
+
size: size,
|
|
70
|
+
sha256: sha256,
|
|
71
|
+
manifest_sha256: manifest_sha256,
|
|
72
|
+
status: status
|
|
73
|
+
}
|
|
74
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
75
|
+
with_exclusive_lock do
|
|
76
|
+
rotate_if_needed!
|
|
77
|
+
File.open(@path, "a", encoding: "UTF-8") do |f|
|
|
78
|
+
f.write("#{entry.to_json}\n")
|
|
79
|
+
f.fsync
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
entry
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Iterate every recorded entry (parsed Hashes) across the current
|
|
86
|
+
# log and all rotations, newest-first.
|
|
87
|
+
#
|
|
88
|
+
# Newest-first means: within each file, lines are yielded in reverse
|
|
89
|
+
# write order (the last appended entry first); files are visited in
|
|
90
|
+
# order current, .1, .2, ... (newest file first). This guarantees a
|
|
91
|
+
# strictly decreasing write-time ordering across the whole log,
|
|
92
|
+
# which is what an auditor scanning "what did kotoshu fetch most
|
|
93
|
+
# recently?" expects.
|
|
94
|
+
def each(&block)
|
|
95
|
+
return enum_for(:each) unless block
|
|
96
|
+
|
|
97
|
+
each_file_reversed(@path, &block)
|
|
98
|
+
return unless rotation_policy&.rotations&.positive?
|
|
99
|
+
|
|
100
|
+
1.upto(rotation_policy.rotations) do |n|
|
|
101
|
+
each_file_reversed("#{@path}.#{n}", &block)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def entries
|
|
106
|
+
each.to_a
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def clear!
|
|
110
|
+
with_exclusive_lock do
|
|
111
|
+
FileUtils.rm_f(@path)
|
|
112
|
+
clear_rotations
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# Yield parsed entries from +path+ newest-first (reverse line order).
|
|
119
|
+
# No-op when the file doesn't exist. Each individual rotation file is
|
|
120
|
+
# bounded by +RotationPolicy.max_bytes+, so loading one file's lines
|
|
121
|
+
# into memory before reversing is acceptable.
|
|
122
|
+
def each_file_reversed(path)
|
|
123
|
+
return unless File.exist?(path)
|
|
124
|
+
|
|
125
|
+
File.readlines(path, encoding: "UTF-8")
|
|
126
|
+
.map(&:strip)
|
|
127
|
+
.reject(&:empty?)
|
|
128
|
+
.reverse_each { |line| yield JSON.parse(line) }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def clear_rotations
|
|
132
|
+
return unless rotation_policy&.rotations&.positive?
|
|
133
|
+
|
|
134
|
+
1.upto(rotation_policy.rotations) do |n|
|
|
135
|
+
FileUtils.rm_f("#{@path}.#{n}")
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def rotate_if_needed!
|
|
140
|
+
return unless rotation_policy
|
|
141
|
+
return unless File.exist?(@path)
|
|
142
|
+
return unless rotation_policy.rotate?(File.size(@path))
|
|
143
|
+
|
|
144
|
+
plan = rotation_policy.plan_for(@path)
|
|
145
|
+
plan[:deletes].each { |p| FileUtils.rm_f(p) }
|
|
146
|
+
plan[:moves].each do |src, dst|
|
|
147
|
+
FileUtils.mv(src, dst) if File.exist?(src)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Exclusive flock on a stable sibling lockfile. The log itself is
|
|
152
|
+
# a poor lock target because rotation renames it out from under
|
|
153
|
+
# any writer mid-flight.
|
|
154
|
+
def with_exclusive_lock
|
|
155
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
156
|
+
File.open(lockfile_path, File::RDWR | File::CREAT, 0o644) do |lock|
|
|
157
|
+
lock.flock(File::LOCK_EX)
|
|
158
|
+
yield
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def lockfile_path
|
|
163
|
+
"#{@path}.lock"
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|