kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "digest"
|
|
5
|
+
|
|
6
|
+
module Kotoshu
|
|
7
|
+
module Integrity
|
|
8
|
+
# Parsed view of a content repo's `manifest.json`.
|
|
9
|
+
#
|
|
10
|
+
# Format (per TODO.impl/09-integrity-security.md task 1):
|
|
11
|
+
#
|
|
12
|
+
# {
|
|
13
|
+
# "version": 1,
|
|
14
|
+
# "generated_at": "2026-06-25T10:00:00Z",
|
|
15
|
+
# "resources": {
|
|
16
|
+
# "en/spelling/index.dic": {
|
|
17
|
+
# "size": 49568,
|
|
18
|
+
# "sha256": "ab12...",
|
|
19
|
+
# "language": "en",
|
|
20
|
+
# "type": "spelling",
|
|
21
|
+
# "license": "LGPL/MPL/GPL",
|
|
22
|
+
# "source": "SCROLL"
|
|
23
|
+
# }
|
|
24
|
+
# }
|
|
25
|
+
# }
|
|
26
|
+
#
|
|
27
|
+
# Construction:
|
|
28
|
+
#
|
|
29
|
+
# manifest = Manifest.parse(json_string)
|
|
30
|
+
# manifest.fetch("en/spelling/index.dic") # => Entry or nil
|
|
31
|
+
# manifest.verify_content!("en/spelling/index.dic", bytes) # raises on mismatch
|
|
32
|
+
#
|
|
33
|
+
# `Manifest.load(url, http:)`, returns nil when the manifest 404s (graceful
|
|
34
|
+
# degradation — see module docs).
|
|
35
|
+
class Manifest
|
|
36
|
+
Entry = Struct.new(:path, :sha256, :size, :language, :type, :license, :source,
|
|
37
|
+
keyword_init: true) do
|
|
38
|
+
def verify?(content)
|
|
39
|
+
Digest::SHA256.hexdigest(content) == sha256
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Parse a manifest JSON string. Returns an empty Manifest if the JSON
|
|
44
|
+
# is parseable but has no resources (caller treats as "no constraints").
|
|
45
|
+
def self.parse(json)
|
|
46
|
+
data = JSON.parse(json)
|
|
47
|
+
entries = {}
|
|
48
|
+
(data["resources"] || {}).each do |path, fields|
|
|
49
|
+
entries[path] = Entry.new(
|
|
50
|
+
path: path,
|
|
51
|
+
sha256: fields["sha256"],
|
|
52
|
+
size: fields["size"],
|
|
53
|
+
language: fields["language"],
|
|
54
|
+
type: fields["type"],
|
|
55
|
+
license: fields["license"],
|
|
56
|
+
source: fields["source"]
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
new(entries, version: data["version"], generated_at: data["generated_at"])
|
|
60
|
+
rescue JSON::ParserError => e
|
|
61
|
+
raise Kotoshu::IntegrityError.new(
|
|
62
|
+
"manifest",
|
|
63
|
+
expected: "<valid JSON>",
|
|
64
|
+
actual: "<parse error: #{e.message}>"
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Fetch and parse a manifest from a URL. Returns nil when the
|
|
69
|
+
# manifest is absent (HTTP 404/410) so callers can fall back to
|
|
70
|
+
# unverified downloads — see module docs. Any other failure
|
|
71
|
+
# (5xx, network error, parse error) raises.
|
|
72
|
+
def self.load(url, http: Kotoshu::Integrity::NetHTTP)
|
|
73
|
+
body = http.get(url)
|
|
74
|
+
return nil if body.nil?
|
|
75
|
+
|
|
76
|
+
parse(body)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
attr_reader :entries, :version, :generated_at
|
|
80
|
+
|
|
81
|
+
def initialize(entries, version: nil, generated_at: nil)
|
|
82
|
+
@entries = entries
|
|
83
|
+
@version = version
|
|
84
|
+
@generated_at = generated_at
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def fetch(path)
|
|
88
|
+
@entries[path]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def empty?
|
|
92
|
+
@entries.empty?
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Verify that content for `path` matches the manifest entry.
|
|
96
|
+
# Raises {Kotoshu::IntegrityError} on mismatch. No-op when the
|
|
97
|
+
# manifest has no entry for `path` (returns nil — caller decides
|
|
98
|
+
# whether to treat absence as failure in strict mode).
|
|
99
|
+
def verify_content!(path, content, url: nil)
|
|
100
|
+
entry = @entries[path]
|
|
101
|
+
return nil unless entry
|
|
102
|
+
|
|
103
|
+
actual = Digest::SHA256.hexdigest(content)
|
|
104
|
+
unless actual == entry.sha256
|
|
105
|
+
raise Kotoshu::IntegrityError.new(
|
|
106
|
+
path,
|
|
107
|
+
expected: entry.sha256,
|
|
108
|
+
actual: actual,
|
|
109
|
+
url: url
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
true
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "uri"
|
|
5
|
+
|
|
6
|
+
module Kotoshu
|
|
7
|
+
module Integrity
|
|
8
|
+
# Thin wrapper around Net::HTTP so manifest fetches are testable
|
|
9
|
+
# without the network. Returns response body as a String on 2xx,
|
|
10
|
+
# nil on 404/410 (so callers can treat "manifest not published yet"
|
|
11
|
+
# as graceful degradation), and raises on other errors.
|
|
12
|
+
module NetHTTP
|
|
13
|
+
class << self
|
|
14
|
+
def get(url, redirect_limit: 3)
|
|
15
|
+
if Kotoshu.configuration.offline
|
|
16
|
+
raise HttpError,
|
|
17
|
+
"GET #{url} refused: offline mode is enabled (KOTOSHU_OFFLINE=1)"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
uri = URI(url)
|
|
21
|
+
raise ArgumentError, "Only http/https supported: #{url}" unless
|
|
22
|
+
["http", "https"].include?(uri.scheme)
|
|
23
|
+
|
|
24
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
25
|
+
http.use_ssl = (uri.scheme == "https")
|
|
26
|
+
http.open_timeout = 10
|
|
27
|
+
http.read_timeout = 30
|
|
28
|
+
|
|
29
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
|
30
|
+
response = http.request(request)
|
|
31
|
+
|
|
32
|
+
case response
|
|
33
|
+
when Net::HTTPSuccess
|
|
34
|
+
response.body
|
|
35
|
+
when Net::HTTPNotFound, Net::HTTPGone
|
|
36
|
+
nil
|
|
37
|
+
when Net::HTTPRedirection
|
|
38
|
+
raise TooManyRedirects if redirect_limit.zero?
|
|
39
|
+
|
|
40
|
+
get(response["location"], redirect_limit: redirect_limit - 1)
|
|
41
|
+
else
|
|
42
|
+
raise HttpError, "GET #{url} failed: #{response.code} #{response.message}"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
class HttpError < StandardError; end
|
|
48
|
+
class TooManyRedirects < StandardError; end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Integrity
|
|
5
|
+
# Pure value object that decides when an {AuditLog} should rotate
|
|
6
|
+
# and produces the rename plan that realises the rotation.
|
|
7
|
+
#
|
|
8
|
+
# The policy is intentionally IO-free so it can be unit-tested in
|
|
9
|
+
# isolation. {AuditLog#record} consults the policy on every write
|
|
10
|
+
# and executes the returned plan via +FileUtils+.
|
|
11
|
+
#
|
|
12
|
+
# Rotation scheme: keep +rotations+ historical files alongside the
|
|
13
|
+
# current one. On rotation:
|
|
14
|
+
#
|
|
15
|
+
# 1. Drop the oldest rotation slot (<path>.<rotations>).
|
|
16
|
+
# 2. Shift each remaining rotation up by one suffix.
|
|
17
|
+
# 3. Promote the current log to <path>.1.
|
|
18
|
+
#
|
|
19
|
+
# Net effect: current + N rotated files, total bounded at
|
|
20
|
+
# +max_bytes * (rotations + 1)+.
|
|
21
|
+
class RotationPolicy
|
|
22
|
+
DEFAULT_MAX_BYTES = 10 * 1024 * 1024 # 10 MB
|
|
23
|
+
DEFAULT_ROTATIONS = 5
|
|
24
|
+
|
|
25
|
+
attr_reader :max_bytes, :rotations
|
|
26
|
+
|
|
27
|
+
def initialize(max_bytes: DEFAULT_MAX_BYTES, rotations: DEFAULT_ROTATIONS)
|
|
28
|
+
@max_bytes = max_bytes.to_i
|
|
29
|
+
@rotations = rotations.to_i
|
|
30
|
+
raise ArgumentError, "max_bytes must be >= 0" if @max_bytes.negative?
|
|
31
|
+
raise ArgumentError, "rotations must be >= 0" if @rotations.negative?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# True when the current file size exceeds the configured ceiling.
|
|
35
|
+
#
|
|
36
|
+
# @param current_size [Integer] Bytes of the current log file.
|
|
37
|
+
def rotate?(current_size)
|
|
38
|
+
current_size > max_bytes
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Plan the file operations required to rotate +path+.
|
|
42
|
+
#
|
|
43
|
+
# Returns a hash with two keys:
|
|
44
|
+
#
|
|
45
|
+
# +:deletes+ — Array<String> of paths to remove (+FileUtils.rm_f+).
|
|
46
|
+
# +:moves+ — Array<[String, String]> of (source, dest) pairs
|
|
47
|
+
# (+FileUtils.mv+).
|
|
48
|
+
#
|
|
49
|
+
# The plan is ordered so that a caller executing the deletes first
|
|
50
|
+
# and then the moves in array order will not clobber any still-live
|
|
51
|
+
# rotation. Sources that happen not to exist are silently skipped
|
|
52
|
+
# by +FileUtils.mv+, so the plan can be applied verbatim.
|
|
53
|
+
#
|
|
54
|
+
# When +rotations+ is zero, the plan deletes the current path; the
|
|
55
|
+
# caller's next append recreates it as an empty file (effectively a
|
|
56
|
+
# truncate).
|
|
57
|
+
#
|
|
58
|
+
# @param path [String] The current log path (e.g. "/x/audit.log").
|
|
59
|
+
def plan_for(path)
|
|
60
|
+
if rotations.positive?
|
|
61
|
+
{
|
|
62
|
+
deletes: [rotated_path(path, rotations)],
|
|
63
|
+
moves: shift_moves(path).push([path, rotated_path(path, 1)])
|
|
64
|
+
}
|
|
65
|
+
else
|
|
66
|
+
{ deletes: [path], moves: [] }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# Pairs that shift every existing rotation up by one suffix.
|
|
73
|
+
# Ordered oldest-first so the rename chain doesn't overwrite
|
|
74
|
+
# in-flight files. For +rotations = 5+ this yields:
|
|
75
|
+
#
|
|
76
|
+
# [[p.4, p.5], [p.3, p.4], [p.2, p.3], [p.1, p.2]]
|
|
77
|
+
def shift_moves(path)
|
|
78
|
+
(rotations - 1).downto(1).map do |n|
|
|
79
|
+
[rotated_path(path, n), rotated_path(path, n + 1)]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def rotated_path(path, suffix)
|
|
84
|
+
"#{path}.#{suffix}"
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Integrity verification for downloaded resources.
|
|
5
|
+
#
|
|
6
|
+
# Two cooperating pieces:
|
|
7
|
+
#
|
|
8
|
+
# - {Manifest} — parsed view of a content repo's `manifest.json`. Lookup
|
|
9
|
+
# by relative path yields the expected SHA-256 and size.
|
|
10
|
+
# - {AuditLog} — append-only JSON log at `$XDG_DATA_HOME/kotoshu/audit.log` recording
|
|
11
|
+
# every download's URL, size, computed SHA-256, manifest hash (when
|
|
12
|
+
# available), and status.
|
|
13
|
+
#
|
|
14
|
+
# Caches call `Manifest.verify_content!` after each download. If the
|
|
15
|
+
# manifest is absent (the upstream repo hasn't shipped one yet), the
|
|
16
|
+
# caller logs the download with status `"unverified"` and proceeds —
|
|
17
|
+
# graceful degradation. When a manifest IS present and the SHA-256
|
|
18
|
+
# doesn't match, {Kotoshu::IntegrityError} is raised and the corrupt
|
|
19
|
+
# bytes are removed before they reach the cache.
|
|
20
|
+
module Integrity
|
|
21
|
+
autoload :Manifest, "kotoshu/integrity/manifest"
|
|
22
|
+
autoload :AuditLog, "kotoshu/integrity/audit_log"
|
|
23
|
+
autoload :NetHTTP, "kotoshu/integrity/net_http"
|
|
24
|
+
autoload :RotationPolicy, "kotoshu/integrity/rotation_policy"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "kotoshu"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
# Jekyll integration (plan 89, item 4). Soft dependency: jekyll is
|
|
7
|
+
# NOT in kotoshu.gemspec; the generator soft-requires it exactly
|
|
8
|
+
# like Models::OnnxModel soft-requires onnxruntime.
|
|
9
|
+
module Jekyll
|
|
10
|
+
# True when jekyll was requireable at load time.
|
|
11
|
+
JEKYLL_LOADED = begin
|
|
12
|
+
require "jekyll"
|
|
13
|
+
true
|
|
14
|
+
rescue LoadError
|
|
15
|
+
false
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Raised when the generator is used without jekyll.
|
|
19
|
+
class JekyllUnavailable < Kotoshu::Error
|
|
20
|
+
def initialize
|
|
21
|
+
super("Kotoshu's Jekyll generator needs the jekyll gem. " \
|
|
22
|
+
"Add gem 'jekyll' to your Gemfile.")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if JEKYLL_LOADED
|
|
27
|
+
# Jekyll generator checking posts and drafts; a new spelling
|
|
28
|
+
# error fails the build. Baselines are respected when
|
|
29
|
+
# .kotoshu-baseline.json sits in the site source (the same file
|
|
30
|
+
# `kotoshu baseline init` writes), so existing errors do not
|
|
31
|
+
# block the build until they are fixed.
|
|
32
|
+
#
|
|
33
|
+
# @example _config.yml
|
|
34
|
+
# plugins:
|
|
35
|
+
# - kotoshu
|
|
36
|
+
#
|
|
37
|
+
# Users register this class in a plugins file:
|
|
38
|
+
# require "kotoshu/jekyll"
|
|
39
|
+
class Generator < ::Jekyll::Generator
|
|
40
|
+
safe true
|
|
41
|
+
priority :low
|
|
42
|
+
|
|
43
|
+
def generate(site)
|
|
44
|
+
documents = site.posts.docs
|
|
45
|
+
if documents.empty?
|
|
46
|
+
::Jekyll.logger.info "kotoshu:", "no posts or drafts to check"
|
|
47
|
+
return
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
baseline = load_baseline(site)
|
|
51
|
+
failures = []
|
|
52
|
+
documents.each do |document|
|
|
53
|
+
# Same plan-116 budget as the CLI/rake paths: covered
|
|
54
|
+
# occurrences skip the suggestion sweep.
|
|
55
|
+
filter = baseline&.suggestions_filter_for(document.relative_path.to_s)
|
|
56
|
+
result = Kotoshu.check(document.content, suggestions_filter: filter)
|
|
57
|
+
application = baseline&.apply(result, file: document.relative_path.to_s)
|
|
58
|
+
result = application.result if application
|
|
59
|
+
failures.concat(failure_lines(document, result))
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
return if failures.empty?
|
|
63
|
+
|
|
64
|
+
raise "kotoshu: spelling errors in #{failures.size} location(s):\n" +
|
|
65
|
+
failures.join("\n")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# Load the baseline from the site source when present.
|
|
71
|
+
#
|
|
72
|
+
# @param site [Jekyll::Site] the site being built
|
|
73
|
+
# @return [Baseline::Store, nil]
|
|
74
|
+
def load_baseline(site)
|
|
75
|
+
path = File.join(site.source, Baseline::Store::DEFAULT_FILENAME)
|
|
76
|
+
File.exist?(path) ? Baseline::Store.load(path) : nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# One failure line per misspelling.
|
|
80
|
+
#
|
|
81
|
+
# @param document [Jekyll::Document] the checked document
|
|
82
|
+
# @param result [Models::Result::DocumentResult] check result
|
|
83
|
+
# @return [Array<String>]
|
|
84
|
+
def failure_lines(document, result)
|
|
85
|
+
result.each_error.map do |error|
|
|
86
|
+
suggestions = error.has_suggestions? ? " -> #{error.top_suggestions(3).join(', ')}" : ""
|
|
87
|
+
" #{document.relative_path}: #{error.word}#{suggestions}"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
else
|
|
92
|
+
# Placeholder raising a friendly error when jekyll is missing;
|
|
93
|
+
# keeps the constant resolvable for tooling.
|
|
94
|
+
class Generator
|
|
95
|
+
def generate(*)
|
|
96
|
+
raise JekyllUnavailable
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
# Base class for keyboard layouts
|
|
6
|
+
#
|
|
7
|
+
# Each layout defines key positions and provides distance calculations
|
|
8
|
+
# for typo detection and suggestion ranking in spell checking.
|
|
9
|
+
#
|
|
10
|
+
# @example Using a keyboard layout
|
|
11
|
+
# layout = Keyboard::Layouts::QWERTY.new
|
|
12
|
+
# layout.distance('q', 'w') # => 1 (adjacent keys)
|
|
13
|
+
# layout.distance('q', 'p') # => 8 (far apart)
|
|
14
|
+
# layout.adjacent_keys('q') # => ['w', 'a', 's']
|
|
15
|
+
#
|
|
16
|
+
# @example Checking language support
|
|
17
|
+
# qwerty = Keyboard::Layouts::QWERTY.new
|
|
18
|
+
# qwerty.supports_language?('en') # => true
|
|
19
|
+
# qwerty.supports_language?('de') # => false
|
|
20
|
+
#
|
|
21
|
+
class Layout
|
|
22
|
+
# @return [String] the name of this keyboard layout
|
|
23
|
+
attr_reader :name
|
|
24
|
+
|
|
25
|
+
# @return [Array<String>] list of language codes this layout supports
|
|
26
|
+
attr_reader :language_codes
|
|
27
|
+
|
|
28
|
+
# @return [Hash] mapping of key characters to [row, col] positions
|
|
29
|
+
attr_reader :key_positions
|
|
30
|
+
|
|
31
|
+
# Initialize a keyboard layout
|
|
32
|
+
#
|
|
33
|
+
# @param name [String] the name of the layout
|
|
34
|
+
# @param language_codes [Array<String>] list of language codes this layout supports
|
|
35
|
+
# @param key_positions [Hash] mapping of key characters to [row, col] positions
|
|
36
|
+
def initialize(name:, language_codes:, key_positions:)
|
|
37
|
+
@name = name
|
|
38
|
+
@language_codes = Array(language_codes).freeze
|
|
39
|
+
@key_positions = key_positions.freeze
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Get position [row, col] for a key
|
|
43
|
+
#
|
|
44
|
+
# @param key [String] the key character to look up
|
|
45
|
+
# @return [Array<Integer>, nil] the [row, col] position, or nil if key not found
|
|
46
|
+
def position(key)
|
|
47
|
+
@key_positions[key.downcase]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Calculate Manhattan distance between two keys
|
|
51
|
+
#
|
|
52
|
+
# Manhattan distance is the sum of absolute differences of row and column:
|
|
53
|
+
# distance = abs(row1 - row2) + abs(col1 - col2)
|
|
54
|
+
#
|
|
55
|
+
# @param key1 [String] first key character
|
|
56
|
+
# @param key2 [String] second key character
|
|
57
|
+
# @return [Integer] Manhattan distance (0 if same key, Float::INFINITY if either key not found)
|
|
58
|
+
def distance(key1, key2)
|
|
59
|
+
pos1 = position(key1)
|
|
60
|
+
pos2 = position(key2)
|
|
61
|
+
|
|
62
|
+
return Float::INFINITY unless pos1 && pos2
|
|
63
|
+
|
|
64
|
+
(pos1[0] - pos2[0]).abs + (pos1[1] - pos2[1]).abs
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Check if layout supports a language
|
|
68
|
+
#
|
|
69
|
+
# Supports both exact matching and language variant matching.
|
|
70
|
+
# For example, if 'en' is supported, then 'en-US', 'en-GB', etc. are also supported.
|
|
71
|
+
#
|
|
72
|
+
# @param language_code [String] the language code to check (e.g., 'en', 'en-US', 'de')
|
|
73
|
+
# @return [Boolean] true if this layout supports the language
|
|
74
|
+
def supports_language?(language_code)
|
|
75
|
+
# Try exact match first
|
|
76
|
+
return true if @language_codes.include?(language_code)
|
|
77
|
+
|
|
78
|
+
# Try base language match (e.g., 'en' for 'en-US')
|
|
79
|
+
base_lang = language_code.to_s.split('-').first
|
|
80
|
+
@language_codes.include?(base_lang)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Get adjacent keys for a given key (within 1 unit distance)
|
|
84
|
+
#
|
|
85
|
+
# Adjacent keys are those that are directly next to the given key
|
|
86
|
+
# horizontally or vertically (not diagonal).
|
|
87
|
+
#
|
|
88
|
+
# @param key [String] the key character to find adjacent keys for
|
|
89
|
+
# @return [Array<String>] list of adjacent key characters
|
|
90
|
+
def adjacent_keys(key)
|
|
91
|
+
pos = position(key)
|
|
92
|
+
return [] unless pos
|
|
93
|
+
|
|
94
|
+
@key_positions.select do |k, p|
|
|
95
|
+
next if k == key
|
|
96
|
+
|
|
97
|
+
((p[0] - pos[0]).abs + (p[1] - pos[1]).abs) == 1
|
|
98
|
+
end.keys
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# String representation of the layout
|
|
102
|
+
#
|
|
103
|
+
# @return [String] layout name
|
|
104
|
+
def to_s
|
|
105
|
+
"Keyboard::#{@name}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Inspect method for debugging
|
|
109
|
+
#
|
|
110
|
+
# @return [String] detailed inspection string
|
|
111
|
+
def inspect
|
|
112
|
+
"#<#{self.class} name=#{@name} languages=#{@language_codes.join(',')}>"
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# Arabic 101 keyboard layout (standard IBM PC Arabic layout).
|
|
7
|
+
#
|
|
8
|
+
# The Arabic letters sit on the QWERTY key faces: the number row
|
|
9
|
+
# is shared with ASCII, and ض ص ث ق ف غ ع ه خ ح ج د / ش س ي ب ل ا
|
|
10
|
+
# ت ن م ك ط / ئ ء ؤ ر لا ى ة و ز ظ fill the three letter rows.
|
|
11
|
+
# Diacritics (tashkeel) are dead keys and are not in the grid;
|
|
12
|
+
# the hamza carriers أ إ آ and the ه/ة ي/ى spelling slips are
|
|
13
|
+
# handled at the normalizer level, not as physical keys.
|
|
14
|
+
#
|
|
15
|
+
# The key grid is mirrored from the models repo eval harness
|
|
16
|
+
# (kotoshu/models-fasttext-onnx eval/noise.py _AR_101, plan 83
|
|
17
|
+
# batch 2 of the model coverage expansion, verified there
|
|
18
|
+
# against the kbdlayout.info Arabic 101 driver table) so
|
|
19
|
+
# typo-adjacency used for suggestion ranking matches the
|
|
20
|
+
# adjacency used to evaluate the models.
|
|
21
|
+
# spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
|
|
22
|
+
# the sync.
|
|
23
|
+
#
|
|
24
|
+
# Languages: ar, ar-SA
|
|
25
|
+
class Arabic101 < Layout
|
|
26
|
+
# Key positions for the Arabic 101 layout.
|
|
27
|
+
# Each key maps to [row, column] coordinates.
|
|
28
|
+
KEY_POSITIONS = {
|
|
29
|
+
# Number row
|
|
30
|
+
'ذ' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
|
|
31
|
+
'4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
|
|
32
|
+
'8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '-' => [0, 11],
|
|
33
|
+
'=' => [0, 12],
|
|
34
|
+
# Top row (ض ص ث ق ف غ ع ه خ ح ج د)
|
|
35
|
+
'ض' => [1, 0], 'ص' => [1, 1], 'ث' => [1, 2], 'ق' => [1, 3],
|
|
36
|
+
'ف' => [1, 4], 'غ' => [1, 5], 'ع' => [1, 6], 'ه' => [1, 7],
|
|
37
|
+
'خ' => [1, 8], 'ح' => [1, 9], 'ج' => [1, 10], 'د' => [1, 11],
|
|
38
|
+
# Home row (ش س ي ب ل ا ت ن م ك ط)
|
|
39
|
+
'ش' => [2, 0], 'س' => [2, 1], 'ي' => [2, 2], 'ب' => [2, 3],
|
|
40
|
+
'ل' => [2, 4], 'ا' => [2, 5], 'ت' => [2, 6], 'ن' => [2, 7],
|
|
41
|
+
'م' => [2, 8], 'ك' => [2, 9], 'ط' => [2, 10],
|
|
42
|
+
# Bottom row (ئ ء ؤ ر لا ى ة و ز ظ)
|
|
43
|
+
'ئ' => [3, 0], 'ء' => [3, 1], 'ؤ' => [3, 2], 'ر' => [3, 3],
|
|
44
|
+
'لا' => [3, 4], 'ى' => [3, 5], 'ة' => [3, 6], 'و' => [3, 7],
|
|
45
|
+
'ز' => [3, 8], 'ظ' => [3, 9]
|
|
46
|
+
}.freeze
|
|
47
|
+
|
|
48
|
+
# Initialize the Arabic 101 layout.
|
|
49
|
+
def initialize
|
|
50
|
+
super(
|
|
51
|
+
name: 'Arabic-101',
|
|
52
|
+
language_codes: %w[ar ar-SA],
|
|
53
|
+
key_positions: KEY_POSITIONS
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# AZERTY keyboard layout
|
|
7
|
+
#
|
|
8
|
+
# Standard AZERTY layout used for:
|
|
9
|
+
# - French (fr, fr-FR)
|
|
10
|
+
# - Belgium (be)
|
|
11
|
+
#
|
|
12
|
+
# Key differences from QWERTY:
|
|
13
|
+
# - a and q are swapped (a/q → q/a)
|
|
14
|
+
# - z and w are swapped (z/w → w/z)
|
|
15
|
+
# - Number row is shifted (requires Shift for numbers)
|
|
16
|
+
# - Has accent keys: é, à, ç, è, ù
|
|
17
|
+
#
|
|
18
|
+
# Key positions use [row, col] coordinates where:
|
|
19
|
+
# - row 0: number/symbol row (²&é"'(-...)
|
|
20
|
+
# - row 1: top row (azerty...)
|
|
21
|
+
# - row 2: home row (qsdfg...)
|
|
22
|
+
# - row 3: bottom row (wxcvb...) - note: w is here
|
|
23
|
+
class AZERTY < Layout
|
|
24
|
+
# Key positions for AZERTY layout
|
|
25
|
+
# Each key maps to [row, column] coordinates
|
|
26
|
+
KEY_POSITIONS = {
|
|
27
|
+
# Top row (number/symbol row - numbers require Shift)
|
|
28
|
+
'`' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3], '4' => [0, 4],
|
|
29
|
+
'5' => [0, 5], '6' => [0, 6], '7' => [0, 7], '8' => [0, 8], '9' => [0, 9],
|
|
30
|
+
'0' => [0, 10], ')' => [0, 11], '=' => [0, 12],
|
|
31
|
+
# Top row (AZERTY - note a and q swapped, z and w swapped)
|
|
32
|
+
'a' => [1, 0], 'z' => [1, 1], 'e' => [1, 2], 'r' => [1, 3], 't' => [1, 4],
|
|
33
|
+
'y' => [1, 5], 'u' => [1, 6], 'i' => [1, 7], 'o' => [1, 8], 'p' => [1, 9],
|
|
34
|
+
'^' => [1, 10], '$' => [1, 11],
|
|
35
|
+
# Home row (QSDFG - note q is here)
|
|
36
|
+
'q' => [2, 0], 's' => [2, 1], 'd' => [2, 2], 'f' => [2, 3], 'g' => [2, 4],
|
|
37
|
+
'h' => [2, 5], 'j' => [2, 6], 'k' => [2, 7], 'l' => [2, 8], 'm' => [2, 9],
|
|
38
|
+
'ù' => [2, 10],
|
|
39
|
+
# Bottom row (WXCVB - note w is here)
|
|
40
|
+
'w' => [3, 0], 'x' => [3, 1], 'c' => [3, 2], 'v' => [3, 3], 'b' => [3, 4],
|
|
41
|
+
'n' => [3, 5], ',' => [3, 6], ';' => [3, 7], ':' => [3, 8], '!' => [3, 9]
|
|
42
|
+
}.freeze
|
|
43
|
+
|
|
44
|
+
# Initialize AZERTY layout
|
|
45
|
+
def initialize
|
|
46
|
+
super(
|
|
47
|
+
name: 'AZERTY',
|
|
48
|
+
language_codes: %w[fr be fr-FR],
|
|
49
|
+
key_positions: KEY_POSITIONS
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Keyboard
|
|
5
|
+
module Layouts
|
|
6
|
+
# Bulgarian BDS keyboard layout (БДС 5237:1978, KBDBUL.DLL).
|
|
7
|
+
#
|
|
8
|
+
# The national Bulgarian phonetic-adjacent ordering: ы у е и ш щ
|
|
9
|
+
# к с д з ц / ь я а о ж г т н в м ч / ю й ъ э ф х п р л б. The
|
|
10
|
+
# Bulgarian-only letters ъ ь are real keys (unlike JCUKEN, where
|
|
11
|
+
# ъ sits top-right and ы is the home-row opener).
|
|
12
|
+
#
|
|
13
|
+
# The key grid is mirrored from the models repo eval harness
|
|
14
|
+
# (kotoshu/models-fasttext-onnx eval/noise.py _BG_BDS, plan 83
|
|
15
|
+
# batch 2 of the model coverage expansion, verified there
|
|
16
|
+
# against the kbdlayout.info KBDBUL driver table) so
|
|
17
|
+
# typo-adjacency used for suggestion ranking matches the
|
|
18
|
+
# adjacency used to evaluate the models.
|
|
19
|
+
# spec/kotoshu/keyboard/layouts/eval_grid_drift_spec.rb enforces
|
|
20
|
+
# the sync.
|
|
21
|
+
#
|
|
22
|
+
# Languages: bg, bg-BG
|
|
23
|
+
class BulgarianBds < Layout
|
|
24
|
+
# Key positions for the Bulgarian BDS layout.
|
|
25
|
+
# Each key maps to [row, column] coordinates.
|
|
26
|
+
KEY_POSITIONS = {
|
|
27
|
+
# Number row
|
|
28
|
+
')' => [0, 0], '1' => [0, 1], '2' => [0, 2], '3' => [0, 3],
|
|
29
|
+
'4' => [0, 4], '5' => [0, 5], '6' => [0, 6], '7' => [0, 7],
|
|
30
|
+
'8' => [0, 8], '9' => [0, 9], '0' => [0, 10], '$' => [0, 11],
|
|
31
|
+
'€' => [0, 12],
|
|
32
|
+
# Top row (ы у е и ш щ к с д з ц)
|
|
33
|
+
'ы' => [1, 0], 'у' => [1, 1], 'е' => [1, 2], 'и' => [1, 3],
|
|
34
|
+
'ш' => [1, 4], 'щ' => [1, 5], 'к' => [1, 6], 'с' => [1, 7],
|
|
35
|
+
'д' => [1, 8], 'з' => [1, 9], 'ц' => [1, 10],
|
|
36
|
+
# Home row (ь я а о ж г т н в м ч)
|
|
37
|
+
'ь' => [2, 0], 'я' => [2, 1], 'а' => [2, 2], 'о' => [2, 3],
|
|
38
|
+
'ж' => [2, 4], 'г' => [2, 5], 'т' => [2, 6], 'н' => [2, 7],
|
|
39
|
+
'в' => [2, 8], 'м' => [2, 9], 'ч' => [2, 10],
|
|
40
|
+
# Bottom row (ю й ъ э ф х п р л б)
|
|
41
|
+
'ю' => [3, 0], 'й' => [3, 1], 'ъ' => [3, 2], 'э' => [3, 3],
|
|
42
|
+
'ф' => [3, 4], 'х' => [3, 5], 'п' => [3, 6], 'р' => [3, 7],
|
|
43
|
+
'л' => [3, 8], 'б' => [3, 9]
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
# Initialize the Bulgarian BDS layout.
|
|
47
|
+
def initialize
|
|
48
|
+
super(
|
|
49
|
+
name: 'Bulgarian-BDS',
|
|
50
|
+
language_codes: %w[bg bg-BG],
|
|
51
|
+
key_positions: KEY_POSITIONS
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|