kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Per-language implementations (English, German, French, etc.).
|
|
5
|
+
#
|
|
6
|
+
# Sibling module to {Language} (the framework). Each per-language file
|
|
7
|
+
# calls Language::Registry.register at load time; {Language::Registry.ensure_languages_loaded}
|
|
8
|
+
# triggers these autoloads on first registry access.
|
|
9
|
+
module Languages
|
|
10
|
+
autoload :Arabic, "kotoshu/languages/ar/language"
|
|
11
|
+
autoload :English, "kotoshu/languages/en/language"
|
|
12
|
+
autoload :French, "kotoshu/languages/fr/language"
|
|
13
|
+
autoload :German, "kotoshu/languages/de/language"
|
|
14
|
+
autoload :Hebrew, "kotoshu/languages/he/language"
|
|
15
|
+
autoload :Japanese, "kotoshu/languages/ja/language"
|
|
16
|
+
autoload :Persian, "kotoshu/languages/fa/language"
|
|
17
|
+
autoload :Portuguese, "kotoshu/languages/pt/language"
|
|
18
|
+
autoload :Russian, "kotoshu/languages/ru/language"
|
|
19
|
+
autoload :Spanish, "kotoshu/languages/es/language"
|
|
20
|
+
|
|
21
|
+
# Shared base for the thin wave-1 modules (plan 84).
|
|
22
|
+
autoload :LatinBase, "kotoshu/languages/latin_base"
|
|
23
|
+
# Wave-1 modules (plan 84, Track B): ca cs da el hu it nl pl ro
|
|
24
|
+
# sv tr uk vi.
|
|
25
|
+
autoload :Catalan, "kotoshu/languages/ca/language"
|
|
26
|
+
autoload :Czech, "kotoshu/languages/cs/language"
|
|
27
|
+
autoload :Danish, "kotoshu/languages/da/language"
|
|
28
|
+
autoload :Greek, "kotoshu/languages/el/language"
|
|
29
|
+
autoload :Hungarian, "kotoshu/languages/hu/language"
|
|
30
|
+
autoload :Italian, "kotoshu/languages/it/language"
|
|
31
|
+
autoload :Dutch, "kotoshu/languages/nl/language"
|
|
32
|
+
autoload :Polish, "kotoshu/languages/pl/language"
|
|
33
|
+
autoload :Romanian, "kotoshu/languages/ro/language"
|
|
34
|
+
autoload :Swedish, "kotoshu/languages/sv/language"
|
|
35
|
+
autoload :NorwegianBokmal, "kotoshu/languages/nb/language"
|
|
36
|
+
autoload :Turkish, "kotoshu/languages/tr/language"
|
|
37
|
+
autoload :Ukrainian, "kotoshu/languages/uk/language"
|
|
38
|
+
autoload :Vietnamese, "kotoshu/languages/vi/language"
|
|
39
|
+
# Batch-3 modules (plan 100): id hr sl sk et lt lv (Latin) and
|
|
40
|
+
# bg sr (Cyrillic). The pre-existing ar fa he modules become
|
|
41
|
+
# full-feature in this batch (layout + downloadable dictionary).
|
|
42
|
+
autoload :Bulgarian, "kotoshu/languages/bg/language"
|
|
43
|
+
autoload :Croatian, "kotoshu/languages/hr/language"
|
|
44
|
+
autoload :Estonian, "kotoshu/languages/et/language"
|
|
45
|
+
autoload :Indonesian, "kotoshu/languages/id/language"
|
|
46
|
+
autoload :Latvian, "kotoshu/languages/lv/language"
|
|
47
|
+
autoload :Lithuanian, "kotoshu/languages/lt/language"
|
|
48
|
+
autoload :Serbian, "kotoshu/languages/sr/language"
|
|
49
|
+
autoload :Slovak, "kotoshu/languages/sk/language"
|
|
50
|
+
autoload :Slovenian, "kotoshu/languages/sl/language"
|
|
51
|
+
# Nynorsk module (plan 107): wired as full feature while basic
|
|
52
|
+
# support opened the staged manifest (dictionary + model staged).
|
|
53
|
+
autoload :NorwegianNynorsk, "kotoshu/languages/nn/language"
|
|
54
|
+
# ko and ne modules (plan 108): the last big-population languages
|
|
55
|
+
# — Hangul eojeol tokenizer + Dubeolsik grid, Devanagari grapheme
|
|
56
|
+
# tokenizer + InScript grid.
|
|
57
|
+
autoload :Korean, "kotoshu/languages/ko/language"
|
|
58
|
+
autoload :Nepali, "kotoshu/languages/ne/language"
|
|
59
|
+
# Serbian Latin module (plan 110): full-feature wiring over the
|
|
60
|
+
# distinct staged sr-Latn dictionary — Latin tokenizer, the
|
|
61
|
+
# South-Slavic QWERTZ grid shared with hr and sl.
|
|
62
|
+
autoload :SerbianLatin, "kotoshu/languages/sr-Latn/language"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Metrics
|
|
5
|
+
# Thread-safe metrics collector.
|
|
6
|
+
#
|
|
7
|
+
# Tracks performance metrics for spellchecking operations:
|
|
8
|
+
# - Lookup counts and timing
|
|
9
|
+
# - Cache hit/miss rates
|
|
10
|
+
# - Suggestion generation stats
|
|
11
|
+
#
|
|
12
|
+
# @example
|
|
13
|
+
# collector = Kotoshu::Metrics::Collector.new
|
|
14
|
+
# collector.record_lookup("hello", result: true, time: 0.5)
|
|
15
|
+
# collector.stats
|
|
16
|
+
# # => { lookups: 1, correct_lookups: 1, avg_lookup_time: 0.5, ... }
|
|
17
|
+
class Collector
|
|
18
|
+
# Initialize a new collector.
|
|
19
|
+
def initialize
|
|
20
|
+
@mutex = Mutex.new
|
|
21
|
+
reset
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Record a lookup operation.
|
|
25
|
+
#
|
|
26
|
+
# @param _word [String] The word being looked up (not retained)
|
|
27
|
+
# @param result [Boolean] The lookup result
|
|
28
|
+
# @param time [Float] Time taken in milliseconds
|
|
29
|
+
def record_lookup(_word, result:, time:)
|
|
30
|
+
@mutex.synchronize do
|
|
31
|
+
@metrics[:lookups] += 1
|
|
32
|
+
@metrics[:correct_lookups] += 1 if result
|
|
33
|
+
@metrics[:misspelled_lookups] += 1 unless result
|
|
34
|
+
|
|
35
|
+
@metrics[:lookup_times] << time
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Record a cache operation.
|
|
40
|
+
#
|
|
41
|
+
# @param cache_type [String] Type of cache (lookup, suggestion)
|
|
42
|
+
# @param hit [Boolean] True if cache hit
|
|
43
|
+
def record_cache(cache_type, hit:)
|
|
44
|
+
@mutex.synchronize do
|
|
45
|
+
key = :"#{cache_type}_cache_hits"
|
|
46
|
+
miss_key = :"#{cache_type}_cache_misses"
|
|
47
|
+
|
|
48
|
+
if hit
|
|
49
|
+
@metrics[key] += 1
|
|
50
|
+
else
|
|
51
|
+
@metrics[miss_key] += 1
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Record suggestion generation.
|
|
57
|
+
#
|
|
58
|
+
# @param _word [String] The input word (not retained)
|
|
59
|
+
# @param count [Integer] Number of suggestions generated
|
|
60
|
+
# @param time [Float] Time taken in milliseconds
|
|
61
|
+
def record_suggestions(_word, count:, time:)
|
|
62
|
+
@mutex.synchronize do
|
|
63
|
+
@metrics[:suggestion_requests] += 1
|
|
64
|
+
@metrics[:suggestions_generated] += count
|
|
65
|
+
|
|
66
|
+
@metrics[:suggestion_times] << time
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Record a skipped semantic rerank (confidence cascade skip).
|
|
71
|
+
def record_semantic_cascade_skip
|
|
72
|
+
@mutex.synchronize do
|
|
73
|
+
@metrics[:semantic_cascade_skips] += 1
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Get current metrics statistics.
|
|
78
|
+
#
|
|
79
|
+
# @return [Hash] Current statistics with computed averages
|
|
80
|
+
def stats
|
|
81
|
+
@mutex.synchronize do
|
|
82
|
+
calculate_stats
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Reset all metrics.
|
|
87
|
+
def reset
|
|
88
|
+
@mutex.synchronize do
|
|
89
|
+
@metrics = {
|
|
90
|
+
lookups: 0,
|
|
91
|
+
correct_lookups: 0,
|
|
92
|
+
misspelled_lookups: 0,
|
|
93
|
+
lookup_times: [],
|
|
94
|
+
|
|
95
|
+
lookup_cache_hits: 0,
|
|
96
|
+
lookup_cache_misses: 0,
|
|
97
|
+
suggestion_cache_hits: 0,
|
|
98
|
+
suggestion_cache_misses: 0,
|
|
99
|
+
|
|
100
|
+
suggestion_requests: 0,
|
|
101
|
+
suggestions_generated: 0,
|
|
102
|
+
suggestion_times: [],
|
|
103
|
+
|
|
104
|
+
semantic_cascade_skips: 0,
|
|
105
|
+
|
|
106
|
+
started_at: Time.now
|
|
107
|
+
}
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Export metrics in StatsD format.
|
|
112
|
+
#
|
|
113
|
+
# @return [String] StatsD protocol lines
|
|
114
|
+
def to_statsd
|
|
115
|
+
s = stats
|
|
116
|
+
prefix = "kotoshu"
|
|
117
|
+
|
|
118
|
+
lines = []
|
|
119
|
+
lines << "#{prefix}.lookups:#{s[:lookups]}|c"
|
|
120
|
+
lines << "#{prefix}.correct_lookups:#{s[:correct_lookups]}|c"
|
|
121
|
+
lines << "#{prefix}.misspelled_lookups:#{s[:misspelled_lookups]}|c"
|
|
122
|
+
lines << "#{prefix}.avg_lookup_time:#{s[:avg_lookup_time]}|ms"
|
|
123
|
+
lines << "#{prefix}.lookup_cache_hits:#{s[:lookup_cache_hits]}|c"
|
|
124
|
+
lines << "#{prefix}.lookup_cache_misses:#{s[:lookup_cache_misses]}|c"
|
|
125
|
+
lines << "#{prefix}.suggestion_requests:#{s[:suggestion_requests]}|c"
|
|
126
|
+
lines << "#{prefix}.suggestions_generated:#{s[:suggestions_generated]}|c"
|
|
127
|
+
lines << "#{prefix}.avg_suggestion_time:#{s[:avg_suggestion_time]}|ms"
|
|
128
|
+
lines << "#{prefix}.semantic_cascade_skips:#{s[:semantic_cascade_skips]}|c"
|
|
129
|
+
|
|
130
|
+
lines.join("\n")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Export metrics in Prometheus exposition format.
|
|
134
|
+
#
|
|
135
|
+
# @return [String] Prometheus format
|
|
136
|
+
def to_prometheus
|
|
137
|
+
s = stats
|
|
138
|
+
|
|
139
|
+
lines = []
|
|
140
|
+
lines << "# HELP kotoshu_lookups Total number of word lookups"
|
|
141
|
+
lines << "# TYPE kotoshu_lookups counter"
|
|
142
|
+
lines << "kotoshu_lookups #{s[:lookups]}"
|
|
143
|
+
|
|
144
|
+
lines << "# HELP kotoshu_correct_lookups Number of correct word lookups"
|
|
145
|
+
lines << "# TYPE kotoshu_correct_lookups counter"
|
|
146
|
+
lines << "kotoshu_correct_lookups #{s[:correct_lookups]}"
|
|
147
|
+
|
|
148
|
+
lines << "# HELP kotoshu_misspelled_lookups Number of misspelled word lookups"
|
|
149
|
+
lines << "# TYPE kotoshu_misspelled_lookups counter"
|
|
150
|
+
lines << "kotoshu_misspelled_lookups #{s[:misspelled_lookups]}"
|
|
151
|
+
|
|
152
|
+
lines << "# HELP kotoshu_avg_lookup_time Average lookup time in milliseconds"
|
|
153
|
+
lines << "# TYPE kotoshu_avg_lookup_time gauge"
|
|
154
|
+
lines << "kotoshu_avg_lookup_time #{s[:avg_lookup_time]}"
|
|
155
|
+
|
|
156
|
+
lines << "# HELP kotoshu_lookup_cache_hits Number of lookup cache hits"
|
|
157
|
+
lines << "# TYPE kotoshu_lookup_cache_hits counter"
|
|
158
|
+
lines << "kotoshu_lookup_cache_hits #{s[:lookup_cache_hits]}"
|
|
159
|
+
|
|
160
|
+
lines << "# HELP kotoshu_lookup_cache_misses Number of lookup cache misses"
|
|
161
|
+
lines << "# TYPE kotoshu_lookup_cache_misses counter"
|
|
162
|
+
lines << "kotoshu_lookup_cache_misses #{s[:lookup_cache_misses]}"
|
|
163
|
+
|
|
164
|
+
lines << "# HELP kotoshu_suggestion_requests Number of suggestion requests"
|
|
165
|
+
lines << "# TYPE kotoshu_suggestion_requests counter"
|
|
166
|
+
lines << "kotoshu_suggestion_requests #{s[:suggestion_requests]}"
|
|
167
|
+
|
|
168
|
+
lines << "# HELP kotoshu_suggestions_generated Total number of suggestions generated"
|
|
169
|
+
lines << "# TYPE kotoshu_suggestions_generated counter"
|
|
170
|
+
lines << "kotoshu_suggestions_generated #{s[:suggestions_generated]}"
|
|
171
|
+
|
|
172
|
+
lines << "# HELP kotoshu_avg_suggestion_time Average suggestion generation time in milliseconds"
|
|
173
|
+
lines << "# TYPE kotoshu_avg_suggestion_time gauge"
|
|
174
|
+
lines << "kotoshu_avg_suggestion_time #{s[:avg_suggestion_time]}"
|
|
175
|
+
|
|
176
|
+
lines << "# HELP kotoshu_semantic_cascade_skips Number of semantic reranks skipped by the confidence cascade"
|
|
177
|
+
lines << "# TYPE kotoshu_semantic_cascade_skips counter"
|
|
178
|
+
lines << "kotoshu_semantic_cascade_skips #{s[:semantic_cascade_skips]}"
|
|
179
|
+
|
|
180
|
+
lines.join("\n")
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
private
|
|
184
|
+
|
|
185
|
+
# Calculate computed statistics.
|
|
186
|
+
#
|
|
187
|
+
# @return [Hash] Statistics with computed values
|
|
188
|
+
def calculate_stats
|
|
189
|
+
lookup_times = @metrics[:lookup_times]
|
|
190
|
+
suggestion_times = @metrics[:suggestion_times]
|
|
191
|
+
|
|
192
|
+
avg_lookup = lookup_times.empty? ? 0 : lookup_times.sum / lookup_times.size
|
|
193
|
+
avg_suggestion = suggestion_times.empty? ? 0 : suggestion_times.sum / suggestion_times.size
|
|
194
|
+
|
|
195
|
+
lookup_hit_rate = calculate_hit_rate(@metrics[:lookup_cache_hits], @metrics[:lookup_cache_misses])
|
|
196
|
+
suggestion_hit_rate = calculate_hit_rate(@metrics[:suggestion_cache_hits], @metrics[:suggestion_cache_misses])
|
|
197
|
+
|
|
198
|
+
{
|
|
199
|
+
lookups: @metrics[:lookups],
|
|
200
|
+
correct_lookups: @metrics[:correct_lookups],
|
|
201
|
+
misspelled_lookups: @metrics[:misspelled_lookups],
|
|
202
|
+
avg_lookup_time: avg_lookup.round(3),
|
|
203
|
+
|
|
204
|
+
lookup_cache_hits: @metrics[:lookup_cache_hits],
|
|
205
|
+
lookup_cache_misses: @metrics[:lookup_cache_misses],
|
|
206
|
+
lookup_cache_hit_rate: lookup_hit_rate,
|
|
207
|
+
|
|
208
|
+
suggestion_cache_hits: @metrics[:suggestion_cache_hits],
|
|
209
|
+
suggestion_cache_misses: @metrics[:suggestion_cache_misses],
|
|
210
|
+
suggestion_cache_hit_rate: suggestion_hit_rate,
|
|
211
|
+
|
|
212
|
+
suggestion_requests: @metrics[:suggestion_requests],
|
|
213
|
+
suggestions_generated: @metrics[:suggestions_generated],
|
|
214
|
+
avg_suggestions_per_request: if @metrics[:suggestion_requests].positive?
|
|
215
|
+
(@metrics[:suggestions_generated].to_f / @metrics[:suggestion_requests]).round(2)
|
|
216
|
+
else
|
|
217
|
+
0
|
|
218
|
+
end,
|
|
219
|
+
avg_suggestion_time: avg_suggestion.round(3),
|
|
220
|
+
|
|
221
|
+
semantic_cascade_skips: @metrics[:semantic_cascade_skips],
|
|
222
|
+
|
|
223
|
+
uptime_seconds: (Time.now - @metrics[:started_at]).round(2)
|
|
224
|
+
}
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Calculate cache hit rate.
|
|
228
|
+
#
|
|
229
|
+
# @param hits [Integer] Number of hits
|
|
230
|
+
# @param misses [Integer] Number of misses
|
|
231
|
+
# @return [Float] Hit rate (0-1)
|
|
232
|
+
def calculate_hit_rate(hits, misses)
|
|
233
|
+
total = hits + misses
|
|
234
|
+
total.positive? ? (hits.to_f / total).round(4) : 0.0
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Metrics and instrumentation for Kotoshu.
|
|
5
|
+
#
|
|
6
|
+
# Provides thread-safe collection of performance metrics:
|
|
7
|
+
# - Lookup counts and timing
|
|
8
|
+
# - Cache hit/miss rates
|
|
9
|
+
# - Suggestion generation stats
|
|
10
|
+
# - Optional export to StatsD or Prometheus
|
|
11
|
+
#
|
|
12
|
+
# @example Enable metrics
|
|
13
|
+
# Kotoshu::Metrics.enable
|
|
14
|
+
# Kotoshu.correct?("hello")
|
|
15
|
+
# Kotoshu::Metrics.stats
|
|
16
|
+
# # => { lookups: 1, cache_hits: 0, cache_misses: 1, ... }
|
|
17
|
+
module Metrics
|
|
18
|
+
autoload :Collector, "kotoshu/metrics_collector"
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
# Enable metrics collection.
|
|
22
|
+
def enable
|
|
23
|
+
@enabled = true
|
|
24
|
+
@collector = Collector.new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Disable metrics collection.
|
|
28
|
+
def disable
|
|
29
|
+
@enabled = false
|
|
30
|
+
@collector = nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Check if metrics are enabled.
|
|
34
|
+
#
|
|
35
|
+
# @return [Boolean] True if enabled
|
|
36
|
+
def enabled?
|
|
37
|
+
@enabled ||= false
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Get the metrics collector.
|
|
41
|
+
#
|
|
42
|
+
# @return [Collector, nil] The collector instance
|
|
43
|
+
attr_reader :collector
|
|
44
|
+
|
|
45
|
+
# Record a lookup operation.
|
|
46
|
+
#
|
|
47
|
+
# @param word [String] The word being looked up
|
|
48
|
+
# @param result [Boolean] The lookup result
|
|
49
|
+
# @param time [Float] Time taken in milliseconds
|
|
50
|
+
def record_lookup(word, result:, time:)
|
|
51
|
+
return unless enabled?
|
|
52
|
+
|
|
53
|
+
collector&.record_lookup(word, result: result, time: time)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Record a cache operation.
|
|
57
|
+
#
|
|
58
|
+
# @param cache_type [String] Type of cache (lookup, suggestion)
|
|
59
|
+
# @param hit [Boolean] True if cache hit
|
|
60
|
+
def record_cache(cache_type, hit:)
|
|
61
|
+
return unless enabled?
|
|
62
|
+
|
|
63
|
+
collector&.record_cache(cache_type, hit: hit)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Record suggestion generation.
|
|
67
|
+
#
|
|
68
|
+
# @param word [String] The input word
|
|
69
|
+
# @param count [Integer] Number of suggestions generated
|
|
70
|
+
# @param time [Float] Time taken in milliseconds
|
|
71
|
+
def record_suggestions(word, count:, time:)
|
|
72
|
+
return unless enabled?
|
|
73
|
+
|
|
74
|
+
collector&.record_suggestions(word, count: count, time: time)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Record a skipped semantic rerank (confidence cascade): the
|
|
78
|
+
# traditional strategies were already confident, so the semantic
|
|
79
|
+
# rerank was not run (TODO.impl/70).
|
|
80
|
+
def record_semantic_cascade_skip
|
|
81
|
+
return unless enabled?
|
|
82
|
+
|
|
83
|
+
collector&.record_semantic_cascade_skip
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Get current metrics statistics.
|
|
87
|
+
#
|
|
88
|
+
# @return [Hash] Current statistics
|
|
89
|
+
def stats
|
|
90
|
+
return {} unless enabled?
|
|
91
|
+
|
|
92
|
+
collector&.stats || {}
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Reset all metrics.
|
|
96
|
+
def reset
|
|
97
|
+
return unless enabled?
|
|
98
|
+
|
|
99
|
+
collector&.reset
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Get metrics as StatsD format.
|
|
103
|
+
#
|
|
104
|
+
# @return [String] StatsD protocol lines
|
|
105
|
+
def to_statsd
|
|
106
|
+
return "" unless enabled?
|
|
107
|
+
|
|
108
|
+
collector&.to_statsd || ""
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Get metrics as Prometheus format.
|
|
112
|
+
#
|
|
113
|
+
# @return [String] Prometheus exposition format
|
|
114
|
+
def to_prometheus
|
|
115
|
+
return "" unless enabled?
|
|
116
|
+
|
|
117
|
+
collector&.to_prometheus || ""
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Models
|
|
5
|
+
# Value object for text context around an error.
|
|
6
|
+
#
|
|
7
|
+
# Provides the surrounding text before, current, and after
|
|
8
|
+
# an error location for context display and analysis.
|
|
9
|
+
#
|
|
10
|
+
# @example Creating context
|
|
11
|
+
# context = Context.new(
|
|
12
|
+
# before: "The quick brown",
|
|
13
|
+
# current: "fox",
|
|
14
|
+
# after: "jumps over",
|
|
15
|
+
# location: Location.new(line: 5, column: 16)
|
|
16
|
+
# )
|
|
17
|
+
# context.full_context # => "The quick brown fox jumps over"
|
|
18
|
+
class Context
|
|
19
|
+
attr_reader :before, :current, :after, :location, :window, :full_context
|
|
20
|
+
|
|
21
|
+
# Create a new context object.
|
|
22
|
+
#
|
|
23
|
+
# @param before [String] Text before the error
|
|
24
|
+
# @param current [String] The current line/text containing the error
|
|
25
|
+
# @param after [String] Text after the error
|
|
26
|
+
# @param location [Object] The error location (line/column holder)
|
|
27
|
+
# @param window [Integer] Window size used for context (default: 5)
|
|
28
|
+
def initialize(before:, current:, after:, location:, window: 5)
|
|
29
|
+
@before = before
|
|
30
|
+
@current = current
|
|
31
|
+
@after = after
|
|
32
|
+
@location = location
|
|
33
|
+
@window = window
|
|
34
|
+
@full_context = [before, current, after].compact.join("\n")
|
|
35
|
+
freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Get surrounding words around the error location.
|
|
39
|
+
#
|
|
40
|
+
# @param n [Integer] Number of words on each side (default: 3)
|
|
41
|
+
# @return [Array<String>] Surrounding words
|
|
42
|
+
def surrounding_words(n = 3)
|
|
43
|
+
return [] if @current.nil? || @current.empty?
|
|
44
|
+
|
|
45
|
+
words = @current.split
|
|
46
|
+
return [] if words.empty?
|
|
47
|
+
|
|
48
|
+
# Try to find the word at the error location
|
|
49
|
+
target_word = word_at_location
|
|
50
|
+
return words unless target_word
|
|
51
|
+
|
|
52
|
+
idx = words.index(target_word)
|
|
53
|
+
return words unless idx
|
|
54
|
+
|
|
55
|
+
# Get n words before and after
|
|
56
|
+
start_idx = [0, idx - n].max
|
|
57
|
+
end_idx = [words.size - 1, idx + n].min
|
|
58
|
+
|
|
59
|
+
words[start_idx..end_idx].to_a
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Get the word at the error location.
|
|
63
|
+
#
|
|
64
|
+
# @return [String, nil] The word at the error location
|
|
65
|
+
def word_at_location
|
|
66
|
+
return nil unless @location
|
|
67
|
+
|
|
68
|
+
if @location.column && @current && @location.column < @current.length
|
|
69
|
+
# Get character at column
|
|
70
|
+
return @current[@location.column]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# For node-based locations, return the current text
|
|
74
|
+
@current
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Check if this context equals another.
|
|
78
|
+
#
|
|
79
|
+
# @param other [Object] Another object
|
|
80
|
+
# @return [Boolean] True if contexts match
|
|
81
|
+
def ==(other)
|
|
82
|
+
return false unless other.is_a?(Context)
|
|
83
|
+
|
|
84
|
+
@location == other.location && @full_context == other.full_context
|
|
85
|
+
end
|
|
86
|
+
alias_method :eql?, :==
|
|
87
|
+
|
|
88
|
+
# Hash code for hash table usage.
|
|
89
|
+
#
|
|
90
|
+
# @return [Integer] Hash code
|
|
91
|
+
def hash
|
|
92
|
+
[@location, @full_context].hash
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# String representation.
|
|
96
|
+
#
|
|
97
|
+
# @return [String] Human-readable representation
|
|
98
|
+
def to_s
|
|
99
|
+
if @location.line
|
|
100
|
+
"Line #{@location.line}: #{@full_context}"
|
|
101
|
+
else
|
|
102
|
+
@full_context
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
alias_method :inspect, :to_s
|
|
106
|
+
|
|
107
|
+
# Get context as a formatted string with error highlighting.
|
|
108
|
+
#
|
|
109
|
+
# @param error_word [String] The error word to highlight
|
|
110
|
+
# @return [String] Formatted context with ANSI codes
|
|
111
|
+
def with_highlight(error_word)
|
|
112
|
+
return @full_context unless error_word
|
|
113
|
+
|
|
114
|
+
# Find and highlight the error word
|
|
115
|
+
@full_context.gsub(error_word) { |m| "\033[4m#{m}\033[0m" }
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|