kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
# One language detection: the top label code (ISO 639) and its
|
|
6
|
+
# probability, the shape the plan-106 facade returns.
|
|
7
|
+
#
|
|
8
|
+
# A Struct (keyword_init) per the plain-data rule — no behavior
|
|
9
|
+
# beyond readers, #to_s (the code, so string interpolation reads
|
|
10
|
+
# naturally), and equality.
|
|
11
|
+
Detection = Struct.new(:code, :score, keyword_init: true) do
|
|
12
|
+
# The code, so `"detected #{detection}"` reads naturally.
|
|
13
|
+
def to_s
|
|
14
|
+
code.to_s
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Document language detection through the pure-Rust LID reader
|
|
19
|
+
# (plan 106).
|
|
20
|
+
#
|
|
21
|
+
# The gem's {LanguageIdentifier} needs the Python fastText
|
|
22
|
+
# bindings (broken under numpy 2), which is why /v1/detect and the
|
|
23
|
+
# CLI shipped a 7-language heuristic. The native extension now
|
|
24
|
+
# wraps kotoshu-rs' `lid` reader over the
|
|
25
|
+
# `kotoshu://models/lid/lid-176` registry artifact pair (the
|
|
26
|
+
# ONNX-converted lid.176 model + vocab sidecar, 176 languages,
|
|
27
|
+
# bit-faithful to the bindings — parity frozen by the kotoshu-rs
|
|
28
|
+
# suite over the shared corpus fixture).
|
|
29
|
+
#
|
|
30
|
+
# Selection mirrors {NativeBackend} with one difference: the
|
|
31
|
+
# native engine serves detect whenever the extension is built and
|
|
32
|
+
# the artifact pair is cached — an explicitly ruby backend
|
|
33
|
+
# (KOTOSHU_BACKEND=ruby) opts out, but the unset default does not,
|
|
34
|
+
# because the {Detector} heuristic is strictly weaker (7 languages
|
|
35
|
+
# vs 176). Pure-Ruby installs keep working unchanged — no new
|
|
36
|
+
# gemspec dependency, no onnxruntime.
|
|
37
|
+
#
|
|
38
|
+
# Two-stage like every resource: {setup} downloads the pair once
|
|
39
|
+
# (registry-resolved, sha-verified); {detect} resolves from the
|
|
40
|
+
# cache only and never downloads implicitly.
|
|
41
|
+
#
|
|
42
|
+
# @example Setup once, then detect
|
|
43
|
+
# Kotoshu.setup_lid
|
|
44
|
+
# Kotoshu.detect_language("Der Ausschuss prüft den Vorschlag")
|
|
45
|
+
# # => #<struct Kotoshu::Language::Detection code="de", score=0.99>
|
|
46
|
+
class LidDetector
|
|
47
|
+
class << self
|
|
48
|
+
# Detect the language of text.
|
|
49
|
+
#
|
|
50
|
+
# Native LID when available (see {LidDetector} for the
|
|
51
|
+
# selection rules), the {Detector} heuristic otherwise.
|
|
52
|
+
#
|
|
53
|
+
# @param text [String] Text to analyze
|
|
54
|
+
# @param config [Configuration] configuration to consult
|
|
55
|
+
# (defaults to the global instance)
|
|
56
|
+
# @return [Detection] code (nil when the heuristic is
|
|
57
|
+
# uncertain) and score in [0, 1]
|
|
58
|
+
def detect(text, config: Kotoshu.configuration)
|
|
59
|
+
# nil is not text; an empty/blank string IS (the native
|
|
60
|
+
# reader scores its EOS token row — "en", ~0.12 — exactly
|
|
61
|
+
# like the fastText bindings, per the frozen corpus).
|
|
62
|
+
return Detection.new(code: nil, score: 0.0) if text.nil?
|
|
63
|
+
|
|
64
|
+
model = native_model(config: config)
|
|
65
|
+
return heuristic_detection(text) unless model
|
|
66
|
+
|
|
67
|
+
row = model.detect(text)
|
|
68
|
+
Detection.new(code: row.fetch("code"), score: row.fetch("score"))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Whether {detect} would run the native engine for this
|
|
72
|
+
# configuration — the extension is built, the backend is not
|
|
73
|
+
# "ruby", and the artifact pair is cached.
|
|
74
|
+
#
|
|
75
|
+
# @param config [Configuration]
|
|
76
|
+
# @return [Boolean]
|
|
77
|
+
def available?(config: Kotoshu.configuration)
|
|
78
|
+
!native_model(config: config).nil?
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Download the LID artifact pair into the model cache (stage
|
|
82
|
+
# one of the two-stage model; registry-resolved, sha-verified).
|
|
83
|
+
#
|
|
84
|
+
# @param config [Configuration]
|
|
85
|
+
# @param force [Boolean] re-fetch even if already cached
|
|
86
|
+
# @return [Hash] { onnx_path:, vocab_path:, metadata }
|
|
87
|
+
# @raise [Kotoshu::Error] no registry entry, or offline with
|
|
88
|
+
# no cached registry (KOTOSHU_OFFLINE=1)
|
|
89
|
+
# @raise [Kotoshu::IntegrityError] downloaded bytes fail the
|
|
90
|
+
# registry checksum
|
|
91
|
+
def setup(config: Kotoshu.configuration, force: false)
|
|
92
|
+
pair = model_cache(config: config).load_cached_lid
|
|
93
|
+
return pair if pair && !force
|
|
94
|
+
|
|
95
|
+
model_cache(config: config).download_lid(force: force)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Whether the LID artifact pair is cached (cache-only check).
|
|
99
|
+
#
|
|
100
|
+
# @param config [Configuration]
|
|
101
|
+
# @return [Boolean]
|
|
102
|
+
def setup?(config: Kotoshu.configuration)
|
|
103
|
+
model_cache(config: config).lid_cached?
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Drop the memoized native model. The next {detect} re-resolves
|
|
107
|
+
# from the cache — the seam tests and embedders use after
|
|
108
|
+
# {setup} or a cache purge (mirrors Kotoshu.reset_spellchecker).
|
|
109
|
+
#
|
|
110
|
+
# @return [nil]
|
|
111
|
+
def reset
|
|
112
|
+
@native_model = nil
|
|
113
|
+
nil
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# The loaded native LID model, memoized per process — one
|
|
119
|
+
# parse of the ~3 MB pair, mirroring how spellcheckers are
|
|
120
|
+
# cached per language. Nil whenever the native engine cannot
|
|
121
|
+
# serve: extension absent, Kotoshu::Native::LidModel missing
|
|
122
|
+
# (extension predates plan 106), an explicit ruby backend, or
|
|
123
|
+
# the pair not set up.
|
|
124
|
+
#
|
|
125
|
+
# @param config [Configuration]
|
|
126
|
+
# @return [Kotoshu::Native::LidModel, nil]
|
|
127
|
+
def native_model(config:)
|
|
128
|
+
return nil unless native_lid_surface?
|
|
129
|
+
return nil if backend_opts_out?(config)
|
|
130
|
+
|
|
131
|
+
@native_model ||= begin
|
|
132
|
+
pair = model_cache(config: config).load_cached_lid
|
|
133
|
+
pair && Kotoshu::Native::LidModel.load(pair[:onnx_path], pair[:vocab_path])
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Whether the extension carries the LID surface at all.
|
|
138
|
+
def native_lid_surface?
|
|
139
|
+
Kotoshu::Native.available? && Kotoshu::Native.const_defined?(:LidModel)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Backend selection for LID (plan 106).
|
|
143
|
+
#
|
|
144
|
+
# KOTOSHU_BACKEND=ruby opts the LID out of the native engine.
|
|
145
|
+
# When the backend is UNSET, LID still prefers the native
|
|
146
|
+
# engine — unlike correct?/suggest, where "ruby" is the engine
|
|
147
|
+
# default and "native" the explicit accelerator. The heuristic
|
|
148
|
+
# is strictly weaker than the 176-language model, so it stays
|
|
149
|
+
# the fallback for the cannot-serve cases (no extension, no
|
|
150
|
+
# cached model), not for an unset preference.
|
|
151
|
+
#
|
|
152
|
+
# Explicitness is read through the resolver layers (live ENV,
|
|
153
|
+
# constructor settings, CLI options) so the "ruby" DEFAULT
|
|
154
|
+
# never counts as an opt-out. Mutating
|
|
155
|
+
# Kotoshu.configuration.backend after construction is not
|
|
156
|
+
# visible here — rebuild the configuration instead.
|
|
157
|
+
def backend_opts_out?(config)
|
|
158
|
+
config.backend.to_s == "ruby" && backend_set?(config)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Whether the backend was explicitly set at any resolver layer
|
|
162
|
+
# (rather than left at its "ruby" default).
|
|
163
|
+
def backend_set?(config)
|
|
164
|
+
resolver = config.resolver
|
|
165
|
+
ENV.key?("KOTOSHU_BACKEND") ||
|
|
166
|
+
resolver.programmatic.key?(:backend) ||
|
|
167
|
+
resolver.cli.key?(:backend)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# The heuristic fallback, wrapped in the same {Detection}
|
|
171
|
+
# shape the native path returns.
|
|
172
|
+
def heuristic_detection(text)
|
|
173
|
+
code, confidence = Detector.detect_with_confidence(text)
|
|
174
|
+
Detection.new(code: code, score: confidence)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# A cache over the configuration's model cache path and
|
|
178
|
+
# source registry (so KOTOSHU_MODELS_PIN is honored).
|
|
179
|
+
def model_cache(config:)
|
|
180
|
+
Cache::ModelCache.new(
|
|
181
|
+
cache_path: config.cache_path,
|
|
182
|
+
source_registry: config.source_registry
|
|
183
|
+
)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Normalizer for Arabic script.
|
|
7
|
+
#
|
|
8
|
+
# Arabic has many presentation forms (the U+FE70..U+FEFF Arabic
|
|
9
|
+
# Presentation Forms-A block plus the FB50..FDFF / FE70..FEFF
|
|
10
|
+
# blocks for Presentation Forms-B) that look the same to a
|
|
11
|
+
# reader but are distinct codepoints. Input text often arrives
|
|
12
|
+
# in any of these forms depending on the source editor — same
|
|
13
|
+
# word can be encoded three different ways and not match a
|
|
14
|
+
# dictionary lookup. This normalizer canonicalizes them all to
|
|
15
|
+
# the base U+0621..U+064A range.
|
|
16
|
+
#
|
|
17
|
+
# Also handles:
|
|
18
|
+
# - NFC composition (combines base + combining marks).
|
|
19
|
+
# - Optional diacritic (tashkeel) stripping for spell checking —
|
|
20
|
+
# Modern Standard Arabic is typically written without
|
|
21
|
+
# diacritics; the dictionary stores bare forms.
|
|
22
|
+
# - Tatweel (U+0640) removal — a decorative elongation mark
|
|
23
|
+
# that creates variants of the same word.
|
|
24
|
+
#
|
|
25
|
+
# @example Default (strip diacritics)
|
|
26
|
+
# norm = Arabic.new
|
|
27
|
+
# norm.normalize_word("كِتَابٌ") # => "كتاب"
|
|
28
|
+
class Arabic < Base
|
|
29
|
+
# Arabic Presentation Forms-A and Forms-B codepoints that
|
|
30
|
+
# have a base-form equivalent in the U+0621..U+064A range.
|
|
31
|
+
# Ruby's String#unicode_normalize(:nfc) handles the
|
|
32
|
+
# composition for most pairs; the legacy Presentation Forms-B
|
|
33
|
+
# range (FE70..FEFF) needs an explicit table because NFC
|
|
34
|
+
# doesn't cover it.
|
|
35
|
+
PRESENTATION_FORM_MAP = {
|
|
36
|
+
"ﺍ" => "ا", "ﺎ" => "ا",
|
|
37
|
+
"ﺏ" => "ب", "ﺑ" => "ب", "ﺒ" => "ب", "ﺓ" => "ة",
|
|
38
|
+
"ﺕ" => "ت", "ﺗ" => "ت", "ﺘ" => "ت", "ﺚ" => "ث",
|
|
39
|
+
"ﺛ" => "ث", "ﺜ" => "ث", "ﺝ" => "ج", "ﺟ" => "ج",
|
|
40
|
+
"ﺠ" => "ج", "ﺡ" => "ح", "ﺣ" => "ح", "ﺤ" => "ح",
|
|
41
|
+
"ﺥ" => "خ", "ﺧ" => "خ", "ﺨ" => "خ", "ﺩ" => "د",
|
|
42
|
+
"ﺪ" => "د", "ﺫ" => "ذ", "ﺬ" => "ذ", "ﺭ" => "ر",
|
|
43
|
+
"ﺮ" => "ر", "ﺯ" => "ز", "ﺰ" => "ز", "ﺱ" => "س",
|
|
44
|
+
"ﺳ" => "س", "ﺴ" => "س", "ﺵ" => "ش", "ﺷ" => "ش",
|
|
45
|
+
"ﺸ" => "ش", "ﺹ" => "ص", "ﺻ" => "ص", "ﺼ" => "ص",
|
|
46
|
+
"ﺽ" => "ض", "ﺿ" => "ض", "ﻀ" => "ض", "ﻁ" => "ط",
|
|
47
|
+
"ﻃ" => "ط", "ﻄ" => "ط", "ﻅ" => "ظ", "ﻇ" => "ظ",
|
|
48
|
+
"ﻈ" => "ظ", "ﻉ" => "ع", "ﻋ" => "ع", "ﻌ" => "ع",
|
|
49
|
+
"ﻍ" => "غ", "ﻏ" => "غ", "ﻐ" => "غ", "ﻑ" => "ف",
|
|
50
|
+
"ﻓ" => "ف", "ﻔ" => "ف", "ﻕ" => "ق", "ﻗ" => "ق",
|
|
51
|
+
"ﻘ" => "ق", "ﻙ" => "ك", "ﻛ" => "ك", "ﻜ" => "ك",
|
|
52
|
+
"ﻝ" => "ل", "ﻟ" => "ل", "ﻠ" => "ل", "ﻡ" => "م",
|
|
53
|
+
"ﻣ" => "م", "ﻤ" => "م", "ﻥ" => "ن", "ﻧ" => "ن",
|
|
54
|
+
"ﻨ" => "ن", "ﻩ" => "ه", "ﻫ" => "ه", "ﻬ" => "ه",
|
|
55
|
+
"ﻭ" => "و", "ﻮ" => "و", "ﻱ" => "ي", "ﻳ" => "ي",
|
|
56
|
+
"ﻴ" => "ي", "ﻵ" => "آ", "ﻶ" => "آ", "ﻷ" => "أ",
|
|
57
|
+
"ﻸ" => "أ", "ﻹ" => "إ", "ﻺ" => "إ", "ﻻ" => "لا",
|
|
58
|
+
"ﻼ" => "لا"
|
|
59
|
+
}.freeze
|
|
60
|
+
|
|
61
|
+
# Arabic diacritics (tashkeel) — harakat, tanwin, shaddah,
|
|
62
|
+
# sukun, dagger alef. Stripped for spell-checking since the
|
|
63
|
+
# dictionary stores bare forms.
|
|
64
|
+
DIACRITICS = /[ؐ-ًؚ-ٰٟۖ-ۜ۟-ۤۧ-۪ۨ-ۭ]/
|
|
65
|
+
|
|
66
|
+
# Tatweel (kashida) — decorative horizontal elongation.
|
|
67
|
+
TATWEEL = "ـ"
|
|
68
|
+
|
|
69
|
+
# @param strip_diacritics [Boolean] default true. When true,
|
|
70
|
+
# tashkeel (vowel points) are removed before lookup.
|
|
71
|
+
# @param strip_tatweel [Boolean] default true. When true,
|
|
72
|
+
# tatweel elongation marks are removed.
|
|
73
|
+
def initialize(strip_diacritics: true, strip_tatweel: true)
|
|
74
|
+
@strip_diacritics = strip_diacritics
|
|
75
|
+
@strip_tatweel = strip_tatweel
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Normalize text: NFC + presentation-form canonicalization +
|
|
79
|
+
# optional diacritic/tatweel stripping. Skips the Base class's
|
|
80
|
+
# downcase step because Arabic has no case.
|
|
81
|
+
#
|
|
82
|
+
# @param text [String]
|
|
83
|
+
# @param _options [Hash] ignored (Arabic has no case to fold)
|
|
84
|
+
# @return [String]
|
|
85
|
+
def normalize(text, _options = {})
|
|
86
|
+
return "" if text.nil? || text.empty?
|
|
87
|
+
|
|
88
|
+
result = text.unicode_normalize(:nfc)
|
|
89
|
+
result = canonicalize_presentation_forms(result)
|
|
90
|
+
result = result.gsub(DIACRITICS, "") if @strip_diacritics
|
|
91
|
+
result = result.delete(TATWEEL) if @strip_tatweel
|
|
92
|
+
result.strip
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Normalize a word: same as +normalize+ but operates on a
|
|
96
|
+
# single token. Arabic words don't need different treatment.
|
|
97
|
+
def normalize_word(word)
|
|
98
|
+
normalize(word)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
# Replace every Presentation Forms-B character with its
|
|
104
|
+
# base-form equivalent. Done character-by-character because
|
|
105
|
+
# the mapping is irregular (one base letter may have up to
|
|
106
|
+
# four positional forms: isolated, initial, medial, final).
|
|
107
|
+
def canonicalize_presentation_forms(text)
|
|
108
|
+
text.chars.map { |c| PRESENTATION_FORM_MAP.fetch(c, c) }.join
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Abstract base class for text normalizers.
|
|
7
|
+
#
|
|
8
|
+
# Normalizers transform text to a standard form for comparison.
|
|
9
|
+
# Different languages use different normalization strategies.
|
|
10
|
+
#
|
|
11
|
+
# Examples of normalization:
|
|
12
|
+
# - Accent removal (café -> cafe)
|
|
13
|
+
# - Case folding (Hello -> hello)
|
|
14
|
+
# - Whitespace normalization
|
|
15
|
+
# - Punctuation normalization
|
|
16
|
+
#
|
|
17
|
+
# @example Implement a normalizer
|
|
18
|
+
# class MyNormalizer < Normalizer::Base
|
|
19
|
+
# def normalize(text)
|
|
20
|
+
# super.downcase.gsub(/[áàâä]/, 'a')
|
|
21
|
+
# end
|
|
22
|
+
# end
|
|
23
|
+
class Base
|
|
24
|
+
# Normalize text.
|
|
25
|
+
#
|
|
26
|
+
# Default implementation:
|
|
27
|
+
# - Strip leading/trailing whitespace
|
|
28
|
+
# - Collapse multiple whitespace to single space
|
|
29
|
+
# - Downcase (optional)
|
|
30
|
+
#
|
|
31
|
+
# @param text [String] Text to normalize
|
|
32
|
+
# @param options [Hash] Normalization options
|
|
33
|
+
# @option options [Boolean] :downcase (true) Convert to lowercase
|
|
34
|
+
# @option options [Boolean] :strip_punct (false) Remove punctuation
|
|
35
|
+
# @option options [Boolean] :collapse_ws (true) Collapse whitespace
|
|
36
|
+
# @return [String] Normalized text
|
|
37
|
+
def normalize(text, options = {})
|
|
38
|
+
return "" if text.nil?
|
|
39
|
+
|
|
40
|
+
defaults = {
|
|
41
|
+
downcase: true,
|
|
42
|
+
strip_punct: false,
|
|
43
|
+
collapse_ws: true
|
|
44
|
+
}
|
|
45
|
+
opts = defaults.merge(options)
|
|
46
|
+
|
|
47
|
+
result = text.dup
|
|
48
|
+
|
|
49
|
+
# Strip whitespace
|
|
50
|
+
result = result.strip
|
|
51
|
+
|
|
52
|
+
# Collapse multiple whitespace
|
|
53
|
+
result = result.gsub(/\s+/, " ") if opts[:collapse_ws]
|
|
54
|
+
|
|
55
|
+
# Downcase
|
|
56
|
+
result = result.downcase if opts[:downcase]
|
|
57
|
+
|
|
58
|
+
# Strip punctuation
|
|
59
|
+
result = strip_punctuation(result) if opts[:strip_punct]
|
|
60
|
+
|
|
61
|
+
result
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Normalize a word.
|
|
65
|
+
#
|
|
66
|
+
# @param word [String] Word to normalize
|
|
67
|
+
# @return [String] Normalized word
|
|
68
|
+
def normalize_word(word)
|
|
69
|
+
normalize(word)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Check if two normalized strings are equal.
|
|
73
|
+
#
|
|
74
|
+
# @param str1 [String] First string
|
|
75
|
+
# @param str2 [String] Second string
|
|
76
|
+
# @return [Boolean] True if equal after normalization
|
|
77
|
+
def normalized_eql?(str1, str2)
|
|
78
|
+
normalize(str1) == normalize(str2)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Strip punctuation from text.
|
|
82
|
+
#
|
|
83
|
+
# @param text [String] Text to strip
|
|
84
|
+
# @return [String] Text without punctuation
|
|
85
|
+
def strip_punctuation(text)
|
|
86
|
+
text.gsub(/[^\p{L}\p{N}\s]/, "")
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Remove accents from characters.
|
|
90
|
+
#
|
|
91
|
+
# @param text [String] Text with accents
|
|
92
|
+
# @return [String] Text without accents
|
|
93
|
+
def remove_accents(text)
|
|
94
|
+
# Unicode normalization form D (decompose)
|
|
95
|
+
normalized = text.unicode_normalize(:nfd)
|
|
96
|
+
|
|
97
|
+
# Remove combining diacritical marks
|
|
98
|
+
normalized.gsub(/[\u0300-\u036F]/, "")
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Normalize quotes to standard ASCII.
|
|
102
|
+
#
|
|
103
|
+
# @param text [String] Text with quotes
|
|
104
|
+
# @return [String] Text with normalized quotes
|
|
105
|
+
def normalize_quotes(text)
|
|
106
|
+
# Left double quote to straight
|
|
107
|
+
text = text.gsub(/[\u201C\u201D]/, '"')
|
|
108
|
+
# Right double quote to straight
|
|
109
|
+
text = text.gsub(/[\u2018\u2019]/, "'")
|
|
110
|
+
# Backticks to quotes
|
|
111
|
+
text = text.gsub('`', "'")
|
|
112
|
+
# Other quote variants
|
|
113
|
+
text = text.gsub("\u00AB", '"') # Left-pointing double angle
|
|
114
|
+
text = text.gsub("\u00BB", '"') # Right-pointing double angle
|
|
115
|
+
text = text.gsub("\u2039", "'") # Single left-pointing
|
|
116
|
+
text.gsub("\u203A", "'") # Single right-pointing
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Normalize whitespace.
|
|
120
|
+
#
|
|
121
|
+
# @param text [String] Text with irregular whitespace
|
|
122
|
+
# @return [String] Text with normalized whitespace
|
|
123
|
+
def normalize_whitespace(text)
|
|
124
|
+
text
|
|
125
|
+
.gsub(/[\u00A0\u202F\u205F]/, " ") # Various space chars
|
|
126
|
+
.gsub(/[\u2000-\u200B]/, " ") # Various space chars
|
|
127
|
+
.gsub(/\s+/, " ") # Collapse multiple spaces
|
|
128
|
+
.strip
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Greek-aware normalizer.
|
|
7
|
+
#
|
|
8
|
+
# Two Greek-specific behaviors over the base normalizer:
|
|
9
|
+
#
|
|
10
|
+
# 1. Final sigma: uppercase words ending in Σ downcase to σ, but
|
|
11
|
+
# dictionary forms end in ς. ΕΛΛΑΣ must look up as ελλας, not
|
|
12
|
+
# ελλασ. Applied at word level (normalize_word), where the
|
|
13
|
+
# word boundary is known.
|
|
14
|
+
# 2. Optional accent stripping (tonos and dialytika) for
|
|
15
|
+
# accent-insensitive comparison: πάτησις -> πατησις. Greek
|
|
16
|
+
# Hunspell dictionaries carry accents, so this is opt-in via
|
|
17
|
+
# the :strip_accents option (default false).
|
|
18
|
+
class Greek < Base
|
|
19
|
+
# Combining marks removed by accent stripping: Greek tonos,
|
|
20
|
+
# dialytika and the polytonic accent/breath marks
|
|
21
|
+
# (U+0300 through U+0345).
|
|
22
|
+
COMBINING_MARKS = /[\u0300-\u0345]/
|
|
23
|
+
|
|
24
|
+
private_constant :COMBINING_MARKS
|
|
25
|
+
|
|
26
|
+
# Normalize text.
|
|
27
|
+
#
|
|
28
|
+
# @param text [String] Text to normalize
|
|
29
|
+
# @param options [Hash] Normalization options
|
|
30
|
+
# @option options [Boolean] :strip_accents (false) Remove
|
|
31
|
+
# Greek accents after downcasing
|
|
32
|
+
# @return [String] Normalized text
|
|
33
|
+
def normalize(text, options = {})
|
|
34
|
+
result = super
|
|
35
|
+
result = strip_accents(result) if options.fetch(:strip_accents, false)
|
|
36
|
+
result
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Normalize a word for dictionary lookup.
|
|
40
|
+
#
|
|
41
|
+
# Downcase plus the final-sigma correction: a word ending in σ
|
|
42
|
+
# folds to ς to match dictionary forms.
|
|
43
|
+
#
|
|
44
|
+
# @param word [String] Word to normalize
|
|
45
|
+
# @return [String] Normalized word
|
|
46
|
+
def normalize_word(word)
|
|
47
|
+
normalized = super
|
|
48
|
+
normalized.gsub(/σ\z/, "ς")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# Strip Greek accents via NFD decomposition.
|
|
54
|
+
#
|
|
55
|
+
# @param text [String] Text to deaccent
|
|
56
|
+
# @return [String] Text without Greek combining marks
|
|
57
|
+
def strip_accents(text)
|
|
58
|
+
text.unicode_normalize(:nfd).gsub(COMBINING_MARKS, "").unicode_normalize(:nfc)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Normalizer for Hebrew script.
|
|
7
|
+
#
|
|
8
|
+
# Modern Hebrew is typically written without niqqud (vowel
|
|
9
|
+
# points); spell-checking dictionaries store the bare
|
|
10
|
+
# consonant form. This normalizer:
|
|
11
|
+
#
|
|
12
|
+
# - Applies NFC composition.
|
|
13
|
+
# - Strips niqqud (U+05B0..U+05BC range, plus geresh/gershayim
|
|
14
|
+
# when used as vowel marks — keeps them when used as
|
|
15
|
+
# punctuation).
|
|
16
|
+
# - Optionally strips dagesh (U+05BC) — a dot inside a letter
|
|
17
|
+
# that affects pronunciation but is not represented in bare
|
|
18
|
+
# dictionary entries.
|
|
19
|
+
# - Normalizes the maqaf (U+05BE, Hebrew hyphen) to ASCII hyphen
|
|
20
|
+
# for consistent tokenization.
|
|
21
|
+
#
|
|
22
|
+
# @example Default (strip niqqud)
|
|
23
|
+
# norm = Hebrew.new
|
|
24
|
+
# norm.normalize_word("שָׁלוֹם") # => "שלום"
|
|
25
|
+
class Hebrew < Base
|
|
26
|
+
# Niqqud + cantillation marks. Hebrew points range U+0591
|
|
27
|
+
# (etnahta) through U+05BD (meteg). Plus geresh (U+05F3),
|
|
28
|
+
# gershayim (U+05F4) when used as accents. Plus paseq (U+05C0),
|
|
29
|
+
# sof-pasuq (U+05C3), upper dot (U+05C4), lower dot (U+05C5),
|
|
30
|
+
# qamats qatan (U+05C7), and raph (U+05BF).
|
|
31
|
+
# Shin dot (U+05C1) and sin dot (U+05C2) are included so
|
|
32
|
+
# שׁ and שׂ both normalize to ש.
|
|
33
|
+
# Dagesh (U+05BC) is handled separately so callers can keep it.
|
|
34
|
+
NIKKUD_AND_CANTILLATION = /[֑-ֻֽֿ׀-ׇׂׅׄ]/
|
|
35
|
+
|
|
36
|
+
DAGESH = "ּ"
|
|
37
|
+
MAQAF = "־"
|
|
38
|
+
|
|
39
|
+
# @param strip_dagesh [Boolean] default true. When true,
|
|
40
|
+
# removes the dagesh dot from letters.
|
|
41
|
+
# @param normalize_maqaf [Boolean] default true. When true,
|
|
42
|
+
# replaces the Hebrew hyphen (maqaf) with an ASCII hyphen
|
|
43
|
+
# for consistent tokenization.
|
|
44
|
+
def initialize(strip_dagesh: true, normalize_maqaf: true)
|
|
45
|
+
@strip_dagesh = strip_dagesh
|
|
46
|
+
@normalize_maqaf = normalize_maqaf
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Normalize text: NFC + niqqud stripping + optional dagesh
|
|
50
|
+
# stripping + maqaf normalization. Skips Base's downcase
|
|
51
|
+
# step because Hebrew has no case.
|
|
52
|
+
#
|
|
53
|
+
# @param text [String]
|
|
54
|
+
# @param _options [Hash] ignored
|
|
55
|
+
# @return [String]
|
|
56
|
+
def normalize(text, _options = {})
|
|
57
|
+
return "" if text.nil? || text.empty?
|
|
58
|
+
|
|
59
|
+
result = text.unicode_normalize(:nfc)
|
|
60
|
+
result = result.gsub(NIKKUD_AND_CANTILLATION, "")
|
|
61
|
+
result = result.delete(DAGESH) if @strip_dagesh
|
|
62
|
+
result = result.tr(MAQAF, "-") if @normalize_maqaf
|
|
63
|
+
result.strip
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def normalize_word(word)
|
|
67
|
+
normalize(word)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Normalizer for Persian (Farsi) script.
|
|
7
|
+
#
|
|
8
|
+
# Extends {Arabic} with Persian-specific letter mappings.
|
|
9
|
+
# Persian shares the Arabic script but uses different
|
|
10
|
+
# codepoints for several letters that look the same to a
|
|
11
|
+
# reader but are distinct in Unicode:
|
|
12
|
+
#
|
|
13
|
+
# - Arabic Yeh (ي U+064A) → Persian Yeh (ی U+06CC)
|
|
14
|
+
# - Arabic Kaf (ك U+0643) → Persian Kaf (ک U+06A9)
|
|
15
|
+
#
|
|
16
|
+
# Input text from Arabic-origin sources may contain either
|
|
17
|
+
# form. This normalizer maps both to the Persian form so a
|
|
18
|
+
# dictionary storing Persian codepoints matches.
|
|
19
|
+
class Persian < Arabic
|
|
20
|
+
# Arabic → Persian letter canonicalization.
|
|
21
|
+
PERSIAN_LETTER_MAP = {
|
|
22
|
+
"ي" => "ی", # Arabic Yeh → Persian Yeh
|
|
23
|
+
"ك" => "ک" # Arabic Kaf → Persian Kaf
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
def normalize(text, _options = {})
|
|
27
|
+
return "" if text.nil? || text.empty?
|
|
28
|
+
|
|
29
|
+
result = super
|
|
30
|
+
PERSIAN_LETTER_MAP.each { |from, to| result = result.gsub(from, to) }
|
|
31
|
+
result
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Normalizer
|
|
6
|
+
# Turkish-aware normalizer.
|
|
7
|
+
#
|
|
8
|
+
# Turkish case folding is not the Unicode default: dotted and
|
|
9
|
+
# dotless i form a separate pair. A plain downcase maps I to i,
|
|
10
|
+
# which turns DİŞARIIDA into disarida — a word no Turkish
|
|
11
|
+
# dictionary contains. Ruby's :turkic case mapping gets the pair
|
|
12
|
+
# right: I -> ı and İ -> i, so İZMİR folds to izmir and
|
|
13
|
+
# ISTANBUL folds to ıstanbul.
|
|
14
|
+
class Turkish < Base
|
|
15
|
+
# Normalize text with Turkish case folding.
|
|
16
|
+
#
|
|
17
|
+
# @param text [String] Text to normalize
|
|
18
|
+
# @param options [Hash] Normalization options
|
|
19
|
+
# @return [String] Normalized text
|
|
20
|
+
def normalize(text, options = {})
|
|
21
|
+
downcase = options.fetch(:downcase, true)
|
|
22
|
+
result = super(text, options.merge(downcase: false))
|
|
23
|
+
downcase ? turkish_downcase(result) : result
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
# Fold case with the Turkish i pair.
|
|
29
|
+
#
|
|
30
|
+
# @param text [String] Text to fold
|
|
31
|
+
# @return [String] Folded text
|
|
32
|
+
def turkish_downcase(text)
|
|
33
|
+
text.downcase(:turkic)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|