kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Portuguese text.
|
|
7
|
+
#
|
|
8
|
+
# Ported from LanguageTool's PortugueseWordTokenizer.
|
|
9
|
+
#
|
|
10
|
+
# Handles:
|
|
11
|
+
# - Decimal comma between digits (3,14)
|
|
12
|
+
# - Dotted numbers (1.000.000)
|
|
13
|
+
# - Dates (01.01.2024, 2024-01-01)
|
|
14
|
+
# - Colons in time (12:25)
|
|
15
|
+
# - Hyphens with do-not-split list
|
|
16
|
+
# - Spaced decimals (2 000 000)
|
|
17
|
+
class PortugueseTokenizer < Base
|
|
18
|
+
# Portuguese word separators - most punctuation and whitespace
|
|
19
|
+
# Note: We protect special patterns before splitting
|
|
20
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>@€£\\$%‰‱ºªᵃᵒˢ|`~#^·]/
|
|
21
|
+
|
|
22
|
+
# Placeholder characters (using non-printing characters)
|
|
23
|
+
DECIMAL_COMMA_SUBST = "\uE001"
|
|
24
|
+
NON_BREAKING_SPACE_SUBST = "\uE002"
|
|
25
|
+
NON_BREAKING_DOT_SUBST = "\uE003"
|
|
26
|
+
NON_BREAKING_COLON_SUBST = "\uE004"
|
|
27
|
+
|
|
28
|
+
# Decimal comma between digits: 3,14
|
|
29
|
+
DECIMAL_COMMA_PATTERN = /(\d),(\d)/
|
|
30
|
+
|
|
31
|
+
# Dotted numbers: 1.000.000
|
|
32
|
+
DOTTED_NUMBERS_PATTERN = /(\d)\.(\d)/
|
|
33
|
+
|
|
34
|
+
# Colon in numbers (time): 12:25
|
|
35
|
+
COLON_NUMBERS_PATTERN = /(\d):(\d)/
|
|
36
|
+
|
|
37
|
+
# Date patterns: 01.01.2024, 2024-01-01
|
|
38
|
+
DATE_PATTERN = /(\d{2})\.(\d{2})\.(\d{4})|(\d{4})\.(\d{2})\.(\d{2})|(\d{4})-(\d{2})-(\d{2})/
|
|
39
|
+
|
|
40
|
+
# Spaced decimals: 2 000 000
|
|
41
|
+
SPACED_DECIMAL_PATTERN = /(?<=^|[\s(])\d{1,3}( \d{3})+(?:[,#{DECIMAL_COMMA_SUBST}#{NON_BREAKING_DOT_SUBST}]\d+)?(?=\D|$)/
|
|
42
|
+
|
|
43
|
+
# Do-not-split list (from LanguageTool)
|
|
44
|
+
DO_NOT_SPLIT = %w[
|
|
45
|
+
mers-cov mcgraw-hill sars-cov-2 sars-cov
|
|
46
|
+
ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
|
|
47
|
+
].freeze
|
|
48
|
+
|
|
49
|
+
def tokenize(text)
|
|
50
|
+
return [] if text.nil? || text.strip.empty?
|
|
51
|
+
|
|
52
|
+
# Handle decimal commas
|
|
53
|
+
if text.include?(",")
|
|
54
|
+
text = text.gsub(DECIMAL_COMMA_PATTERN, "\\1#{DECIMAL_COMMA_SUBST}\\2")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Handle dots in numbers and dates
|
|
58
|
+
if text.include?(".")
|
|
59
|
+
# Handle dates first (before dotted numbers to avoid conflicts)
|
|
60
|
+
text = text.gsub(DATE_PATTERN) do |match|
|
|
61
|
+
# match[0] is the full match, match[1-9] are the capture groups
|
|
62
|
+
if match[1] && match[2] && match[3] # DD.MM.YYYY
|
|
63
|
+
"#{match[1]}#{NON_BREAKING_DOT_SUBST}#{match[2]}#{NON_BREAKING_DOT_SUBST}#{match[3]}"
|
|
64
|
+
elsif match[4] && match[5] && match[6] # YYYY.MM.DD
|
|
65
|
+
"#{match[4]}#{NON_BREAKING_DOT_SUBST}#{match[5]}#{NON_BREAKING_DOT_SUBST}#{match[6]}"
|
|
66
|
+
elsif match[7] && match[8] && match[9] # YYYY-MM-DD (keep as-is)
|
|
67
|
+
match[0]
|
|
68
|
+
else
|
|
69
|
+
match[0]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
text = text.gsub(DOTTED_NUMBERS_PATTERN, "\\1#{NON_BREAKING_DOT_SUBST}\\2")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Handle spaced decimals: 2 000 000
|
|
76
|
+
text = handle_spaced_decimals(text)
|
|
77
|
+
|
|
78
|
+
# Handle colons in time: 12:25
|
|
79
|
+
if text.include?(":")
|
|
80
|
+
text = text.gsub(COLON_NUMBERS_PATTERN, "\\1#{NON_BREAKING_COLON_SUBST}\\2")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Split on word boundaries
|
|
84
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
85
|
+
|
|
86
|
+
# Process each token
|
|
87
|
+
tokens = []
|
|
88
|
+
raw_tokens.each do |token|
|
|
89
|
+
next if token.empty?
|
|
90
|
+
|
|
91
|
+
# Restore placeholders
|
|
92
|
+
token = restore_placeholders(token)
|
|
93
|
+
|
|
94
|
+
# Handle hyphenated words
|
|
95
|
+
parts = words_to_add(token)
|
|
96
|
+
tokens.concat(parts)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Filter and normalize
|
|
100
|
+
tokens
|
|
101
|
+
.map { |token| normalize(token) }
|
|
102
|
+
.reject { |token| skip_token?(token) }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
protected
|
|
106
|
+
|
|
107
|
+
# Restore placeholders to original characters.
|
|
108
|
+
#
|
|
109
|
+
# @param token [String] Token with placeholders
|
|
110
|
+
# @return [String] Token with restored characters
|
|
111
|
+
def restore_placeholders(token)
|
|
112
|
+
token
|
|
113
|
+
.gsub(DECIMAL_COMMA_SUBST, ",")
|
|
114
|
+
.gsub(NON_BREAKING_COLON_SUBST, ":")
|
|
115
|
+
.gsub(NON_BREAKING_SPACE_SUBST, " ")
|
|
116
|
+
.gsub(NON_BREAKING_DOT_SUBST, ".")
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Split a word into tokens, handling hyphens.
|
|
120
|
+
#
|
|
121
|
+
# @param word [String] Word to split
|
|
122
|
+
# @return [Array<String>] Array of tokens
|
|
123
|
+
def words_to_add(word)
|
|
124
|
+
return [word] unless word.include?("-")
|
|
125
|
+
|
|
126
|
+
# Check do-not-split list
|
|
127
|
+
return [word] if DO_NOT_SPLIT.include?(word.downcase)
|
|
128
|
+
|
|
129
|
+
# For now, split on hyphens if not in do-not-split list
|
|
130
|
+
# Future: integrate with tagger for better handling
|
|
131
|
+
word.split("-", -1).flat_map do |part|
|
|
132
|
+
part.empty? ? ["-"] : [part]
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def word_separators
|
|
137
|
+
WORD_SEPARATORS
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
private
|
|
141
|
+
|
|
142
|
+
# Handle spaced decimals: 2 000 000.
|
|
143
|
+
#
|
|
144
|
+
# @param text [String] Input text
|
|
145
|
+
# @return [String] Text with non-breaking spaces
|
|
146
|
+
def handle_spaced_decimals(text)
|
|
147
|
+
result = text
|
|
148
|
+
text.scan(SPACED_DECIMAL_PATTERN) do
|
|
149
|
+
match = Regexp.last_match
|
|
150
|
+
split_number = match[0]
|
|
151
|
+
split_number_adjusted = split_number.gsub(" ", NON_BREAKING_SPACE_SUBST)
|
|
152
|
+
split_number_adjusted = split_number_adjusted.gsub("\u00A0", NON_BREAKING_SPACE_SUBST)
|
|
153
|
+
result = result.sub(split_number, split_number_adjusted)
|
|
154
|
+
end
|
|
155
|
+
result
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Russian text.
|
|
7
|
+
#
|
|
8
|
+
# Ported from LanguageTool's RussianWordTokenizer.
|
|
9
|
+
#
|
|
10
|
+
# Handles:
|
|
11
|
+
# - Apostrophe as word character
|
|
12
|
+
# - Dot as word character (for abbreviations)
|
|
13
|
+
# - Special abbreviations: б/у (second-hand), б/н (new)
|
|
14
|
+
# - Spaced dots: .. , .
|
|
15
|
+
class RussianTokenizer < Base
|
|
16
|
+
# Russian-specific word separators (exclude apostrophe and dot)
|
|
17
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,;:!?\\\/|`~@#$%^&*+\-·]/
|
|
18
|
+
|
|
19
|
+
# Special abbreviations that should not be split
|
|
20
|
+
# Using non-printing characters as placeholders
|
|
21
|
+
ABBREVIATION_PLACEHOLDERS = {
|
|
22
|
+
"б/у" => "\u0001\u0001SOCR_BU\u0001\u0001",
|
|
23
|
+
"б/н" => "\u0001\u0001SOCR_BN\u0001\u0001"
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
# Reverse placeholders for restoration
|
|
27
|
+
PLACEHOLDER_RESTORE = {
|
|
28
|
+
"\u0001\u0001SOCR_BU\u0001\u0001" => "б/у",
|
|
29
|
+
"\u0001\u0001SOCR_BN\u0001\u0001" => "б/н",
|
|
30
|
+
"\u0001\u0001SP_DDOT_SP\u0001\u0001" => " .. ",
|
|
31
|
+
"\u0001\u0001SP_DOT_SP\u0001\u0001" => " . ",
|
|
32
|
+
"\u0001\u0001SP_DOT\u0001\u0001" => "."
|
|
33
|
+
}.freeze
|
|
34
|
+
|
|
35
|
+
def tokenize(text)
|
|
36
|
+
return [] if text.nil? || text.strip.empty?
|
|
37
|
+
|
|
38
|
+
# Replace abbreviations with placeholders
|
|
39
|
+
text = replace_abbreviations(text)
|
|
40
|
+
|
|
41
|
+
# Split on word boundaries
|
|
42
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
43
|
+
|
|
44
|
+
# Restore abbreviations and filter
|
|
45
|
+
raw_tokens
|
|
46
|
+
.map { |token| restore_abbreviations(token) }
|
|
47
|
+
.map { |token| normalize(token) }
|
|
48
|
+
.reject { |token| skip_token?(token) }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
protected
|
|
52
|
+
|
|
53
|
+
def word_separators
|
|
54
|
+
WORD_SEPARATORS
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Replace special abbreviations with placeholders.
|
|
60
|
+
#
|
|
61
|
+
# @param text [String] Input text
|
|
62
|
+
# @return [String] Text with placeholders
|
|
63
|
+
def replace_abbreviations(text)
|
|
64
|
+
result = text
|
|
65
|
+
ABBREVIATION_PLACEHOLDERS.each do |abbr, placeholder|
|
|
66
|
+
result = result.gsub(abbr, placeholder)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Handle spaced dots
|
|
70
|
+
result = result.gsub(" .. ", "\u0001\u0001SP_DDOT_SP\u0001\u0001")
|
|
71
|
+
result = result.gsub(" . ", "\u0001\u0001SP_DOT_SP\u0001\u0001")
|
|
72
|
+
result = result.gsub(" .", " \u0001\u0001SP_DOT\u0001\u0001")
|
|
73
|
+
|
|
74
|
+
# Restore spaced dots first, then single dot pattern
|
|
75
|
+
result = result.gsub("\u0001\u0001SP_DDOT_SP\u0001\u0001", " .. ")
|
|
76
|
+
result.gsub("\u0001\u0001SP_DOT_SP\u0001\u0001", " . ")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Restore abbreviations from placeholders.
|
|
80
|
+
#
|
|
81
|
+
# @param text [String] Text with placeholders
|
|
82
|
+
# @return [String] Text with restored abbreviations
|
|
83
|
+
def restore_abbreviations(text)
|
|
84
|
+
result = text
|
|
85
|
+
PLACEHOLDER_RESTORE.each do |placeholder, original|
|
|
86
|
+
result = result.gsub(placeholder, original)
|
|
87
|
+
end
|
|
88
|
+
result
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Regex-parameterized tokenizer for scripts without a dedicated
|
|
7
|
+
# tokenizer class (plan 107 basic tier).
|
|
8
|
+
#
|
|
9
|
+
# Languages without a gem module still need the plan-91 contract
|
|
10
|
+
# — #spellcheck_word_regex deciding which letters form words —
|
|
11
|
+
# for their script. Latin, Cyrillic and Greek reuse their real
|
|
12
|
+
# tokenizer classes; every other script (Arabic, Hebrew,
|
|
13
|
+
# Armenian, Georgian, Devanagari, Hangul) is served by this
|
|
14
|
+
# generic, word-run tokenizer parameterized with the script's
|
|
15
|
+
# character class. Modules remain the upgrade path: when one
|
|
16
|
+
# lands it brings tokenizer care (contractions, jamo, conjuncts)
|
|
17
|
+
# and this fallback stops being consulted for that language.
|
|
18
|
+
class ScriptTokenizer < Base
|
|
19
|
+
# @param word_regex [Regexp] characters forming a word, as a
|
|
20
|
+
# single-character regex (the plan-91 set)
|
|
21
|
+
def initialize(word_regex:)
|
|
22
|
+
super()
|
|
23
|
+
@word_regex = word_regex
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Tokenize text into words by scanning runs of word characters.
|
|
27
|
+
#
|
|
28
|
+
# @param text [String] Text to tokenize
|
|
29
|
+
# @return [Array<String>] Array of tokens
|
|
30
|
+
def tokenize(text)
|
|
31
|
+
return [] if text.nil? || text.empty?
|
|
32
|
+
|
|
33
|
+
text.scan(/#{@word_regex}+/).reject { |token| skip_token?(token) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Get word boundary regex.
|
|
37
|
+
#
|
|
38
|
+
# @return [Regexp] Word boundary regex
|
|
39
|
+
def word_boundary_regex
|
|
40
|
+
@word_regex
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Spell-check word characters (plan 91): the script set this
|
|
44
|
+
# tokenizer was built with.
|
|
45
|
+
#
|
|
46
|
+
# @return [Regexp] Regex matching a single word character
|
|
47
|
+
def spellcheck_word_regex
|
|
48
|
+
@word_regex
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
protected
|
|
52
|
+
|
|
53
|
+
# Get word characters.
|
|
54
|
+
#
|
|
55
|
+
# @return [String] Character class source
|
|
56
|
+
def word_chars
|
|
57
|
+
@word_regex.source
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Spanish text.
|
|
7
|
+
#
|
|
8
|
+
# Ported from LanguageTool's SpanishWordTokenizer.
|
|
9
|
+
#
|
|
10
|
+
# Handles:
|
|
11
|
+
# - Decimal point between digits (3.14)
|
|
12
|
+
# - Decimal comma between digits (3,14)
|
|
13
|
+
# - Ordinals (1.º, 2.ª, 1.er, 1.os, 1.as)
|
|
14
|
+
# - Hyphens (with do-not-split list since no tagger)
|
|
15
|
+
# - Soft hyphens
|
|
16
|
+
# - Inverted punctuation (¡, ¿)
|
|
17
|
+
class SpanishTokenizer < Base
|
|
18
|
+
# Spanish word separators - most punctuation and whitespace
|
|
19
|
+
# Note: We need to handle decimals specially, so we protect them first
|
|
20
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
|
|
21
|
+
|
|
22
|
+
# Decimal point between digits: 3.14
|
|
23
|
+
DECIMAL_POINT = /(\d)\.(\d)/
|
|
24
|
+
|
|
25
|
+
# Decimal comma between digits: 3,14
|
|
26
|
+
DECIMAL_COMMA = /(\d),(\d)/
|
|
27
|
+
|
|
28
|
+
# Ordinal patterns: 1.º, 2.ª, 1.er, 1.os, 1.as
|
|
29
|
+
ORDINAL = /\b(\d+)\.(º|ª|o|a|er|os|as)\b/
|
|
30
|
+
|
|
31
|
+
# Placeholders for special patterns
|
|
32
|
+
DECIMAL_POINT_PLACEHOLDER = "\uE101"
|
|
33
|
+
DECIMAL_COMMA_PLACEHOLDER = "\uE102"
|
|
34
|
+
ORDINAL_PLACEHOLDER = "\uE103"
|
|
35
|
+
|
|
36
|
+
# Soft hyphen
|
|
37
|
+
SOFT_HYPHEN = "\u00AD"
|
|
38
|
+
|
|
39
|
+
# Do-not-split list (from LanguageTool)
|
|
40
|
+
DO_NOT_SPLIT = %w[
|
|
41
|
+
mers-cov mcgraw-hill sars-cov-2 sars-cov
|
|
42
|
+
ph-metre ph-metres
|
|
43
|
+
].freeze
|
|
44
|
+
|
|
45
|
+
def tokenize(text)
|
|
46
|
+
return [] if text.nil? || text.strip.empty?
|
|
47
|
+
|
|
48
|
+
# Replace hyphen variants
|
|
49
|
+
text = text.gsub("\u2010", "\u002d") # hyphen to hyphen-minus
|
|
50
|
+
text = text.gsub("\u2011", "\u002d") # non-breaking hyphen to hyphen-minus
|
|
51
|
+
|
|
52
|
+
# Protect decimal points
|
|
53
|
+
text = text.gsub(DECIMAL_POINT, "\\1#{DECIMAL_POINT_PLACEHOLDER}\\2")
|
|
54
|
+
|
|
55
|
+
# Protect decimal commas
|
|
56
|
+
text = text.gsub(DECIMAL_COMMA, "\\1#{DECIMAL_COMMA_PLACEHOLDER}\\2")
|
|
57
|
+
|
|
58
|
+
# Protect ordinals
|
|
59
|
+
text = text.gsub(ORDINAL, "\\1#{ORDINAL_PLACEHOLDER}\\2")
|
|
60
|
+
|
|
61
|
+
# Split on word boundaries
|
|
62
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
63
|
+
|
|
64
|
+
# Process each token
|
|
65
|
+
tokens = []
|
|
66
|
+
raw_tokens.each do |token|
|
|
67
|
+
next if token.empty?
|
|
68
|
+
|
|
69
|
+
# Restore placeholders
|
|
70
|
+
token = restore_placeholders(token)
|
|
71
|
+
|
|
72
|
+
# Handle hyphenated words
|
|
73
|
+
parts = words_to_add(token)
|
|
74
|
+
tokens.concat(parts)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Filter and normalize
|
|
78
|
+
tokens
|
|
79
|
+
.map { |token| normalize(token) }
|
|
80
|
+
.reject { |token| skip_token?(token) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
protected
|
|
84
|
+
|
|
85
|
+
# Restore placeholders to original characters.
|
|
86
|
+
#
|
|
87
|
+
# @param token [String] Token with placeholders
|
|
88
|
+
# @return [String] Token with restored characters
|
|
89
|
+
def restore_placeholders(token)
|
|
90
|
+
token
|
|
91
|
+
.gsub(DECIMAL_POINT_PLACEHOLDER, ".")
|
|
92
|
+
.gsub(DECIMAL_COMMA_PLACEHOLDER, ",")
|
|
93
|
+
.gsub(ORDINAL_PLACEHOLDER, ".")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Split a word into tokens, handling hyphens.
|
|
97
|
+
#
|
|
98
|
+
# @param word [String] Word to split
|
|
99
|
+
# @return [Array<String>] Array of tokens
|
|
100
|
+
def words_to_add(word)
|
|
101
|
+
return [word] unless word.include?("-")
|
|
102
|
+
|
|
103
|
+
# Check do-not-split list
|
|
104
|
+
return [word] if DO_NOT_SPLIT.include?(word.downcase)
|
|
105
|
+
|
|
106
|
+
# Remove soft hyphens and check
|
|
107
|
+
normalized = word.gsub(SOFT_HYPHEN, "").gsub("'", "'")
|
|
108
|
+
|
|
109
|
+
# For now, split on hyphens if not in do-not-split list
|
|
110
|
+
# Future: integrate with tagger for better handling
|
|
111
|
+
normalized.split("-", -1).flat_map do |part|
|
|
112
|
+
part.empty? ? ["-"] : [part]
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def word_separators
|
|
117
|
+
WORD_SEPARATORS
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
# Language module for multi-language support.
|
|
5
|
+
#
|
|
6
|
+
# Provides language detection, tokenization, and normalization
|
|
7
|
+
# for different languages with proper OOP design.
|
|
8
|
+
#
|
|
9
|
+
# @example Detect language
|
|
10
|
+
# Kotoshu::Language.detect("Hello world") # => "en"
|
|
11
|
+
#
|
|
12
|
+
# @example Get language class
|
|
13
|
+
# lang_class = Kotoshu::Language.get("en-US")
|
|
14
|
+
#
|
|
15
|
+
# @example List supported languages
|
|
16
|
+
# Kotoshu::Language.supported_codes # => ["de-DE", "en-US", ...]
|
|
17
|
+
module Language
|
|
18
|
+
autoload :Registry, "kotoshu/language/registry"
|
|
19
|
+
autoload :Detector, "kotoshu/language/detector"
|
|
20
|
+
# Detection value + the native LID reader behind it (plan 106) share
|
|
21
|
+
# lid_detector.rb.
|
|
22
|
+
autoload :Detection, "kotoshu/language/lid_detector"
|
|
23
|
+
autoload :LanguageIdentifier, "kotoshu/language/identifier"
|
|
24
|
+
autoload :LidDetector, "kotoshu/language/lid_detector"
|
|
25
|
+
autoload :Segmenter, "kotoshu/language/segmenter"
|
|
26
|
+
autoload :Segment, "kotoshu/language/segmenter"
|
|
27
|
+
autoload :Suika, "kotoshu/language/suika"
|
|
28
|
+
# Script classification for module-less languages (plan 107).
|
|
29
|
+
autoload :Script, "kotoshu/language/script"
|
|
30
|
+
|
|
31
|
+
module Tokenizer
|
|
32
|
+
autoload :Base, "kotoshu/language/tokenizer/base"
|
|
33
|
+
autoload :LatinTokenizer, "kotoshu/language/tokenizer/latin_tokenizer"
|
|
34
|
+
autoload :FrenchTokenizer, "kotoshu/language/tokenizer/french_tokenizer"
|
|
35
|
+
autoload :GermanTokenizer, "kotoshu/language/tokenizer/german_tokenizer"
|
|
36
|
+
autoload :SpanishTokenizer, "kotoshu/language/tokenizer/spanish_tokenizer"
|
|
37
|
+
autoload :PortugueseTokenizer, "kotoshu/language/tokenizer/portuguese_tokenizer"
|
|
38
|
+
autoload :RussianTokenizer, "kotoshu/language/tokenizer/russian_tokenizer"
|
|
39
|
+
autoload :JapaneseTokenizer, "kotoshu/language/tokenizer/japanese_tokenizer"
|
|
40
|
+
autoload :GreekTokenizer, "kotoshu/language/tokenizer/greek_tokenizer"
|
|
41
|
+
autoload :CyrillicTokenizer, "kotoshu/language/tokenizer/cyrillic_tokenizer"
|
|
42
|
+
autoload :ScriptTokenizer, "kotoshu/language/tokenizer/script_tokenizer"
|
|
43
|
+
autoload :HangulTokenizer, "kotoshu/language/tokenizer/hangul_tokenizer"
|
|
44
|
+
autoload :DevanagariTokenizer, "kotoshu/language/tokenizer/devanagari_tokenizer"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
module Normalizer
|
|
48
|
+
autoload :Base, "kotoshu/language/normalizer/base"
|
|
49
|
+
autoload :Arabic, "kotoshu/language/normalizer/arabic"
|
|
50
|
+
autoload :Hebrew, "kotoshu/language/normalizer/hebrew"
|
|
51
|
+
autoload :Persian, "kotoshu/language/normalizer/persian"
|
|
52
|
+
autoload :Greek, "kotoshu/language/normalizer/greek"
|
|
53
|
+
autoload :Turkish, "kotoshu/language/normalizer/turkish"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Base class for per-language implementations (Kotoshu::Languages::*).
|
|
57
|
+
# File path is historical (languages/base.rb under language/).
|
|
58
|
+
autoload :Base, "kotoshu/language/languages/base"
|
|
59
|
+
|
|
60
|
+
class << self
|
|
61
|
+
# Detect language from text.
|
|
62
|
+
#
|
|
63
|
+
# Delegates to Detector.
|
|
64
|
+
#
|
|
65
|
+
# @param text [String] Text to analyze
|
|
66
|
+
# @return [String, nil] Detected language code
|
|
67
|
+
def detect(text)
|
|
68
|
+
Detector.detect(text)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Detect with confidence score.
|
|
72
|
+
#
|
|
73
|
+
# @param text [String] Text to analyze
|
|
74
|
+
# @return [Array<String, Float>] Language code and confidence
|
|
75
|
+
def detect_with_confidence(text)
|
|
76
|
+
Detector.detect_with_confidence(text)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Get language class by code.
|
|
80
|
+
#
|
|
81
|
+
# Delegates to Registry.
|
|
82
|
+
#
|
|
83
|
+
# @param code [String] Language code
|
|
84
|
+
# @return [Class, nil] Language class or nil
|
|
85
|
+
def get(code)
|
|
86
|
+
Registry.get(code)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Check if language is registered.
|
|
90
|
+
#
|
|
91
|
+
# @param code [String] Language code
|
|
92
|
+
# @return [Boolean] True if registered
|
|
93
|
+
def registered?(code)
|
|
94
|
+
Registry.registered?(code)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Get all supported language codes.
|
|
98
|
+
#
|
|
99
|
+
# @return [Array<String>] List of codes
|
|
100
|
+
def supported_codes
|
|
101
|
+
Registry.supported_codes
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Get language info.
|
|
105
|
+
#
|
|
106
|
+
# @param code [String] Language code
|
|
107
|
+
# @return [Hash, nil] Language info or nil
|
|
108
|
+
def info(code)
|
|
109
|
+
Registry.info(code)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Register a language.
|
|
113
|
+
#
|
|
114
|
+
# @param code [String] Language code
|
|
115
|
+
# @param klass [Class] Language class
|
|
116
|
+
# @return [void]
|
|
117
|
+
def register(code, klass)
|
|
118
|
+
Registry.register(code, klass)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Register the default detector with the registry.
|
|
125
|
+
# Both Registry and Detector are autoloaded above; referencing them
|
|
126
|
+
# here triggers their loads.
|
|
127
|
+
Kotoshu::Language::Registry.register_detector(Kotoshu::Language::Detector)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Arabic language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Registers the Arabic language with {Language::Registry} under the
|
|
8
|
+
# +ar+ and +ar-SA+ codes. Uses the {Language::Normalizer::Arabic}
|
|
9
|
+
# normalizer (NFC + presentation-form canonicalization + tashkeel
|
|
10
|
+
# stripping) so that the same word typed in different input
|
|
11
|
+
# methods produces the same normalized form for dictionary lookup.
|
|
12
|
+
#
|
|
13
|
+
# @example
|
|
14
|
+
# lang = Kotoshu::Languages::Arabic.new
|
|
15
|
+
# lang.script_type # => :arabic
|
|
16
|
+
# lang.rtl? # => true
|
|
17
|
+
#
|
|
18
|
+
# Full-feature since plan 100 (batch 3): the spelling dictionary
|
|
19
|
+
# downloads from the dictionaries repo (AVAILABLE_LANGUAGES) and
|
|
20
|
+
# keyboard proximity uses the Arabic 101 grid
|
|
21
|
+
# (Keyboard::Layouts::Arabic101).
|
|
22
|
+
class Arabic < Language::Base
|
|
23
|
+
register "ar"
|
|
24
|
+
register "ar-SA"
|
|
25
|
+
|
|
26
|
+
def initialize(code: "ar", name: "Arabic", variant: nil)
|
|
27
|
+
super
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def description
|
|
31
|
+
name
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def tokenizer
|
|
35
|
+
@tokenizer ||= Tokenizer.new
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def normalizer
|
|
39
|
+
@normalizer ||= Language::Normalizer::Arabic.new
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def dictionary_class
|
|
43
|
+
Dictionary::Custom
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def default_dictionary_paths
|
|
47
|
+
["/usr/share/dict/arabic"]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def script_type
|
|
51
|
+
:arabic
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Arabic is written right-to-left.
|
|
55
|
+
def rtl?
|
|
56
|
+
true
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Minimal Arabic tokenizer.
|
|
60
|
+
#
|
|
61
|
+
# Splits on whitespace and standard punctuation — Arabic text
|
|
62
|
+
# is space-delimited. Uses Base#skip_token? which correctly
|
|
63
|
+
# handles Unicode letters via \p{L}, so Arabic characters are
|
|
64
|
+
# kept while pure numbers and single non-letter chars are
|
|
65
|
+
# filtered.
|
|
66
|
+
class Tokenizer < Language::Tokenizer::Base
|
|
67
|
+
ARABIC_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·،؛؟]/
|
|
68
|
+
|
|
69
|
+
def tokenize(text)
|
|
70
|
+
return [] if text.nil? || text.strip.empty?
|
|
71
|
+
|
|
72
|
+
text.split(ARABIC_SEPARATORS)
|
|
73
|
+
.map { |token| normalize(token) }
|
|
74
|
+
.reject { |token| skip_token?(token) }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Spell-check word characters (plan 91 pattern, added by
|
|
78
|
+
# plan 100): Arabic-script letters. Arabic needs no ZWNJ (the
|
|
79
|
+
# staged dictionary carries none) and no apostrophe.
|
|
80
|
+
#
|
|
81
|
+
# @return [Regexp] Regex matching a single word character
|
|
82
|
+
def spellcheck_word_regex
|
|
83
|
+
/\p{Arabic}/
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
protected
|
|
87
|
+
|
|
88
|
+
def normalize(token)
|
|
89
|
+
token.strip
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def word_separators
|
|
93
|
+
ARABIC_SEPARATORS
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Bulgarian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Cyrillic composition like Ukrainian: the Cyrillic tokenizer
|
|
8
|
+
# keeps Bulgarian-script words, the base normalizer suffices —
|
|
9
|
+
# Bulgarian case folding has no special pairs. The Bulgarian-only
|
|
10
|
+
# letter ъ (and ь) are real keys on the Bulgarian BDS grid
|
|
11
|
+
# (Keyboard::Layouts::BulgarianBds), not the Russian JCUKEN the
|
|
12
|
+
# pre-existing Cyrillic layout models.
|
|
13
|
+
class Bulgarian < LatinBase
|
|
14
|
+
register "bg"
|
|
15
|
+
register "bg-BG"
|
|
16
|
+
|
|
17
|
+
def initialize(code: "bg", name: "Bulgarian")
|
|
18
|
+
super
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Cyrillic-script tokenizer.
|
|
22
|
+
#
|
|
23
|
+
# @return [Language::Tokenizer::CyrillicTokenizer]
|
|
24
|
+
def tokenizer
|
|
25
|
+
@tokenizer ||= Language::Tokenizer::CyrillicTokenizer.new
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def script_type
|
|
29
|
+
:cyrillic
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def default_dictionary_paths
|
|
33
|
+
["/usr/share/dict/bulgarian"]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|