kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
5
|
+
module Kotoshu
|
|
6
|
+
module Language
|
|
7
|
+
# Script classification for languages without a module (plan 107).
|
|
8
|
+
#
|
|
9
|
+
# Language modules are the full-feature tier: they bring keyboard
|
|
10
|
+
# layouts, tokenizer care and verified specimens. Every language
|
|
11
|
+
# staged in the dictionaries manifest is still usable without
|
|
12
|
+
# one — this module maps a language code to its writing script
|
|
13
|
+
# and supplies the tokenizer the spell-check word extraction
|
|
14
|
+
# (plan 91) rides on:
|
|
15
|
+
#
|
|
16
|
+
# - Latin, Cyrillic and Greek reuse their real tokenizer classes;
|
|
17
|
+
# - other scripts get a generic {Tokenizer::ScriptTokenizer}
|
|
18
|
+
# parameterized with the script's Unicode set;
|
|
19
|
+
# - unknown codes return nil and the ASCII fallback applies.
|
|
20
|
+
#
|
|
21
|
+
# The map covers the languages of the dictionaries manifest plus
|
|
22
|
+
# their near neighbors, so a code is misclassified only when the
|
|
23
|
+
# manifest grows without this file — which the
|
|
24
|
+
# kotoshu:staged_languages:sync rake task surfaces.
|
|
25
|
+
module Script
|
|
26
|
+
# Language code -> script symbol for every manifest language.
|
|
27
|
+
# Latin is the default and therefore not listed.
|
|
28
|
+
SCRIPT_BY_LANGUAGE = {
|
|
29
|
+
# Cyrillic (module languages bg ru sr uk plus staged mk mn).
|
|
30
|
+
"bg" => :cyrillic, "ru" => :cyrillic, "sr" => :cyrillic,
|
|
31
|
+
"uk" => :cyrillic, "be" => :cyrillic,
|
|
32
|
+
"mk" => :cyrillic, "mn" => :cyrillic,
|
|
33
|
+
# Greek (el; the manifest also stages el-polyton).
|
|
34
|
+
"el" => :greek, "el-polyton" => :greek,
|
|
35
|
+
# Arabic script (ar fa; fa also needs ZWNJ, which its module
|
|
36
|
+
# tokenizer handles — this generic set is the module-less
|
|
37
|
+
# fallback only).
|
|
38
|
+
"ar" => :arabic, "fa" => :arabic, "ur" => :arabic, "ps" => :arabic,
|
|
39
|
+
# Hebrew script.
|
|
40
|
+
"he" => :hebrew, "yi" => :hebrew,
|
|
41
|
+
# Armenian (staged hy hyw).
|
|
42
|
+
"hy" => :armenian, "hyw" => :armenian,
|
|
43
|
+
# Georgian (staged ka).
|
|
44
|
+
"ka" => :georgian,
|
|
45
|
+
# Devanagari (staged ne; hi mr sa ride the same script).
|
|
46
|
+
"ne" => :devanagari, "hi" => :devanagari, "mr" => :devanagari,
|
|
47
|
+
"sa" => :devanagari,
|
|
48
|
+
# Hangul (staged ko).
|
|
49
|
+
"ko" => :hangul
|
|
50
|
+
}.freeze
|
|
51
|
+
|
|
52
|
+
# Word-character regex per script, for the generic tokenizer.
|
|
53
|
+
SCRIPT_WORD_REGEXES = {
|
|
54
|
+
arabic: /\p{Arabic}/,
|
|
55
|
+
hebrew: /\p{Hebrew}/,
|
|
56
|
+
armenian: /\p{Armenian}/,
|
|
57
|
+
georgian: /\p{Georgian}/
|
|
58
|
+
}.freeze
|
|
59
|
+
|
|
60
|
+
# Tokenizer classes reused from the dedicated script tokenizers
|
|
61
|
+
# (plan 108 added Hangul and Devanagari alongside the plan 84
|
|
62
|
+
# Cyrillic and Greek classes).
|
|
63
|
+
SCRIPT_TOKENIZER_CLASSES = {
|
|
64
|
+
cyrillic: Tokenizer::CyrillicTokenizer,
|
|
65
|
+
greek: Tokenizer::GreekTokenizer,
|
|
66
|
+
hangul: Tokenizer::HangulTokenizer,
|
|
67
|
+
devanagari: Tokenizer::DevanagariTokenizer
|
|
68
|
+
}.freeze
|
|
69
|
+
|
|
70
|
+
class << self
|
|
71
|
+
# The writing script of a language code (:latin default).
|
|
72
|
+
#
|
|
73
|
+
# @param code [String] Language code (e.g. "mk", "ko")
|
|
74
|
+
# @return [Symbol] Script name (:latin, :cyrillic, ...)
|
|
75
|
+
def script_for(code)
|
|
76
|
+
return :latin if code.nil? || code.empty?
|
|
77
|
+
|
|
78
|
+
SCRIPT_BY_LANGUAGE.fetch(normalized_base(code), :latin)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The script-appropriate tokenizer for a language without a
|
|
82
|
+
# module, or nil when the language is unknown (the ASCII
|
|
83
|
+
# fallback then applies, exactly as before plan 107).
|
|
84
|
+
#
|
|
85
|
+
# Latin-script languages return the shared LatinTokenizer —
|
|
86
|
+
# the same tokenizer every LatinBase module uses — so the
|
|
87
|
+
# basic tier and the full-feature tier agree on word
|
|
88
|
+
# extraction until a module adds language-specific care.
|
|
89
|
+
#
|
|
90
|
+
# @param code [String] Language code
|
|
91
|
+
# @return [Tokenizer::Base, nil] Tokenizer or nil
|
|
92
|
+
def tokenizer_for(code)
|
|
93
|
+
return nil if code.nil? || code.empty?
|
|
94
|
+
|
|
95
|
+
script = script_for(code)
|
|
96
|
+
if script == :latin
|
|
97
|
+
latin_script_languages.include?(normalized_base(code)) ? Tokenizer::LatinTokenizer.new : nil
|
|
98
|
+
elsif SCRIPT_TOKENIZER_CLASSES.key?(script)
|
|
99
|
+
SCRIPT_TOKENIZER_CLASSES.fetch(script).new
|
|
100
|
+
elsif SCRIPT_WORD_REGEXES.key?(script)
|
|
101
|
+
Tokenizer::ScriptTokenizer.new(word_regex: SCRIPT_WORD_REGEXES.fetch(script))
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
# Downcase and strip the region ("sv-FI" -> "sv", "en-GB" ->
|
|
108
|
+
# "en"), mirroring ResourceManager's language normalization.
|
|
109
|
+
def normalized_base(code)
|
|
110
|
+
code.to_s.split("-").first.split("_").first.downcase
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Latin-script codes this module knows: the manifest's Latin
|
|
114
|
+
# languages. Unknown Latin-ish codes stay nil so new manifest
|
|
115
|
+
# entries do not silently widen word extraction beyond the
|
|
116
|
+
# vendored list (the sync rake task refreshes it).
|
|
117
|
+
def latin_script_languages
|
|
118
|
+
@latin_script_languages ||= begin
|
|
119
|
+
classified = SCRIPT_BY_LANGUAGE.keys.map { |k| normalized_base(k) }.to_set
|
|
120
|
+
Cache::StagedLanguages::CODES.map { |c| normalized_base(c) }.uniq
|
|
121
|
+
.reject { |c| classified.include?(c) }
|
|
122
|
+
.to_set
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
# Value object: a contiguous run of text nodes that all share a
|
|
6
|
+
# detected language. Produced by {Segmenter#segment}.
|
|
7
|
+
#
|
|
8
|
+
# Carries the detected +language_code+, the +text_nodes+ belonging
|
|
9
|
+
# to the segment (in source order), and the +confidence+ the
|
|
10
|
+
# detector returned for the segment's text. Multi-language
|
|
11
|
+
# checkers consume segments one at a time, resolving resources
|
|
12
|
+
# per-language.
|
|
13
|
+
Segment = Struct.new(:language_code, :text_nodes, :confidence, keyword_init: true) do
|
|
14
|
+
# Concatenated flattened text of every node in this segment.
|
|
15
|
+
#
|
|
16
|
+
# @return [String]
|
|
17
|
+
def flattened_text
|
|
18
|
+
text_nodes.map(&:text).join
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Number of text nodes in this segment.
|
|
22
|
+
#
|
|
23
|
+
# @return [Integer]
|
|
24
|
+
def length
|
|
25
|
+
text_nodes.length
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# True when no text nodes are in the segment.
|
|
29
|
+
#
|
|
30
|
+
# @return [Boolean]
|
|
31
|
+
def empty?
|
|
32
|
+
text_nodes.nil? || text_nodes.empty?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Splits a {Documents::Document} into language-tagged segments.
|
|
37
|
+
#
|
|
38
|
+
# Walks +document.text_nodes+, detects the dominant language for
|
|
39
|
+
# each node via the configured detector (default: {Detector}, the
|
|
40
|
+
# pure-Ruby character-set-based detector — no fasttext needed),
|
|
41
|
+
# and groups consecutive same-language nodes into {Segment}s.
|
|
42
|
+
#
|
|
43
|
+
# The detector is pluggable via the +detector:+ constructor
|
|
44
|
+
# argument. Any object that responds to +detect_with_confidence+
|
|
45
|
+
# and returns +[code, confidence]+ will do — the FastText-backed
|
|
46
|
+
# {LanguageIdentifier} is a drop-in for higher accuracy at the
|
|
47
|
+
# cost of a model download.
|
|
48
|
+
class Segmenter
|
|
49
|
+
DEFAULT_DETECTOR = Detector
|
|
50
|
+
|
|
51
|
+
# Minimum text length (chars) for a node to participate in
|
|
52
|
+
# detection. Shorter nodes ("a", "the", whitespace) don't carry
|
|
53
|
+
# enough signal — they inherit the surrounding language.
|
|
54
|
+
MIN_NODE_LENGTH_FOR_DETECTION = 3
|
|
55
|
+
|
|
56
|
+
# @param detector [Object, nil] responds to +detect_with_confidence+.
|
|
57
|
+
# Defaults to {DEFAULT_DETECTOR}.
|
|
58
|
+
# @param min_confidence [Float] detection results below this
|
|
59
|
+
# threshold are treated as unknown (segment language becomes
|
|
60
|
+
# the fallback language, default "en").
|
|
61
|
+
# @param fallback_language [String] language code to use when
|
|
62
|
+
# detection is inconclusive.
|
|
63
|
+
def initialize(detector: nil, min_confidence: 0.3, fallback_language: "en")
|
|
64
|
+
@detector = detector || DEFAULT_DETECTOR
|
|
65
|
+
@min_confidence = min_confidence
|
|
66
|
+
@fallback_language = fallback_language
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Segment a document.
|
|
70
|
+
#
|
|
71
|
+
# @param document [Kotoshu::Documents::Document]
|
|
72
|
+
# @return [Array<Segment>] language-tagged segments, in source order
|
|
73
|
+
def segment(document)
|
|
74
|
+
unless document.is_a?(Kotoshu::Documents::Document)
|
|
75
|
+
raise ArgumentError,
|
|
76
|
+
"document must be a Kotoshu::Documents::Document"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
segments = []
|
|
80
|
+
current = nil
|
|
81
|
+
|
|
82
|
+
document.text_nodes.each do |node|
|
|
83
|
+
code, confidence = detect_for_node(node)
|
|
84
|
+
if current && current.language_code == code
|
|
85
|
+
current.text_nodes << node
|
|
86
|
+
else
|
|
87
|
+
segments << current if current && !current.empty?
|
|
88
|
+
current = Segment.new(language_code: code, text_nodes: [node],
|
|
89
|
+
confidence: confidence)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
segments << current if current && !current.empty?
|
|
93
|
+
segments
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
# Detect language for a single node. Short nodes (< MIN length)
|
|
99
|
+
# inherit nil and the caller's group-merge logic picks up the
|
|
100
|
+
# previous segment's language (because nil != previous code).
|
|
101
|
+
# To avoid fragmenting segments on short connector words, we
|
|
102
|
+
# return the previous detection — but Ruby doesn't easily carry
|
|
103
|
+
# state into a `private` helper without ivars, so the caller
|
|
104
|
+
# handles short-node inheritance via the group-merge rule.
|
|
105
|
+
#
|
|
106
|
+
# For now: short nodes return the fallback language so they
|
|
107
|
+
# merge with whatever segment is open (assuming the previous
|
|
108
|
+
# detection was also the fallback). This is a heuristic; a
|
|
109
|
+
# richer segmenter would carry last-detected-code as state.
|
|
110
|
+
def detect_for_node(node)
|
|
111
|
+
text = node.text
|
|
112
|
+
return [@fallback_language, 0.0] if text.nil? || text.strip.empty?
|
|
113
|
+
return [@fallback_language, 0.0] if text.strip.length < MIN_NODE_LENGTH_FOR_DETECTION
|
|
114
|
+
|
|
115
|
+
code, confidence = @detector.detect_with_confidence(text)
|
|
116
|
+
return [@fallback_language, 0.0] if code.nil? || confidence < @min_confidence
|
|
117
|
+
|
|
118
|
+
[code, confidence]
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
# Soft-load suika. The gem is intentionally NOT a hard runtime
|
|
6
|
+
# dependency — its native extension (dartsclone) fails to build on
|
|
7
|
+
# slim/minimal environments and would block install for users who
|
|
8
|
+
# only want non-Japanese spell-checking. Japanese tokenization and
|
|
9
|
+
# POS tagging light up automatically when the gem is present.
|
|
10
|
+
#
|
|
11
|
+
# KOTOSHU_NO_SUIKA=1 forces Japanese analysis off even when the gem
|
|
12
|
+
# is installed (useful for benchmarks / CI determinism).
|
|
13
|
+
module Suika
|
|
14
|
+
LOADED = begin
|
|
15
|
+
if ENV["KOTOSHU_NO_SUIKA"] == "1"
|
|
16
|
+
false
|
|
17
|
+
else
|
|
18
|
+
require "suika"
|
|
19
|
+
true
|
|
20
|
+
end
|
|
21
|
+
rescue LoadError
|
|
22
|
+
false
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
@tagger = nil
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
# Return a process-wide memoized Suika::Tagger, or raise
|
|
29
|
+
# {SuikaUnavailable} when the gem is missing.
|
|
30
|
+
def tagger
|
|
31
|
+
raise Kotoshu::SuikaUnavailable unless LOADED
|
|
32
|
+
|
|
33
|
+
@tagger ||= ::Suika::Tagger.new
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Abstract base class for tokenizers.
|
|
7
|
+
#
|
|
8
|
+
# Uses Strategy pattern to allow different tokenization approaches
|
|
9
|
+
# for different languages.
|
|
10
|
+
#
|
|
11
|
+
# Subclasses must implement the tokenize method.
|
|
12
|
+
#
|
|
13
|
+
# @example Implement a tokenizer
|
|
14
|
+
# class MyTokenizer < Tokenizer::Base
|
|
15
|
+
# def tokenize(text)
|
|
16
|
+
# text.split(/ /)
|
|
17
|
+
# end
|
|
18
|
+
# end
|
|
19
|
+
class Base
|
|
20
|
+
# Tokenize text into words.
|
|
21
|
+
#
|
|
22
|
+
# @param text [String] Text to tokenize
|
|
23
|
+
# @return [Array<String>] Array of tokens
|
|
24
|
+
# @raise [NotImplementedError] Must be implemented by subclass
|
|
25
|
+
def tokenize(text)
|
|
26
|
+
raise NotImplementedError, "#{self.class} must implement #tokenize"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Tokenize text with positions.
|
|
30
|
+
#
|
|
31
|
+
# Returns tokens along with their position information.
|
|
32
|
+
#
|
|
33
|
+
# @param text [String] Text to tokenize
|
|
34
|
+
# @return [Array<Hash>] Array of {token:, start:, end:, line:, column:}
|
|
35
|
+
def tokenize_with_positions(text)
|
|
36
|
+
return [] if text.nil?
|
|
37
|
+
return [] if text.empty?
|
|
38
|
+
|
|
39
|
+
tokens = []
|
|
40
|
+
line = 1
|
|
41
|
+
column = 1
|
|
42
|
+
position = 0
|
|
43
|
+
|
|
44
|
+
while position < text.length
|
|
45
|
+
# Skip whitespace
|
|
46
|
+
while position < text.length && text[position].match?(/\s/)
|
|
47
|
+
if text[position] == "\n"
|
|
48
|
+
line += 1
|
|
49
|
+
column = 1
|
|
50
|
+
else
|
|
51
|
+
column += 1
|
|
52
|
+
end
|
|
53
|
+
position += 1
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
break if position >= text.length
|
|
57
|
+
|
|
58
|
+
# Find token
|
|
59
|
+
start_pos = position
|
|
60
|
+
start_line = line
|
|
61
|
+
start_column = column
|
|
62
|
+
|
|
63
|
+
token_text = extract_next_token(text, position)
|
|
64
|
+
|
|
65
|
+
if token_text
|
|
66
|
+
tokens << {
|
|
67
|
+
token: token_text,
|
|
68
|
+
start: start_pos,
|
|
69
|
+
end: start_pos + token_text.length,
|
|
70
|
+
line: start_line,
|
|
71
|
+
column: start_column
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
token_text.each_char do |char|
|
|
75
|
+
column += 1
|
|
76
|
+
position += 1
|
|
77
|
+
if char == "\n"
|
|
78
|
+
line += 1
|
|
79
|
+
column = 1
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
else
|
|
83
|
+
position += 1
|
|
84
|
+
column += 1
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
tokens
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Check if a character is a word character.
|
|
92
|
+
#
|
|
93
|
+
# @param char [String] Single character
|
|
94
|
+
# @return [Boolean] True if word character
|
|
95
|
+
def word_char?(char)
|
|
96
|
+
match?(word_boundary_regex, char)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Get word boundary regex for this tokenizer.
|
|
100
|
+
#
|
|
101
|
+
# Subclasses should override this to define word boundaries.
|
|
102
|
+
#
|
|
103
|
+
# @return [Regexp] Word boundary regex
|
|
104
|
+
def word_boundary_regex
|
|
105
|
+
raise NotImplementedError, "#{self.class} must implement #word_boundary_regex"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Regex matching one word-forming character for spell-check text
|
|
109
|
+
# extraction (plan 91).
|
|
110
|
+
#
|
|
111
|
+
# The Spellchecker walks text character by character with this
|
|
112
|
+
# regex, so it must cover exactly the letters this tokenizer
|
|
113
|
+
# recognizes: script subclasses override it with their script's
|
|
114
|
+
# letters, and wrong-script characters stay non-word so foreign
|
|
115
|
+
# words are never checked. The Base default is the historical
|
|
116
|
+
# ASCII set, so tokenizers without an override keep their
|
|
117
|
+
# extraction behavior; digits are never included so number
|
|
118
|
+
# handling stays as it was.
|
|
119
|
+
#
|
|
120
|
+
# @return [Regexp] Regex matching a single word character
|
|
121
|
+
def spellcheck_word_regex
|
|
122
|
+
/[a-zA-Z']/
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Normalize a token.
|
|
126
|
+
#
|
|
127
|
+
# Subclasses can override this for language-specific normalization.
|
|
128
|
+
#
|
|
129
|
+
# @param token [String] Token to normalize
|
|
130
|
+
# @return [String] Normalized token
|
|
131
|
+
def normalize(token)
|
|
132
|
+
token
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Check if a token should be skipped.
|
|
136
|
+
#
|
|
137
|
+
# Subclasses can override this for language-specific filtering.
|
|
138
|
+
#
|
|
139
|
+
# @param token [String] Token to check
|
|
140
|
+
# @return [Boolean] True if token should be skipped
|
|
141
|
+
def skip_token?(token)
|
|
142
|
+
return true if token.empty?
|
|
143
|
+
return true if token.match?(/^\d+$/) # Pure numbers
|
|
144
|
+
return true if token.length < 2 && token.match?(/^[^\p{L}]$/)
|
|
145
|
+
|
|
146
|
+
false
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
protected
|
|
150
|
+
|
|
151
|
+
# Extract the next token from text at position.
|
|
152
|
+
#
|
|
153
|
+
# @param text [String] Full text
|
|
154
|
+
# @param position [Integer] Current position
|
|
155
|
+
# @return [String, nil] Next token or nil
|
|
156
|
+
def extract_next_token(text, position)
|
|
157
|
+
remaining = text[position..]
|
|
158
|
+
match = remaining.match(/^#{word_pattern}/)
|
|
159
|
+
match ? match[0] : nil
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Get pattern for matching tokens.
|
|
163
|
+
#
|
|
164
|
+
# @return [String] Regex pattern string
|
|
165
|
+
def word_pattern
|
|
166
|
+
"[#{word_chars}]+"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Get word characters for this tokenizer.
|
|
170
|
+
#
|
|
171
|
+
# @return [String] Character class of word characters
|
|
172
|
+
def word_chars
|
|
173
|
+
raise NotImplementedError, "#{self.class} must implement #word_chars"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Check if string matches regex.
|
|
177
|
+
#
|
|
178
|
+
# @param regex [Regexp] Regex to match
|
|
179
|
+
# @param string [String] String to check
|
|
180
|
+
# @return [Boolean] True if matches
|
|
181
|
+
def match?(regex, string)
|
|
182
|
+
regex.match?(string)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Cyrillic-script text without a language-specific
|
|
7
|
+
# subclass (RussianTokenizer carries Russian abbreviation
|
|
8
|
+
# handling; Ukrainian, Belarusian, Bulgarian, Serbian and friends
|
|
9
|
+
# need only the script-aware split).
|
|
10
|
+
#
|
|
11
|
+
# Splits on the same punctuation/whitespace separators as the
|
|
12
|
+
# Latin tokenizer, but keeps tokens that contain Cyrillic
|
|
13
|
+
# letters — the Latin tokenizer's letter checks would drop a
|
|
14
|
+
# pure Cyrillic word. The apostrophe stays a word character so
|
|
15
|
+
# Ukrainian names like Мар'яна tokenize as one word.
|
|
16
|
+
class CyrillicTokenizer < Base
|
|
17
|
+
# Separators between Cyrillic words. The apostrophe is NOT a
|
|
18
|
+
# separator (Ukrainian uses it inside words); the dot is, like
|
|
19
|
+
# in the Latin tokenizer.
|
|
20
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
|
|
21
|
+
|
|
22
|
+
# Tokenize text into Cyrillic words.
|
|
23
|
+
#
|
|
24
|
+
# @param text [String] Text to tokenize
|
|
25
|
+
# @return [Array<String>] Array of tokens
|
|
26
|
+
def tokenize(text)
|
|
27
|
+
return [] if text.nil? || text.strip.empty?
|
|
28
|
+
|
|
29
|
+
text.split(WORD_SEPARATORS)
|
|
30
|
+
.map(&:strip)
|
|
31
|
+
.reject { |token| skip_token?(token) }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Get word boundary regex.
|
|
35
|
+
#
|
|
36
|
+
# @return [Regexp] Word boundary regex
|
|
37
|
+
def word_boundary_regex
|
|
38
|
+
/\p{Cyrillic}/
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Spell-check word characters (plan 91): the Cyrillic letters
|
|
42
|
+
# and apostrophe this tokenizer already declares in
|
|
43
|
+
# #word_chars (Ukrainian names like Мар'яна keep the
|
|
44
|
+
# apostrophe in-word).
|
|
45
|
+
#
|
|
46
|
+
# @return [Regexp] Regex matching a single word character
|
|
47
|
+
def spellcheck_word_regex
|
|
48
|
+
/[#{word_chars}]/o
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Check if token should be skipped.
|
|
52
|
+
#
|
|
53
|
+
# Skips tokens that contain no Cyrillic letters; Base's rules
|
|
54
|
+
# (empty, pure numbers) also apply.
|
|
55
|
+
#
|
|
56
|
+
# @param token [String] Token to check
|
|
57
|
+
# @return [Boolean] True if should skip
|
|
58
|
+
def skip_token?(token)
|
|
59
|
+
return true if super
|
|
60
|
+
|
|
61
|
+
!token.match?(/\p{Cyrillic}/)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
protected
|
|
65
|
+
|
|
66
|
+
# Get word characters.
|
|
67
|
+
#
|
|
68
|
+
# @return [String] Character class of Cyrillic letters and the
|
|
69
|
+
# apostrophe
|
|
70
|
+
def word_chars
|
|
71
|
+
"\\p{Cyrillic}'"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Devanagari-script languages (plan 108: ne).
|
|
7
|
+
#
|
|
8
|
+
# A Devanagari word is a run of block characters in which the
|
|
9
|
+
# combining marks stay attached to their base: inherent-vowel
|
|
10
|
+
# killers and matras (U+093E-094C), the virama (U+094D) that
|
|
11
|
+
# forms conjunct consonants, anusvara/visarga/candrabindu and
|
|
12
|
+
# the vowel signs — all share the Devanagari script property, so
|
|
13
|
+
# one script run IS one grapheme-cluster sequence and the
|
|
14
|
+
# plan-91 per-character regex machinery keeps each cluster
|
|
15
|
+
# intact (क + ि renders as कि and extracts as one token).
|
|
16
|
+
# ZWJ/ZWNJ (U+200C/U+200D) join the word set because conjunct
|
|
17
|
+
# spellings insert them mid-word; the danda marks (U+0964-0965)
|
|
18
|
+
# are sentence punctuation and the Devanagari digits
|
|
19
|
+
# (U+0966-096F) are digits, so both stay separators — plan 91
|
|
20
|
+
# keeps digits out of words in every script.
|
|
21
|
+
class DevanagariTokenizer < Base
|
|
22
|
+
# Devanagari block minus danda/digits/abbreviation sign, plus
|
|
23
|
+
# ZWJ/ZWNJ (regex source text, like CyrillicTokenizer's
|
|
24
|
+
# word_chars).
|
|
25
|
+
DEVANAGARI_WORD_CHARS = "\\u0900-\\u0963\\u0971-\\u097F\\u200C\\u200D"
|
|
26
|
+
|
|
27
|
+
# One word (a run of word characters).
|
|
28
|
+
WORD_RUN = /[#{DEVANAGARI_WORD_CHARS}]+/
|
|
29
|
+
|
|
30
|
+
# One word character.
|
|
31
|
+
WORD_CHAR = /[#{DEVANAGARI_WORD_CHARS}]/
|
|
32
|
+
|
|
33
|
+
# Tokenize text into words.
|
|
34
|
+
#
|
|
35
|
+
# @param text [String] Text to tokenize
|
|
36
|
+
# @return [Array<String>] Array of tokens
|
|
37
|
+
def tokenize(text)
|
|
38
|
+
return [] if text.nil? || text.empty?
|
|
39
|
+
|
|
40
|
+
text.scan(WORD_RUN).reject { |token| skip_token?(token) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Get word boundary regex.
|
|
44
|
+
#
|
|
45
|
+
# @return [Regexp] Word boundary regex
|
|
46
|
+
def word_boundary_regex
|
|
47
|
+
WORD_CHAR
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Spell-check word characters (plan 91): the Devanagari set —
|
|
51
|
+
# base consonants with their matras and virama conjuncts, no
|
|
52
|
+
# digits and no Latin.
|
|
53
|
+
#
|
|
54
|
+
# @return [Regexp] Regex matching a single word character
|
|
55
|
+
def spellcheck_word_regex
|
|
56
|
+
WORD_CHAR
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
protected
|
|
60
|
+
|
|
61
|
+
# Get word characters.
|
|
62
|
+
#
|
|
63
|
+
# @return [String] Character class source
|
|
64
|
+
def word_chars
|
|
65
|
+
DEVANAGARI_WORD_CHARS
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|