kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for French text.
|
|
7
|
+
#
|
|
8
|
+
# Ported from LanguageTool's FrenchWordTokenizer.
|
|
9
|
+
#
|
|
10
|
+
# Handles:
|
|
11
|
+
# - Apostrophes (l', d', qu', c'est, j'ai, etc.)
|
|
12
|
+
# - Hyphens (c'est-à-dire, rendez-vous, etc.)
|
|
13
|
+
# - Decimal points/commas
|
|
14
|
+
# - Multiple contraction patterns (7 total)
|
|
15
|
+
class FrenchTokenizer < Base
|
|
16
|
+
# French word separators - most punctuation and whitespace
|
|
17
|
+
# Note: apostrophe (') is NOT a separator in French (used for contractions)
|
|
18
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
|
|
19
|
+
|
|
20
|
+
# Do-not-split list (from LanguageTool)
|
|
21
|
+
DO_NOT_SPLIT = %w[
|
|
22
|
+
mers-cov mcgraw-hill sars-cov-2 sars-cov
|
|
23
|
+
ph-metre ph-metres anti-ivg anti-uv anti-vih al-qaïda
|
|
24
|
+
c'est-à-dire add-on add-ons rendez-vous garde-à-vous
|
|
25
|
+
chez-eux chez-moi chez-nous chez-soi chez-toi chez-vous
|
|
26
|
+
m'as-tu-vu
|
|
27
|
+
].freeze
|
|
28
|
+
|
|
29
|
+
# Contraction patterns (from LanguageTool)
|
|
30
|
+
# French contractions are complex: l', d', qu', c'est, j'ai, n'a, etc.
|
|
31
|
+
CONTRACTION_PATTERNS = [
|
|
32
|
+
# c' followed by word: c'est, c'était, etc.
|
|
33
|
+
/^(c')$/i,
|
|
34
|
+
# j' (je): j'ai, j'aime, etc.
|
|
35
|
+
/^(j')$/i,
|
|
36
|
+
# n' (ne): n'a, n'est, etc.
|
|
37
|
+
/^(n')$/i,
|
|
38
|
+
# m' (me): m'a, m'appelle, etc.
|
|
39
|
+
/^(m')$/i,
|
|
40
|
+
# t' (te): t'a, t'asseoir, etc.
|
|
41
|
+
/^(t')$/i,
|
|
42
|
+
# s' (se): s'a, s'appelle, etc.
|
|
43
|
+
/^(s')$/i,
|
|
44
|
+
# l' (le/la): l'a, l'homme, l'eau, etc.
|
|
45
|
+
/^(l')$/i,
|
|
46
|
+
# d' (de): d'un, d'une, d'abord, etc.
|
|
47
|
+
/^(d')$/i,
|
|
48
|
+
# qu' (que): qu'un, qu'une, qu'est, etc.
|
|
49
|
+
/^(qu')$/i,
|
|
50
|
+
# jusqu'à, jusqu'aux, etc.
|
|
51
|
+
/^(jusqu')$/i,
|
|
52
|
+
# puisque, puisqu'il, etc.
|
|
53
|
+
/^(puisqu')$/i,
|
|
54
|
+
# quoique, quoiqu'il, etc.
|
|
55
|
+
/^(quoiqu')$/i,
|
|
56
|
+
# lorsque, lorsqu'il, etc.
|
|
57
|
+
/^(lorsqu')$/i,
|
|
58
|
+
].freeze
|
|
59
|
+
|
|
60
|
+
def tokenize(text)
|
|
61
|
+
return [] if text.nil? || text.strip.empty?
|
|
62
|
+
|
|
63
|
+
# Replace hyphen variants
|
|
64
|
+
text = text.gsub("\u2010", "\u002d")
|
|
65
|
+
text = text.gsub("\u2011", "\u002d")
|
|
66
|
+
|
|
67
|
+
# Normalize apostrophes
|
|
68
|
+
text = normalize_apostrophes(text)
|
|
69
|
+
|
|
70
|
+
# Split on word boundaries
|
|
71
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
72
|
+
|
|
73
|
+
# Process each token
|
|
74
|
+
tokens = []
|
|
75
|
+
raw_tokens.each do |token|
|
|
76
|
+
next if token.empty?
|
|
77
|
+
|
|
78
|
+
# Try to split contractions and hyphenated words
|
|
79
|
+
parts = split_french_word(token)
|
|
80
|
+
tokens.concat(parts)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Filter and normalize
|
|
84
|
+
tokens
|
|
85
|
+
.map { |token| normalize(token) }
|
|
86
|
+
.reject { |token| skip_token?(token) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
protected
|
|
90
|
+
|
|
91
|
+
# Normalize apostrophes to straight quotes.
|
|
92
|
+
#
|
|
93
|
+
# @param text [String] Input text
|
|
94
|
+
# @return [String] Text with normalized apostrophes
|
|
95
|
+
def normalize_apostrophes(text)
|
|
96
|
+
text
|
|
97
|
+
.gsub("'", "'")
|
|
98
|
+
.gsub("'", "'")
|
|
99
|
+
.gsub("'", "'")
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Split French word, handling contractions and hyphens.
|
|
103
|
+
#
|
|
104
|
+
# @param word [String] Word to split
|
|
105
|
+
# @return [Array<String>] Array of tokens
|
|
106
|
+
def split_french_word(word)
|
|
107
|
+
# Check do-not-split list
|
|
108
|
+
return [word] if DO_NOT_SPLIT.include?(word.downcase)
|
|
109
|
+
|
|
110
|
+
# Handle hyphens first (but not for do-not-split words)
|
|
111
|
+
if word.include?("-")
|
|
112
|
+
# Check if it's a contraction pattern like "jusqu'à-ce"
|
|
113
|
+
if word.match?(/^(jusqu'|[cç]'|j'|n'|m'|t'|s'|l'|d'|qu'|lorsqu'|puisqu'|quoiqu')/)
|
|
114
|
+
# Split on hyphen for contractions
|
|
115
|
+
parts = []
|
|
116
|
+
word.split("-", -1).each do |part|
|
|
117
|
+
next if part.empty?
|
|
118
|
+
|
|
119
|
+
parts.concat(split_contractions(part))
|
|
120
|
+
end
|
|
121
|
+
return parts
|
|
122
|
+
else
|
|
123
|
+
# Regular hyphenated word - split it
|
|
124
|
+
return word.split("-", -1).reject(&:empty?)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Handle contractions
|
|
129
|
+
if word.include?("'")
|
|
130
|
+
return split_contractions(word)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# No special handling needed
|
|
134
|
+
[word]
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Split contractions into component parts.
|
|
138
|
+
#
|
|
139
|
+
# @param word [String] Word that might be a contraction
|
|
140
|
+
# @return [Array<String>] Array of tokens
|
|
141
|
+
def split_contractions(word)
|
|
142
|
+
# Try each contraction pattern
|
|
143
|
+
CONTRACTION_PATTERNS.each do |pattern|
|
|
144
|
+
match = word.match(pattern)
|
|
145
|
+
if match
|
|
146
|
+
# Return the contraction and the rest of the word
|
|
147
|
+
contraction = match[1]
|
|
148
|
+
rest = word.sub(/^#{Regexp.escape(contraction)}/, "")
|
|
149
|
+
return [contraction, rest] unless rest.empty?
|
|
150
|
+
|
|
151
|
+
return [contraction]
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Handle special case: word starts with apostrophe
|
|
156
|
+
if word.match?(/^[cç]'|^[a-z]'/i)
|
|
157
|
+
# Split at the apostrophe
|
|
158
|
+
parts = word.split("'", 2)
|
|
159
|
+
return parts if parts.length == 2
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# No pattern matched, return the word as-is
|
|
163
|
+
[word]
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def word_separators
|
|
167
|
+
WORD_SEPARATORS
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for German text.
|
|
7
|
+
#
|
|
8
|
+
# Ported from LanguageTool's GermanWordTokenizer.
|
|
9
|
+
#
|
|
10
|
+
# Handles:
|
|
11
|
+
# - Underscore as word character (not a separator)
|
|
12
|
+
# - Single low quote (‚) as word character (not a separator)
|
|
13
|
+
# - Umlauts (ä, ö, ü, ß)
|
|
14
|
+
#
|
|
15
|
+
# The LanguageTool implementation adds two characters to the word characters:
|
|
16
|
+
# underscore (_) and single low quote (‚ - U+201A).
|
|
17
|
+
class GermanTokenizer < Base
|
|
18
|
+
# German-specific word separators (exclude underscore and single low quote)
|
|
19
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·]/
|
|
20
|
+
|
|
21
|
+
def tokenize(text)
|
|
22
|
+
return [] if text.nil? || text.strip.empty?
|
|
23
|
+
|
|
24
|
+
# Split on word boundaries
|
|
25
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
26
|
+
|
|
27
|
+
# Filter and normalize
|
|
28
|
+
raw_tokens
|
|
29
|
+
.map { |token| normalize(token) }
|
|
30
|
+
.reject { |token| skip_token?(token) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
protected
|
|
34
|
+
|
|
35
|
+
def word_separators
|
|
36
|
+
WORD_SEPARATORS
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Greek-script text.
|
|
7
|
+
#
|
|
8
|
+
# Splits on the same punctuation/whitespace separators as the
|
|
9
|
+
# Latin tokenizer, but keeps tokens that contain Greek letters
|
|
10
|
+
# (the Latin tokenizer's letter checks would drop a pure Greek
|
|
11
|
+
# word). Greek punctuation keeps its separators; the Greek
|
|
12
|
+
# question mark is the semicolon character and is treated as
|
|
13
|
+
# punctuation, matching the physical Greek keyboard where it is
|
|
14
|
+
# the shift of the question-mark key.
|
|
15
|
+
class GreekTokenizer < Base
|
|
16
|
+
# Separators between Greek words. Mirrors LatinTokenizer's
|
|
17
|
+
# separator set; Greek text needs no contraction handling.
|
|
18
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
|
|
19
|
+
|
|
20
|
+
# Tokenize text into Greek words.
|
|
21
|
+
#
|
|
22
|
+
# @param text [String] Text to tokenize
|
|
23
|
+
# @return [Array<String>] Array of tokens
|
|
24
|
+
def tokenize(text)
|
|
25
|
+
return [] if text.nil? || text.strip.empty?
|
|
26
|
+
|
|
27
|
+
text.split(WORD_SEPARATORS)
|
|
28
|
+
.map(&:strip)
|
|
29
|
+
.reject { |token| skip_token?(token) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Get word boundary regex.
|
|
33
|
+
#
|
|
34
|
+
# @return [Regexp] Word boundary regex
|
|
35
|
+
def word_boundary_regex
|
|
36
|
+
/\p{Greek}/
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Spell-check word characters (plan 91): Greek letters plus the
|
|
40
|
+
# apostrophe, which is not a Greek separator so elisions such
|
|
41
|
+
# as απ’ keep it in-word.
|
|
42
|
+
#
|
|
43
|
+
# @return [Regexp] Regex matching a single word character
|
|
44
|
+
def spellcheck_word_regex
|
|
45
|
+
/[\p{Greek}']/
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Check if token should be skipped.
|
|
49
|
+
#
|
|
50
|
+
# Skips tokens that contain no Greek letters; Base's rules
|
|
51
|
+
# (empty, pure numbers) also apply.
|
|
52
|
+
#
|
|
53
|
+
# @param token [String] Token to check
|
|
54
|
+
# @return [Boolean] True if should skip
|
|
55
|
+
def skip_token?(token)
|
|
56
|
+
return true if super
|
|
57
|
+
|
|
58
|
+
!token.match?(/\p{Greek}/)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
protected
|
|
62
|
+
|
|
63
|
+
# Get word characters.
|
|
64
|
+
#
|
|
65
|
+
# @return [String] Character class of Greek letters
|
|
66
|
+
def word_chars
|
|
67
|
+
"\\p{Greek}"
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Korean (plan 108).
|
|
7
|
+
#
|
|
8
|
+
# Korean text is space-delimited: an eojeol is a run of Hangul
|
|
9
|
+
# characters between spaces or punctuation, and every character
|
|
10
|
+
# inside the run belongs to the word — precomposed syllables
|
|
11
|
+
# (U+AC00-D7A3) as typed, plus any bare jamo (combining jamo
|
|
12
|
+
# U+1100-11FF, compatibility jamo U+3130-318F) that escaped IME
|
|
13
|
+
# composition and attaches to the preceding block. The staged ko
|
|
14
|
+
# dictionary stores words jamo-decomposed behind an ICONV table,
|
|
15
|
+
# so both surface forms must stay word characters for the
|
|
16
|
+
# spell-check extraction (plan 91) to hand the engine whole
|
|
17
|
+
# eojeol.
|
|
18
|
+
#
|
|
19
|
+
# Word segmentation (morpheme splitting inside an eojeol) is out
|
|
20
|
+
# of scope — like the pre-plan-108 basic tier, checking is
|
|
21
|
+
# eojeol-granular, matching how the staged dictionary is keyed.
|
|
22
|
+
class HangulTokenizer < Base
|
|
23
|
+
# Precomposed syllables plus the two jamo blocks (regex source
|
|
24
|
+
# text, like CyrillicTokenizer's word_chars).
|
|
25
|
+
HANGUL_WORD_CHARS = "\\uAC00-\\uD7A3\\u1100-\\u11FF\\u3130-\\u318F"
|
|
26
|
+
|
|
27
|
+
# One eojeol (a run of word characters).
|
|
28
|
+
WORD_RUN = /[#{HANGUL_WORD_CHARS}]+/
|
|
29
|
+
|
|
30
|
+
# One word character.
|
|
31
|
+
WORD_CHAR = /[#{HANGUL_WORD_CHARS}]/
|
|
32
|
+
|
|
33
|
+
# Tokenize text into eojeol.
|
|
34
|
+
#
|
|
35
|
+
# @param text [String] Text to tokenize
|
|
36
|
+
# @return [Array<String>] Array of eojeol tokens
|
|
37
|
+
def tokenize(text)
|
|
38
|
+
return [] if text.nil? || text.empty?
|
|
39
|
+
|
|
40
|
+
text.scan(WORD_RUN).reject { |token| skip_token?(token) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Get word boundary regex.
|
|
44
|
+
#
|
|
45
|
+
# @return [Regexp] Word boundary regex
|
|
46
|
+
def word_boundary_regex
|
|
47
|
+
WORD_CHAR
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Spell-check word characters (plan 91): the Hangul set —
|
|
51
|
+
# syllable blocks and jamo, no apostrophe (Korean has no
|
|
52
|
+
# in-word apostrophe) and no digits.
|
|
53
|
+
#
|
|
54
|
+
# @return [Regexp] Regex matching a single word character
|
|
55
|
+
def spellcheck_word_regex
|
|
56
|
+
WORD_CHAR
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
protected
|
|
60
|
+
|
|
61
|
+
# Get word characters.
|
|
62
|
+
#
|
|
63
|
+
# @return [String] Character class source
|
|
64
|
+
def word_chars
|
|
65
|
+
HANGUL_WORD_CHARS
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Japanese text.
|
|
7
|
+
#
|
|
8
|
+
# Uses Suika gem for morphological analysis. Suika is a soft runtime
|
|
9
|
+
# dependency — see {Kotoshu::Language::Suika} for load status and
|
|
10
|
+
# {Kotoshu::SuikaUnavailable} for the error raised when Japanese
|
|
11
|
+
# tokenization is requested without it.
|
|
12
|
+
#
|
|
13
|
+
# @see https://github.com/yoshoku/suika
|
|
14
|
+
class JapaneseTokenizer < Base
|
|
15
|
+
# Japanese word separators - keep it simple since Suika handles tokenization
|
|
16
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*·]/
|
|
17
|
+
|
|
18
|
+
def tokenize(text)
|
|
19
|
+
return [] if text.nil? || text.strip.empty?
|
|
20
|
+
|
|
21
|
+
# Suika::Tagger is process-wide memoized in Language::Suika;
|
|
22
|
+
# raises SuikaUnavailable when the gem is missing.
|
|
23
|
+
tagger = Kotoshu::Language::Suika.tagger
|
|
24
|
+
|
|
25
|
+
# Suika.parse returns an array of "surface\tfeatures" strings
|
|
26
|
+
tokens = []
|
|
27
|
+
parsed = tagger.parse(text)
|
|
28
|
+
|
|
29
|
+
parsed.each do |token|
|
|
30
|
+
# Suika returns: "すもも 名詞,一般,*,*,*,*,すもも,スモモ,スモモ"
|
|
31
|
+
# The surface form is tab-separated from the POS features
|
|
32
|
+
surface = token.split("\t").first
|
|
33
|
+
tokens << surface if surface && !surface.strip.empty?
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
tokens
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
protected
|
|
40
|
+
|
|
41
|
+
# Detect if text contains Japanese script.
|
|
42
|
+
#
|
|
43
|
+
# @param text [String] Text to check
|
|
44
|
+
# @return [Boolean] True if Japanese
|
|
45
|
+
def japanese?(text)
|
|
46
|
+
text.match?(/[-ゟ゠-ヿ]/) # Hiragana or Katakana
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def word_separators
|
|
50
|
+
WORD_SEPARATORS
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Language
|
|
5
|
+
module Tokenizer
|
|
6
|
+
# Tokenizer for Latin-script languages.
|
|
7
|
+
#
|
|
8
|
+
# Base tokenizer for English, French, German, Spanish, Portuguese,
|
|
9
|
+
# and other European languages using Latin script.
|
|
10
|
+
#
|
|
11
|
+
# Handles:
|
|
12
|
+
# - Standard word boundaries (whitespace, punctuation)
|
|
13
|
+
# - Apostrophes within words (contractions, elisions)
|
|
14
|
+
# - Hyphenated words
|
|
15
|
+
# - Numbers with units
|
|
16
|
+
#
|
|
17
|
+
# Subclasses can override for language-specific handling.
|
|
18
|
+
class LatinTokenizer < Base
|
|
19
|
+
# Latin word characters including accented characters
|
|
20
|
+
WORD_CHARS = "a-zA-Zà-ÿ0-9'"
|
|
21
|
+
|
|
22
|
+
# Punctuation that separates words
|
|
23
|
+
WORD_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-=_]/
|
|
24
|
+
|
|
25
|
+
# Contractions that should stay together
|
|
26
|
+
CONTRACTIONS = %w[
|
|
27
|
+
I'm I'd I've I'll you're you'd you've you'll he's he'd he'll
|
|
28
|
+
she's she'd she'll it's it'd we're we'd we've we'll they're
|
|
29
|
+
they'd they've they'll that's that'd that'll who's who'd who'll
|
|
30
|
+
what's what'd what'll where's where'd when's when'd why's why'd
|
|
31
|
+
how's how'd can't won't don't shouldn't couldn't wouldn't didn't
|
|
32
|
+
isn't aren't wasn't weren't hasn't haven't hadn't doesn't do
|
|
33
|
+
doesn't didn't mightn't mustn't shan't shouldn't wouldn't
|
|
34
|
+
].freeze
|
|
35
|
+
|
|
36
|
+
# Tokenize text into words.
|
|
37
|
+
#
|
|
38
|
+
# @param text [String] Text to tokenize
|
|
39
|
+
# @return [Array<String>] Array of tokens
|
|
40
|
+
def tokenize(text)
|
|
41
|
+
return [] if text.nil? || text.strip.empty?
|
|
42
|
+
|
|
43
|
+
# Split on word boundaries
|
|
44
|
+
raw_tokens = text.split(WORD_SEPARATORS)
|
|
45
|
+
|
|
46
|
+
# Filter and normalize
|
|
47
|
+
raw_tokens
|
|
48
|
+
.map { |token| normalize(token) }
|
|
49
|
+
.reject { |token| skip_token?(token) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Get word boundary regex.
|
|
53
|
+
#
|
|
54
|
+
# @return [Regexp] Word boundary regex
|
|
55
|
+
def word_boundary_regex
|
|
56
|
+
/[#{WORD_CHARS}]/o
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Spell-check word characters (plan 91): Latin-script letters
|
|
60
|
+
# plus the apostrophe, mirroring the Greek/Cyrillic overrides
|
|
61
|
+
# (\p{Latin} covers the accented letters WORD_CHARS spells out
|
|
62
|
+
# by range, including capitals such as Å and Ä). Digits are
|
|
63
|
+
# excluded so number handling is unchanged from the ASCII-era
|
|
64
|
+
# extraction.
|
|
65
|
+
#
|
|
66
|
+
# @return [Regexp] Regex matching a single word character
|
|
67
|
+
def spellcheck_word_regex
|
|
68
|
+
/[\p{Latin}']/
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Normalize token.
|
|
72
|
+
#
|
|
73
|
+
# Subclasses can override for language-specific normalization.
|
|
74
|
+
#
|
|
75
|
+
# @param token [String] Token to normalize
|
|
76
|
+
# @return [String] Normalized token
|
|
77
|
+
def normalize(token)
|
|
78
|
+
token.strip
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Check if token should be skipped.
|
|
82
|
+
#
|
|
83
|
+
# @param token [String] Token to check
|
|
84
|
+
# @return [Boolean] True if should skip
|
|
85
|
+
def skip_token?(token)
|
|
86
|
+
return true if super
|
|
87
|
+
|
|
88
|
+
# Skip pure numbers
|
|
89
|
+
return true if token.match?(/^\d+$/)
|
|
90
|
+
|
|
91
|
+
# Skip single characters (unless a word)
|
|
92
|
+
return true if token.length == 1 && token.match?(/[^a-zA-Zà-ÿ]/)
|
|
93
|
+
|
|
94
|
+
# Skip empty tokens
|
|
95
|
+
return true if token.empty?
|
|
96
|
+
|
|
97
|
+
# Skip tokens with no letters
|
|
98
|
+
return true unless token.match?(/[a-zA-Zà-ÿ]/)
|
|
99
|
+
|
|
100
|
+
false
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
protected
|
|
104
|
+
|
|
105
|
+
# Get word characters.
|
|
106
|
+
#
|
|
107
|
+
# @return [String] Character class
|
|
108
|
+
def word_chars
|
|
109
|
+
WORD_CHARS
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Handle contractions to keep them together.
|
|
113
|
+
#
|
|
114
|
+
# @param text [String] Input text
|
|
115
|
+
# @return [String] Text with protected contractions
|
|
116
|
+
def handle_contractions(text)
|
|
117
|
+
result = text.dup
|
|
118
|
+
|
|
119
|
+
# Protect common contractions
|
|
120
|
+
CONTRACTIONS.each do |contraction|
|
|
121
|
+
# Use word boundaries to avoid partial matches
|
|
122
|
+
result = result.gsub(/\b#{Regexp.escape(contraction)}\b/, contraction.gsub("'", "\uFEFF"))
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
result
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Extract next token with position.
|
|
129
|
+
#
|
|
130
|
+
# Override to handle apostrophes within words.
|
|
131
|
+
#
|
|
132
|
+
# @param text [String] Full text
|
|
133
|
+
# @param position [Integer] Current position
|
|
134
|
+
# @return [String, nil] Next token or nil
|
|
135
|
+
def extract_next_token(text, position)
|
|
136
|
+
remaining = text[position..]
|
|
137
|
+
|
|
138
|
+
# Check for contraction first
|
|
139
|
+
CONTRACTIONS.each do |contraction|
|
|
140
|
+
if remaining.start_with?(contraction) &&
|
|
141
|
+
remaining[contraction.length]&.match?(/\s|[^a-zA-Zà-ÿ]/)
|
|
142
|
+
return contraction
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Extract word with potential apostrophe
|
|
147
|
+
match = remaining.match(/^([#{WORD_CHARS}]+(?:'[#{WORD_CHARS}]+)?)/o)
|
|
148
|
+
match ? match[1] : nil
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|