kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Japanese language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Supports ja-JP with full CJK script support.
|
|
8
|
+
#
|
|
9
|
+
# Uses morphological analysis via Suika gem for tokenization and POS tagging.
|
|
10
|
+
# Japanese spell checking uses dictionary lookup with CJK character support.
|
|
11
|
+
class Japanese < Language::Base
|
|
12
|
+
# Japanese spell checker using dictionary lookup.
|
|
13
|
+
#
|
|
14
|
+
# Japanese uses morphological analysis rather than traditional Hunspell
|
|
15
|
+
# dictionaries. Spell checking is done through dictionary lookup of segmented
|
|
16
|
+
# words from the morphological analyzer.
|
|
17
|
+
class SpellChecker < Components::SpellChecker
|
|
18
|
+
attr_reader :dic_path, :script
|
|
19
|
+
|
|
20
|
+
def initialize(dic_path:, script: :cjk)
|
|
21
|
+
@dic_path = dic_path
|
|
22
|
+
@script = script
|
|
23
|
+
# Japanese dictionaries are typically in custom formats
|
|
24
|
+
# Load dictionary into memory for fast lookup
|
|
25
|
+
@dictionary = load_dictionary(dic_path)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def check(word)
|
|
29
|
+
return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
|
|
30
|
+
|
|
31
|
+
# Check if word exists in dictionary
|
|
32
|
+
found = @dictionary.include?(word)
|
|
33
|
+
|
|
34
|
+
if found
|
|
35
|
+
{ found: true, stem: word, flags: [] }
|
|
36
|
+
else
|
|
37
|
+
# For CJK text, we might want to check if it contains valid characters
|
|
38
|
+
# but not actual word validation
|
|
39
|
+
{ found: false, stem: nil, flags: [] }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def suggest(word, max_suggestions: 10)
|
|
44
|
+
return [] if word.nil? || word.empty?
|
|
45
|
+
return [] if @dictionary.include?(word)
|
|
46
|
+
|
|
47
|
+
# Generate suggestions based on common Japanese errors
|
|
48
|
+
generate_suggestions(word, max_suggestions).take(max_suggestions)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def correct?(word)
|
|
52
|
+
check(word)[:found]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def load_dictionary(path)
|
|
58
|
+
# Simple in-memory dictionary for Japanese words
|
|
59
|
+
# In production, this would use a proper CJK dictionary
|
|
60
|
+
@dictionary = Set.new
|
|
61
|
+
if File.exist?(path)
|
|
62
|
+
File.readlines(path, encoding: 'UTF-8').each do |line|
|
|
63
|
+
@dictionary.add(line.strip)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
@dictionary
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def generate_suggestions(word, max_suggestions)
|
|
70
|
+
variations = []
|
|
71
|
+
|
|
72
|
+
# Japanese character substitutions (common errors)
|
|
73
|
+
japanese_substitutions = {
|
|
74
|
+
'あ' => %w[ああ],
|
|
75
|
+
'い' => %w[いい],
|
|
76
|
+
'う' => %w[うう],
|
|
77
|
+
'え' => %w[ええ],
|
|
78
|
+
'お' => %w[おお],
|
|
79
|
+
'か' => %w[かが],
|
|
80
|
+
'き' => %w[きぎ],
|
|
81
|
+
'く' => %w[くぐ],
|
|
82
|
+
'け' => %w[けげ],
|
|
83
|
+
'こ' => %w[こご],
|
|
84
|
+
'さ' => %w[さざ],
|
|
85
|
+
'し' => %w[しじ],
|
|
86
|
+
'す' => %w[すず],
|
|
87
|
+
'せ' => %w[せぜ],
|
|
88
|
+
'そ' => %w[そぞ],
|
|
89
|
+
'た' => %w[ただ],
|
|
90
|
+
'ち' => %w[ちぢ],
|
|
91
|
+
'つ' => %w[つづ],
|
|
92
|
+
'て' => %w[てで],
|
|
93
|
+
'と' => %w[とど],
|
|
94
|
+
'は' => %w[はば],
|
|
95
|
+
'ひ' => %w[ひび],
|
|
96
|
+
'ふ' => %w[ふぶ],
|
|
97
|
+
'へ' => %w[へべ],
|
|
98
|
+
'ほ' => %w[ほぼ],
|
|
99
|
+
'ま' => %w[まま],
|
|
100
|
+
'み' => %w[みみ],
|
|
101
|
+
'む' => %w[むむ],
|
|
102
|
+
'め' => %w[めめ],
|
|
103
|
+
'も' => %w[もも],
|
|
104
|
+
'や' => %w[やや],
|
|
105
|
+
'ゆ' => %w[ゆゆ],
|
|
106
|
+
'よ' => %w[よよ],
|
|
107
|
+
'ら' => %w[らら],
|
|
108
|
+
'り' => %w[りり],
|
|
109
|
+
'る' => %w[るる],
|
|
110
|
+
'れ' => %w[れれ],
|
|
111
|
+
'ろ' => %w[ろろ],
|
|
112
|
+
'わ' => %w[わわ],
|
|
113
|
+
'を' => %w[お],
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
word.chars.each_with_index do |char, i|
|
|
117
|
+
next unless japanese_substitutions.key?(char)
|
|
118
|
+
|
|
119
|
+
japanese_substitutions[char].each do |sub|
|
|
120
|
+
substituted = word.dup
|
|
121
|
+
substituted[i] = sub
|
|
122
|
+
variations << substituted if @dictionary.include?(substituted)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Suggest similar dictionary words
|
|
127
|
+
if word.length >= 2
|
|
128
|
+
@dictionary.each do |dict_word|
|
|
129
|
+
distance = levenshtein_distance(word, dict_word)
|
|
130
|
+
if distance <= 2 && distance > 0
|
|
131
|
+
variations << dict_word
|
|
132
|
+
end
|
|
133
|
+
break if variations.length >= max_suggestions * 2
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
variations.uniq.first(max_suggestions)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def levenshtein_distance(a, b)
|
|
141
|
+
return a.length if b.empty?
|
|
142
|
+
return b.length if a.empty?
|
|
143
|
+
|
|
144
|
+
matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
|
|
145
|
+
(1..b.length).each { |j| matrix[0][j] = j }
|
|
146
|
+
(1..a.length).each do |i|
|
|
147
|
+
(1..b.length).each do |j|
|
|
148
|
+
cost = a[i - 1] == b[j - 1] ? 0 : 1
|
|
149
|
+
matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
matrix[a.length][b.length]
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Japanese tokenizer with morphological analysis.
|
|
157
|
+
class Tokenizer < Language::Tokenizer::JapaneseTokenizer
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Japanese POS tagger using morphological analysis.
|
|
161
|
+
#
|
|
162
|
+
# Japanese POS tagging is integrated with tokenization via Suika gem,
|
|
163
|
+
# which provides both segmentation and part-of-speech information.
|
|
164
|
+
#
|
|
165
|
+
# Suika output format: surface<TAB>POS,subcat1,subcat2,subcat3,conj_type,conj_form,lemma,reading,pronunciation
|
|
166
|
+
# Example: "すもも\t名詞,一般,*,*,*,*,すもも,スモモ,スモモ"
|
|
167
|
+
#
|
|
168
|
+
# POS tags use universal English categories for common types, and ROMAJI
|
|
169
|
+
# (Latin script) identifiers based on Japanese terminology only for
|
|
170
|
+
# language-specific categories without universal equivalents.
|
|
171
|
+
class POSTagger < Components::PosTagger
|
|
172
|
+
# Japanese POS tag mappings from Suika to standard identifiers.
|
|
173
|
+
#
|
|
174
|
+
# Strategy: Use universal English POS tags (NOUN, VERB, etc.) with
|
|
175
|
+
# English suffixes for subcategories. All identifiers are ASCII.
|
|
176
|
+
#
|
|
177
|
+
# Main categories (field 0) - universal:
|
|
178
|
+
# - 名詞 → NOUN
|
|
179
|
+
# - 動詞 → VERB
|
|
180
|
+
# - 助詞 → PARTICLE
|
|
181
|
+
# - 助動詞 → AUX
|
|
182
|
+
#
|
|
183
|
+
# Noun subcategories (field 1):
|
|
184
|
+
# - NOUN_COMMON: 一般 - common nouns
|
|
185
|
+
# - NOUN_PROPER: 固有名詞 - proper nouns
|
|
186
|
+
# - NOUN_PROPER_GEOGRAPHIC: 固有名詞,地域 - proper noun, geographic
|
|
187
|
+
# - NOUN_SUFFIX: 接尾 - suffixes
|
|
188
|
+
# - NOUN_DEPENDENT: 非自立 - dependent nouns (cannot stand alone)
|
|
189
|
+
# - NOUN_SA_CONNECTION: サ変接続 - sa-variant connection nouns
|
|
190
|
+
#
|
|
191
|
+
# Particle subcategories (field 1):
|
|
192
|
+
# - PARTICLE_GRAMMAR: 格助詞 - grammar/case particles (が, を, に, etc.)
|
|
193
|
+
# - PARTICLE_BINDING: 係助詞 - binding particles (は, も, etc.)
|
|
194
|
+
# - PARTICLE_ADNOMINAL: 連体化 - adnominal particles (の)
|
|
195
|
+
#
|
|
196
|
+
# Verb subcategories (field 1):
|
|
197
|
+
# - VERB_INDEPENDENT: 自立 - independent verbs
|
|
198
|
+
FLAG_TO_POS = {
|
|
199
|
+
# Main categories - universal English
|
|
200
|
+
'名詞' => 'NOUN',
|
|
201
|
+
'動詞' => 'VERB',
|
|
202
|
+
'助詞' => 'PARTICLE',
|
|
203
|
+
'助動詞' => 'AUX',
|
|
204
|
+
|
|
205
|
+
# Noun subcategories
|
|
206
|
+
'名詞,一般' => 'NOUN_COMMON',
|
|
207
|
+
'名詞,固有名詞' => 'NOUN_PROPER',
|
|
208
|
+
'名詞,固有名詞,地域' => 'NOUN_PROPER_GEOGRAPHIC',
|
|
209
|
+
'名詞,接尾' => 'NOUN_SUFFIX',
|
|
210
|
+
'名詞,非自立' => 'NOUN_DEPENDENT',
|
|
211
|
+
'名詞,サ変接続' => 'NOUN_SA_CONNECTION',
|
|
212
|
+
|
|
213
|
+
# Particle subcategories
|
|
214
|
+
'助詞,格助詞' => 'PARTICLE_GRAMMAR',
|
|
215
|
+
'助詞,係助詞' => 'PARTICLE_BINDING',
|
|
216
|
+
'助詞,連体化' => 'PARTICLE_ADNOMINAL',
|
|
217
|
+
|
|
218
|
+
# Verb subcategories
|
|
219
|
+
'動詞,自立' => 'VERB_INDEPENDENT',
|
|
220
|
+
}.freeze
|
|
221
|
+
|
|
222
|
+
def initialize(dictionary_path: nil, flag_mapping: FLAG_TO_POS)
|
|
223
|
+
@dictionary_path = dictionary_path
|
|
224
|
+
@flag_mapping = flag_mapping
|
|
225
|
+
@suika_tagger = nil
|
|
226
|
+
@lookup_cache = {}
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def tag(tokens)
|
|
230
|
+
return [] if tokens.nil? || tokens.empty?
|
|
231
|
+
|
|
232
|
+
# Suika::Tagger is process-wide memoized in Language::Suika;
|
|
233
|
+
# raises SuikaUnavailable when the gem is missing.
|
|
234
|
+
@suika_tagger ||= Kotoshu::Language::Suika.tagger
|
|
235
|
+
|
|
236
|
+
tokens.map do |token|
|
|
237
|
+
word = token[:token]
|
|
238
|
+
if word.nil? || word.empty?
|
|
239
|
+
token.merge(pos_tag: nil, lemma: nil)
|
|
240
|
+
else
|
|
241
|
+
lookup_result = lookup_with_pos(word)
|
|
242
|
+
token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def flag_mapping
|
|
248
|
+
@flag_mapping
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def flag_mapping=(mapping)
|
|
252
|
+
@flag_mapping = mapping
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def clear_cache
|
|
256
|
+
@lookup_cache.clear
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
private
|
|
260
|
+
|
|
261
|
+
def lookup_with_pos(word)
|
|
262
|
+
return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
|
|
263
|
+
return @lookup_cache[word] if @lookup_cache.key?(word)
|
|
264
|
+
|
|
265
|
+
# Use Suika to parse and get POS information
|
|
266
|
+
parsed = @suika_tagger.parse(word)
|
|
267
|
+
|
|
268
|
+
# Suika returns tab-separated values: surface\tfeatures
|
|
269
|
+
# Features contain POS information
|
|
270
|
+
pos_tag = extract_pos_from_suika(parsed)
|
|
271
|
+
lemma = extract_lemma_from_suika(parsed)
|
|
272
|
+
|
|
273
|
+
cache_result = { pos_tag: pos_tag, lemma: lemma }
|
|
274
|
+
@lookup_cache[word] = cache_result
|
|
275
|
+
cache_result
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def extract_pos_from_suika(parsed)
|
|
279
|
+
return nil unless parsed && parsed.first
|
|
280
|
+
|
|
281
|
+
# Parse features from Suika output
|
|
282
|
+
# Format: surface<TAB>POS,sub1,sub2,sub3,conj_type,conj_form,lemma,reading,pronunciation
|
|
283
|
+
parts = parsed.first.split("\t")
|
|
284
|
+
return nil unless parts.length > 1
|
|
285
|
+
|
|
286
|
+
# Features are comma-separated
|
|
287
|
+
# Field 0: Surface form
|
|
288
|
+
# Field 1: Main POS category (e.g., 名詞, 動詞, 助詞)
|
|
289
|
+
# Field 2-6: POS subcategories and conjugation info
|
|
290
|
+
# Field 7: Lemma (dictionary form)
|
|
291
|
+
# Field 8: Reading (katakana)
|
|
292
|
+
# Field 9: Pronunciation (katakana)
|
|
293
|
+
features = parts[1].split(',')
|
|
294
|
+
|
|
295
|
+
# Build hierarchical POS paths from most specific to least specific
|
|
296
|
+
# e.g., ["名詞,固有名詞,地域", "名詞,固有名詞", "名詞"]
|
|
297
|
+
pos_paths = []
|
|
298
|
+
6.times do |i|
|
|
299
|
+
path = features[0..i].join(',')
|
|
300
|
+
pos_paths << path
|
|
301
|
+
end
|
|
302
|
+
# Reverse to check most specific first
|
|
303
|
+
pos_paths.reverse!
|
|
304
|
+
|
|
305
|
+
# Try to match from most specific to least specific
|
|
306
|
+
pos_paths.each do |pos_path|
|
|
307
|
+
if FLAG_TO_POS.key?(pos_path)
|
|
308
|
+
return FLAG_TO_POS[pos_path]
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
nil
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def extract_lemma_from_suika(parsed)
|
|
316
|
+
return nil unless parsed && parsed.first
|
|
317
|
+
|
|
318
|
+
parts = parsed.first.split("\t")
|
|
319
|
+
return nil unless parts.length > 1
|
|
320
|
+
|
|
321
|
+
# Extract lemma from Suika features
|
|
322
|
+
# Format is complex, so simplified version
|
|
323
|
+
parts[0] # Return surface form as lemma
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Japanese grammar rules module.
|
|
328
|
+
module GrammarRules
|
|
329
|
+
class Rule
|
|
330
|
+
attr_reader :id, :name, :description
|
|
331
|
+
|
|
332
|
+
def initialize(id, name, description)
|
|
333
|
+
@id = id
|
|
334
|
+
@name = name
|
|
335
|
+
@description = description
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def check(tokens)
|
|
339
|
+
raise NotImplementedError, "#{self.class} must implement #check"
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Rule: Particle usage (wa vs ga)
|
|
344
|
+
class ParticleRule < Rule
|
|
345
|
+
def initialize
|
|
346
|
+
super('JA_PARTICLE_USAGE', 'Particle Usage', 'Correct usage of topic marker は vs subject marker が.')
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def check(_tokens)
|
|
350
|
+
# Simplified implementation
|
|
351
|
+
[]
|
|
352
|
+
end
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Rule: Script mixing
|
|
356
|
+
class ScriptMixingRule < Rule
|
|
357
|
+
def initialize
|
|
358
|
+
super('JA_SCRIPT_MIXING', 'Script Mixing', 'Japanese text uses multiple scripts (Hiragana, Katakana, Kanji).')
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def check(tokens)
|
|
362
|
+
errors = []
|
|
363
|
+
tokens.each do |token|
|
|
364
|
+
word = token[:token]
|
|
365
|
+
next if word.nil? || word.empty?
|
|
366
|
+
|
|
367
|
+
# Check for script mixing inconsistencies
|
|
368
|
+
has_hiragana = word.match?(/[\u3040-\u309F]/)
|
|
369
|
+
has_katakana = word.match?(/[\u30A0-\u30FF]/)
|
|
370
|
+
has_kanji = word.match?(/[\u4E00-\u9FFF]/)
|
|
371
|
+
|
|
372
|
+
# Words typically shouldn't mix all three scripts
|
|
373
|
+
if has_hiragana && has_katakana && has_kanji
|
|
374
|
+
errors << {
|
|
375
|
+
rule_id: @id,
|
|
376
|
+
position: token[:position],
|
|
377
|
+
message: "Unusual script mixing in word '#{word}'",
|
|
378
|
+
suggestion: 'Review script usage',
|
|
379
|
+
context: word,
|
|
380
|
+
suggestions: ['Use consistent script']
|
|
381
|
+
}
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
errors
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
class RuleRegistry
|
|
389
|
+
class << self
|
|
390
|
+
def default_rules
|
|
391
|
+
[ParticleRule.new, ScriptMixingRule.new]
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def get_rule(id)
|
|
395
|
+
default_rules.find { |rule| rule.id == id }
|
|
396
|
+
end
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# Registration
|
|
402
|
+
register "ja"
|
|
403
|
+
register "ja-JP"
|
|
404
|
+
|
|
405
|
+
HUNSPELL_DICTIONARIES = {
|
|
406
|
+
'ja-JP' => {
|
|
407
|
+
# Japanese dictionaries are in custom formats
|
|
408
|
+
# Suika uses its own dictionary format
|
|
409
|
+
}
|
|
410
|
+
}.freeze
|
|
411
|
+
|
|
412
|
+
VARIANT_NAMES = {
|
|
413
|
+
'JP' => 'Japan'
|
|
414
|
+
}.freeze
|
|
415
|
+
|
|
416
|
+
def initialize(code: "ja", name: "Japanese", variant: nil)
|
|
417
|
+
variant ||= extract_region_code(code)
|
|
418
|
+
super
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def description
|
|
422
|
+
return name unless variant
|
|
423
|
+
|
|
424
|
+
variant_name = VARIANT_NAMES[variant] || variant
|
|
425
|
+
"#{name} (#{variant_name})"
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
def tokenizer
|
|
429
|
+
@tokenizer ||= Tokenizer.new
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
def normalizer
|
|
433
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def dictionary_class
|
|
437
|
+
Dictionary::UnixWords
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def default_dictionary_paths
|
|
441
|
+
["/usr/share/dict/words"]
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
def script_type
|
|
445
|
+
:cjk
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
def create_spell_checker
|
|
449
|
+
# Japanese uses custom dictionary, not Hunspell format
|
|
450
|
+
SpellChecker.new(
|
|
451
|
+
dic_path: default_dictionary_paths.first,
|
|
452
|
+
script: :cjk
|
|
453
|
+
)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def create_tokenizer
|
|
457
|
+
Tokenizer.new
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def create_pos_tagger
|
|
461
|
+
POSTagger.new(
|
|
462
|
+
dictionary_path: default_dictionary_paths.first,
|
|
463
|
+
flag_mapping: POSTagger::FLAG_TO_POS
|
|
464
|
+
)
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
private
|
|
468
|
+
|
|
469
|
+
def extract_region_code(code)
|
|
470
|
+
return nil unless code.include?("-")
|
|
471
|
+
|
|
472
|
+
code.split("-", 2).last.upcase
|
|
473
|
+
end
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Korean language implementation (plan 108).
|
|
6
|
+
#
|
|
7
|
+
# Registers +ko+ and +ko-KR+. The staged ko dictionary (LibreOffice
|
|
8
|
+
# ko_KR, jamo-decomposed behind an ICONV table of all 11,172
|
|
9
|
+
# syllables) serves checking through the standard Hunspell engine;
|
|
10
|
+
# this module contributes the language wiring the basic tier
|
|
11
|
+
# lacks: the eojeol-aware Hangul tokenizer for word extraction and
|
|
12
|
+
# the Dubeolsik (KS X 5002) keyboard grid for jamo-proximity
|
|
13
|
+
# suggestion ranking.
|
|
14
|
+
#
|
|
15
|
+
# @example
|
|
16
|
+
# lang = Kotoshu::Languages::Korean.new
|
|
17
|
+
# lang.script_type # => :hangul
|
|
18
|
+
# lang.rtl? # => false
|
|
19
|
+
class Korean < Language::Base
|
|
20
|
+
register "ko"
|
|
21
|
+
register "ko-KR"
|
|
22
|
+
|
|
23
|
+
def initialize(code: "ko", name: "Korean", variant: nil)
|
|
24
|
+
super
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def description
|
|
28
|
+
name
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def tokenizer
|
|
32
|
+
@tokenizer ||= Language::Tokenizer::HangulTokenizer.new
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def normalizer
|
|
36
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def dictionary_class
|
|
40
|
+
Dictionary::Custom
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def default_dictionary_paths
|
|
44
|
+
["/usr/share/dict/korean"]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def script_type
|
|
48
|
+
:hangul
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def rtl?
|
|
52
|
+
false
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Shared base for thin Latin-script language modules.
|
|
6
|
+
#
|
|
7
|
+
# The wave-1 modules (plan 84, Track B) are compositions, not
|
|
8
|
+
# engines: the shared Latin tokenizer, the base normalizer, and a
|
|
9
|
+
# registry entry. Language-specific care overrides one piece:
|
|
10
|
+
# Turkish swaps in the Turkish normalizer (dotless-i folding),
|
|
11
|
+
# Greek swaps in the Greek tokenizer and normalizer, Ukrainian the
|
|
12
|
+
# Cyrillic tokenizer.
|
|
13
|
+
#
|
|
14
|
+
# The existing ten modules (ar de en es fa fr he ja pt ru) are
|
|
15
|
+
# untouched and keep their own class bodies.
|
|
16
|
+
class LatinBase < Language::Base
|
|
17
|
+
# Latin tokenizer shared by every Latin-script module.
|
|
18
|
+
#
|
|
19
|
+
# @return [Language::Tokenizer::LatinTokenizer]
|
|
20
|
+
def tokenizer
|
|
21
|
+
@tokenizer ||= Language::Tokenizer::LatinTokenizer.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Base normalizer (whitespace, downcase).
|
|
25
|
+
#
|
|
26
|
+
# @return [Language::Normalizer::Base]
|
|
27
|
+
def normalizer
|
|
28
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Dictionary backend for this language.
|
|
32
|
+
#
|
|
33
|
+
# @return [Class]
|
|
34
|
+
def dictionary_class
|
|
35
|
+
Dictionary::UnixWords
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Script type.
|
|
39
|
+
#
|
|
40
|
+
# @return [Symbol] :latin
|
|
41
|
+
def script_type
|
|
42
|
+
:latin
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Create the tokenizer (memoized; tokenizers are stateless).
|
|
46
|
+
#
|
|
47
|
+
# @return [Language::Tokenizer::Base]
|
|
48
|
+
def create_tokenizer
|
|
49
|
+
tokenizer
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Lithuanian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. The Lithuanian letters ą č ę ė į š ų ū ž are
|
|
9
|
+
# dead-key / AltGr sequences over the US QWERTY grid
|
|
10
|
+
# (Keyboard::Layouts::Latin::Lithuanian), like the eval model.
|
|
11
|
+
class Lithuanian < LatinBase
|
|
12
|
+
register "lt"
|
|
13
|
+
register "lt-LT"
|
|
14
|
+
|
|
15
|
+
def initialize(code: "lt", name: "Lithuanian")
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def default_dictionary_paths
|
|
20
|
+
["/usr/share/dict/lithuanian"]
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Latvian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. The Latvian letters ā č ē ģ ī ķ ļ ņ š ū ž are
|
|
9
|
+
# dead-key / AltGr sequences over the US QWERTY grid
|
|
10
|
+
# (Keyboard::Layouts::Latin::Latvian), like the eval model.
|
|
11
|
+
class Latvian < LatinBase
|
|
12
|
+
register "lv"
|
|
13
|
+
register "lv-LV"
|
|
14
|
+
|
|
15
|
+
def initialize(code: "lv", name: "Latvian")
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def default_dictionary_paths
|
|
20
|
+
["/usr/share/dict/latvian"]
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Norwegian Bokmål (registry v1.3.0: models ship as `nb` from
|
|
6
|
+
# fastText's Bokmål-dominated cc.no vectors; the Language registry
|
|
7
|
+
# aliases `no` here). Thin composition like its Nordic siblings —
|
|
8
|
+
# shared Latin tokenizer, base normalizer, registry entry; å æ ø
|
|
9
|
+
# are real keys on the Nordic layout.
|
|
10
|
+
class NorwegianBokmal < LatinBase
|
|
11
|
+
register "nb"
|
|
12
|
+
register "nb-NO"
|
|
13
|
+
register "no"
|
|
14
|
+
register "no-NO"
|
|
15
|
+
|
|
16
|
+
def initialize(code: "nb", name: "Norwegian Bokmål")
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def default_dictionary_paths
|
|
21
|
+
["/usr/share/dict/norwegian"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|