kotoshu 1.0.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Nepali language implementation (plan 108).
|
|
6
|
+
#
|
|
7
|
+
# Registers +ne+ and +ne-NP+. The staged ne dictionary (LibreOffice
|
|
8
|
+
# ne_NP) serves checking through the standard Hunspell engine;
|
|
9
|
+
# this module contributes the language wiring the basic tier
|
|
10
|
+
# lacks: the Devanagari tokenizer that keeps matras and virama
|
|
11
|
+
# conjuncts attached to their base consonant during word
|
|
12
|
+
# extraction, and the Devanagari InScript grid (the Indian
|
|
13
|
+
# national standard layout, mirrored from the models repo eval
|
|
14
|
+
# harness) for typo-proximity suggestion ranking. The Nepali
|
|
15
|
+
# Traditional Romanized layout is more common on the ground, but
|
|
16
|
+
# it is not a documented standard the way InScript is — the same
|
|
17
|
+
# choice the eval harness made.
|
|
18
|
+
#
|
|
19
|
+
# @example
|
|
20
|
+
# lang = Kotoshu::Languages::Nepali.new
|
|
21
|
+
# lang.script_type # => :devanagari
|
|
22
|
+
# lang.rtl? # => false
|
|
23
|
+
class Nepali < Language::Base
|
|
24
|
+
register "ne"
|
|
25
|
+
register "ne-NP"
|
|
26
|
+
|
|
27
|
+
def initialize(code: "ne", name: "Nepali", variant: nil)
|
|
28
|
+
super
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def description
|
|
32
|
+
name
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def tokenizer
|
|
36
|
+
@tokenizer ||= Language::Tokenizer::DevanagariTokenizer.new
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def normalizer
|
|
40
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def dictionary_class
|
|
44
|
+
Dictionary::Custom
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def default_dictionary_paths
|
|
48
|
+
["/usr/share/dict/nepali"]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def script_type
|
|
52
|
+
:devanagari
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def rtl?
|
|
56
|
+
false
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Dutch language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Dutch loanword accents (é ï ö ü) are dead-key sequences on the US physical layout.
|
|
9
|
+
class Dutch < LatinBase
|
|
10
|
+
register "nl"
|
|
11
|
+
register "nl-NL"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "nl", name: "Dutch")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/dutch"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Norwegian Nynorsk (plan 107): the second Norwegian written
|
|
6
|
+
# standard, wired while basic support opened the staged manifest —
|
|
7
|
+
# dictionary and model are both staged (`nn` in the dictionaries
|
|
8
|
+
# repo, `nn` in the models registry), so Nynorsk users get the
|
|
9
|
+
# full-feature tier rather than the script fallback. Thin
|
|
10
|
+
# composition like its Bokmål sibling — shared Latin tokenizer,
|
|
11
|
+
# base normalizer, registry entry; å æ ø are real keys on the
|
|
12
|
+
# Nordic layout the nb module resolves to.
|
|
13
|
+
class NorwegianNynorsk < LatinBase
|
|
14
|
+
register "nn"
|
|
15
|
+
register "nn-NO"
|
|
16
|
+
|
|
17
|
+
def initialize(code: "nn", name: "Norwegian Nynorsk")
|
|
18
|
+
super
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def default_dictionary_paths
|
|
22
|
+
["/usr/share/dict/norwegian"]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Polish language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Polish diacritics (ą ć ę ł ń ó ś ź ż) are AltGr sequences on the programmers layout.
|
|
9
|
+
class Polish < LatinBase
|
|
10
|
+
register "pl"
|
|
11
|
+
register "pl-PL"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "pl", name: "Polish")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/polish"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Portuguese language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Supports multiple dialects: pt-BR, pt-PT, pt-AO, pt-MZ, pt-GW, pt-CV
|
|
8
|
+
#
|
|
9
|
+
# Full Hunspell integration with spell checking, POS tagging, and grammar rules
|
|
10
|
+
# specifically handling Portuguese accents and Brazilian vs European differences.
|
|
11
|
+
class Portuguese < Language::Base
|
|
12
|
+
# Portuguese spell checker with Hunspell integration.
|
|
13
|
+
class SpellChecker < Components::SpellChecker
|
|
14
|
+
attr_reader :aff_path, :dic_path, :script
|
|
15
|
+
|
|
16
|
+
# Portuguese-specific character substitutions
|
|
17
|
+
PORTUGUESE_SUBSTITUTIONS = {
|
|
18
|
+
'á' => %w[a],
|
|
19
|
+
'â' => %w[a],
|
|
20
|
+
'ã' => %w[a],
|
|
21
|
+
'à' => %w[a],
|
|
22
|
+
'é' => %w[e],
|
|
23
|
+
'ê' => %w[e],
|
|
24
|
+
'í' => %w[i],
|
|
25
|
+
'ó' => %w[o],
|
|
26
|
+
'ô' => %w[o],
|
|
27
|
+
'õ' => %w[o],
|
|
28
|
+
'ú' => %w[u],
|
|
29
|
+
'ü' => %w[u],
|
|
30
|
+
'ç' => %w[c],
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
33
|
+
def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
|
|
34
|
+
@aff_path = aff_path
|
|
35
|
+
@dic_path = dic_path
|
|
36
|
+
@script = script
|
|
37
|
+
@encoding = encoding
|
|
38
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def check(word)
|
|
42
|
+
return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
|
|
43
|
+
|
|
44
|
+
first_form = @lookuper.good_forms(word).first
|
|
45
|
+
if first_form
|
|
46
|
+
{ found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
|
|
47
|
+
else
|
|
48
|
+
{ found: false, stem: nil, flags: [] }
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def suggest(word, max_suggestions: 10)
|
|
53
|
+
return [] if word.nil? || word.empty?
|
|
54
|
+
|
|
55
|
+
first_form = @lookuper.good_forms(word).first
|
|
56
|
+
return [] if first_form
|
|
57
|
+
|
|
58
|
+
generate_suggestions(word, max_suggestions).take(max_suggestions)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def correct?(word)
|
|
62
|
+
check(word)[:found]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def lookuper
|
|
66
|
+
@lookuper
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def calculate_distance(a, b)
|
|
72
|
+
return a.length if b.empty?
|
|
73
|
+
return b.length if a.empty?
|
|
74
|
+
|
|
75
|
+
matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
|
|
76
|
+
(1..b.length).each { |j| matrix[0][j] = j }
|
|
77
|
+
(1..a.length).each do |i|
|
|
78
|
+
(1..b.length).each do |j|
|
|
79
|
+
cost = a[i - 1] == b[j - 1] ? 0 : 1
|
|
80
|
+
matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
matrix[a.length][b.length]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def calculate_score(original, suggestion, rank)
|
|
87
|
+
distance = calculate_distance(original, suggestion)
|
|
88
|
+
max_len = [original.length, suggestion.length].max
|
|
89
|
+
distance_score = 1.0 - (distance.to_f / max_len)
|
|
90
|
+
rank_penalty = rank * 0.05
|
|
91
|
+
[distance_score - rank_penalty, 0.0].max
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def generate_suggestions(word, _max_suggestions)
|
|
95
|
+
variations = []
|
|
96
|
+
|
|
97
|
+
# Missing accents
|
|
98
|
+
word.downcase.chars.each_with_index do |char, i|
|
|
99
|
+
PORTUGUESE_SUBSTITUTIONS.each do |accented, variants|
|
|
100
|
+
variants.each do |variant|
|
|
101
|
+
if char == variant
|
|
102
|
+
accented_word = word.dup
|
|
103
|
+
accented_word[i] = accented
|
|
104
|
+
variations << accented_word if @lookuper.good_forms(accented_word).first
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Common substitutions
|
|
111
|
+
word.chars.each_with_index do |char, i|
|
|
112
|
+
next unless PORTUGUESE_SUBSTITUTIONS.key?(char.downcase)
|
|
113
|
+
|
|
114
|
+
PORTUGUESE_SUBSTITUTIONS[char.downcase].each do |sub|
|
|
115
|
+
substituted = word.dup
|
|
116
|
+
substituted[i] = sub
|
|
117
|
+
variations << substituted if @lookuper.good_forms(substituted).first
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Doubled and deleted letters
|
|
122
|
+
word.chars.each_with_index do |char, i|
|
|
123
|
+
next if i == 0
|
|
124
|
+
|
|
125
|
+
doubled = word.dup
|
|
126
|
+
doubled.insert(i, char)
|
|
127
|
+
variations << doubled if @lookuper.good_forms(doubled).first
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
(0...word.length).each do |i|
|
|
131
|
+
deleted = word.dup
|
|
132
|
+
deleted.slice!(i)
|
|
133
|
+
next if deleted.empty?
|
|
134
|
+
|
|
135
|
+
variations << deleted if @lookuper.good_forms(deleted).first
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
variations.uniq!
|
|
139
|
+
variations.map do |suggestion|
|
|
140
|
+
{ word: suggestion, distance: calculate_distance(word, suggestion),
|
|
141
|
+
score: calculate_score(word, suggestion, 0) }
|
|
142
|
+
end.sort_by { |s| s[:distance] }
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Portuguese tokenizer with number and date handling.
|
|
147
|
+
class Tokenizer < Language::Tokenizer::PortugueseTokenizer
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Portuguese POS tagger.
|
|
151
|
+
class POSTagger < Components::PosTagger
|
|
152
|
+
FLAG_TO_POS = {
|
|
153
|
+
'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
|
|
154
|
+
'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
|
|
155
|
+
'VBP' => 'VERB', 'VBZ' => 'VERB',
|
|
156
|
+
'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
|
|
157
|
+
'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
|
|
158
|
+
'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
|
|
159
|
+
'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
|
|
160
|
+
'WP' => 'PRON', 'WP$' => 'PRON_POSS',
|
|
161
|
+
'I' => 'PREP', 'IN' => 'PREP',
|
|
162
|
+
'C' => 'CONJ', 'CC' => 'CONJ',
|
|
163
|
+
'U' => 'PART', 'RP' => 'PART',
|
|
164
|
+
'INTJ' => 'INTJ', 'UH' => 'INTJ',
|
|
165
|
+
'CD' => 'NUM',
|
|
166
|
+
'FW' => 'X',
|
|
167
|
+
'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
|
|
168
|
+
'?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
|
|
169
|
+
}.freeze
|
|
170
|
+
|
|
171
|
+
attr_reader :aff_path, :dic_path, :script
|
|
172
|
+
|
|
173
|
+
def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
|
|
174
|
+
@aff_path = aff_path
|
|
175
|
+
@dic_path = dic_path
|
|
176
|
+
@script = script
|
|
177
|
+
@encoding = encoding
|
|
178
|
+
@flag_mapping = flag_mapping
|
|
179
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
180
|
+
@lookup_cache = {}
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def tag(tokens)
|
|
184
|
+
return [] if tokens.nil? || tokens.empty?
|
|
185
|
+
|
|
186
|
+
tokens.map do |token|
|
|
187
|
+
word = token[:token]
|
|
188
|
+
if word.nil? || word.empty?
|
|
189
|
+
token.merge(pos_tag: nil, lemma: nil)
|
|
190
|
+
else
|
|
191
|
+
lookup_result = lookup_with_pos(word)
|
|
192
|
+
token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def flag_mapping
|
|
198
|
+
@flag_mapping
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def flag_mapping=(mapping)
|
|
202
|
+
@flag_mapping = mapping
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def clear_cache
|
|
206
|
+
@lookup_cache.clear
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def cache_size
|
|
210
|
+
@lookup_cache.size
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
private
|
|
214
|
+
|
|
215
|
+
def lookup_with_pos(word)
|
|
216
|
+
return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
|
|
217
|
+
return @lookup_cache[word] if @lookup_cache.key?(word)
|
|
218
|
+
|
|
219
|
+
first_form = @lookuper.good_forms(word).first
|
|
220
|
+
pos_tag = derive_pos_tag(first_form)
|
|
221
|
+
cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
|
|
222
|
+
@lookup_cache[word] = cache_result
|
|
223
|
+
cache_result
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def derive_pos_tag(result)
|
|
227
|
+
return nil unless result
|
|
228
|
+
|
|
229
|
+
flags = result.flags&.to_a || []
|
|
230
|
+
return guess_pos_from_affix(result) if flags.empty?
|
|
231
|
+
|
|
232
|
+
flags.each do |flag|
|
|
233
|
+
pos_tag = flag_to_pos(flag)
|
|
234
|
+
return pos_tag if pos_tag
|
|
235
|
+
end
|
|
236
|
+
guess_pos_from_affix(result)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def flag_to_pos(flag)
|
|
240
|
+
return @flag_mapping[flag] if @flag_mapping.key?(flag)
|
|
241
|
+
|
|
242
|
+
first_char = flag[0]
|
|
243
|
+
@flag_mapping[first_char]
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def guess_pos_from_affix(result)
|
|
247
|
+
suffix = result.suffix
|
|
248
|
+
return guess_pos_from_suffix(suffix) if suffix
|
|
249
|
+
|
|
250
|
+
nil
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def guess_pos_from_suffix(suffix)
|
|
254
|
+
return 'VERB' if suffix.match?(/^(ar|er|ir|ando|endo|indo|ado|ido)$/)
|
|
255
|
+
return 'ADV' if suffix.end_with?('mente')
|
|
256
|
+
return 'NOUN' if suffix.match?(/^(ção|são|mento|dade|eza|ismo|ista|or|nte)$/)
|
|
257
|
+
return 'ADJ' if suffix.match?(/%(oso|ável|ível|ico|ica|ante)$/)
|
|
258
|
+
|
|
259
|
+
nil
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# Portuguese grammar rules module.
|
|
264
|
+
module GrammarRules
|
|
265
|
+
class Rule
|
|
266
|
+
attr_reader :id, :name, :description
|
|
267
|
+
|
|
268
|
+
def initialize(id, name, description)
|
|
269
|
+
@id = id
|
|
270
|
+
@name = name
|
|
271
|
+
@description = description
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def check(tokens)
|
|
275
|
+
raise NotImplementedError, "#{self.class} must implement #check"
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Rule: Personal infinitive agreement
|
|
280
|
+
class PersonalInfinitiveRule < Rule
|
|
281
|
+
def initialize
|
|
282
|
+
super('PT_PERSONAL_INFINITIVE', 'Personal Infinitive', 'Personal infinitive must agree with the subject.')
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def check(_tokens)
|
|
286
|
+
# Simplified implementation
|
|
287
|
+
[]
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Rule: Crase (à vs a)
|
|
292
|
+
class CraseRule < Rule
|
|
293
|
+
def initialize
|
|
294
|
+
super('PT_CRASE', 'Crase Usage', 'Use crase (à) before feminine nouns indicating place/time.')
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def check(tokens)
|
|
298
|
+
errors = []
|
|
299
|
+
tokens.each_cons(2) do |prev_token, current_token|
|
|
300
|
+
prev_word = prev_token[:token]&.downcase
|
|
301
|
+
next unless %w[a ema].include?(prev_word)
|
|
302
|
+
|
|
303
|
+
# Check if next word starts with 'a' sound and is feminine
|
|
304
|
+
next_word = current_token[:token]
|
|
305
|
+
next if next_word.nil? || next_word.empty?
|
|
306
|
+
|
|
307
|
+
if next_word&.match?(/^[aáãâä]/i)
|
|
308
|
+
# Suggest using crase
|
|
309
|
+
errors << {
|
|
310
|
+
rule_id: @id,
|
|
311
|
+
position: prev_token[:position],
|
|
312
|
+
message: "Possible crase usage needed: '#{prev_word}' -> 'à'",
|
|
313
|
+
suggestion: 'à',
|
|
314
|
+
context: "#{prev_word} #{next_word}",
|
|
315
|
+
suggestions: ['à']
|
|
316
|
+
}
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
errors
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
class RuleRegistry
|
|
324
|
+
class << self
|
|
325
|
+
def default_rules
|
|
326
|
+
[PersonalInfinitiveRule.new, CraseRule.new]
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
def get_rule(id)
|
|
330
|
+
default_rules.find { |rule| rule.id == id }
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# Registration
|
|
337
|
+
register "pt"
|
|
338
|
+
register "pt-BR"
|
|
339
|
+
register "pt-PT"
|
|
340
|
+
register "pt-AO"
|
|
341
|
+
register "pt-MZ"
|
|
342
|
+
register "pt-GW"
|
|
343
|
+
register "pt-CV"
|
|
344
|
+
|
|
345
|
+
HUNSPELL_DICTIONARIES = {
|
|
346
|
+
'pt-BR' => {
|
|
347
|
+
aff: 'spec/integrational/fixtures/pt_BR.aff',
|
|
348
|
+
dic: 'spec/integrational/fixtures/pt_BR.dic'
|
|
349
|
+
},
|
|
350
|
+
'pt-PT' => {
|
|
351
|
+
aff: 'spec/integrational/fixtures/pt_PT.aff',
|
|
352
|
+
dic: 'spec/integrational/fixtures/pt_PT.dic'
|
|
353
|
+
}
|
|
354
|
+
}.freeze
|
|
355
|
+
|
|
356
|
+
VARIANT_NAMES = {
|
|
357
|
+
'BR' => 'Brazilian',
|
|
358
|
+
'PT' => 'European',
|
|
359
|
+
'AO' => 'Angolan',
|
|
360
|
+
'MZ' => 'Mozambican',
|
|
361
|
+
'GW' => 'Guinea-Bissau',
|
|
362
|
+
'CV' => 'Cape Verdean'
|
|
363
|
+
}.freeze
|
|
364
|
+
|
|
365
|
+
def initialize(code: "pt", name: "Portuguese", variant: nil)
|
|
366
|
+
variant ||= extract_region_code(code)
|
|
367
|
+
super
|
|
368
|
+
@hunspell_paths = resolve_hunspell_paths(code)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
def description
|
|
372
|
+
return name unless variant
|
|
373
|
+
|
|
374
|
+
variant_name = VARIANT_NAMES[variant] || variant
|
|
375
|
+
"#{name} (#{variant_name})"
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def tokenizer
|
|
379
|
+
@tokenizer ||= Tokenizer.new
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def normalizer
|
|
383
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
def dictionary_class
|
|
387
|
+
Dictionary::UnixWords
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def default_dictionary_paths
|
|
391
|
+
case code
|
|
392
|
+
when "pt-BR"
|
|
393
|
+
["/usr/share/dict/brazilian"]
|
|
394
|
+
when "pt-PT"
|
|
395
|
+
["/usr/share/dict/portuguese"]
|
|
396
|
+
else
|
|
397
|
+
["/usr/share/dict/words"]
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
def script_type
|
|
402
|
+
:latin
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def create_spell_checker
|
|
406
|
+
SpellChecker.new(
|
|
407
|
+
aff_path: @hunspell_paths[:aff],
|
|
408
|
+
dic_path: @hunspell_paths[:dic],
|
|
409
|
+
script: :latin
|
|
410
|
+
)
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def create_tokenizer
|
|
414
|
+
Tokenizer.new
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
def create_pos_tagger
|
|
418
|
+
POSTagger.new(
|
|
419
|
+
aff_path: @hunspell_paths[:aff],
|
|
420
|
+
dic_path: @hunspell_paths[:dic],
|
|
421
|
+
script: :latin,
|
|
422
|
+
flag_mapping: POSTagger::FLAG_TO_POS
|
|
423
|
+
)
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
private
|
|
427
|
+
|
|
428
|
+
def extract_region_code(code)
|
|
429
|
+
return nil unless code.include?("-")
|
|
430
|
+
|
|
431
|
+
code.split("-", 2).last.upcase
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def resolve_hunspell_paths(code)
|
|
435
|
+
HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['pt-BR']
|
|
436
|
+
end
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Romanian language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Romanian diacritics (ă â î ș ț) are AltGr sequences on the US physical layout.
|
|
9
|
+
class Romanian < LatinBase
|
|
10
|
+
register "ro"
|
|
11
|
+
register "ro-RO"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "ro", name: "Romanian")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/romanian"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|