kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# French language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Supports multiple dialects: fr-FR, fr-CA, fr-BE, fr-CH, fr-LU, fr-MC
|
|
8
|
+
#
|
|
9
|
+
# Full Hunspell integration with spell checking, POS tagging, and grammar rules.
|
|
10
|
+
class French < Language::Base
|
|
11
|
+
# French spell checker with Hunspell integration.
|
|
12
|
+
#
|
|
13
|
+
# Uses the Lookup algorithm with Hunspell-format dictionaries
|
|
14
|
+
# and French-specific character handling (accents, ligatures).
|
|
15
|
+
class SpellChecker < Components::SpellChecker
|
|
16
|
+
attr_reader :aff_path, :dic_path, :script
|
|
17
|
+
|
|
18
|
+
# French-specific character substitutions for suggestions
|
|
19
|
+
FRENCH_SUBSTITUTIONS = {
|
|
20
|
+
'à' => %w[a],
|
|
21
|
+
'â' => %w[a],
|
|
22
|
+
'ä' => %w[a],
|
|
23
|
+
'é' => %w[e],
|
|
24
|
+
'è' => %w[e],
|
|
25
|
+
'ê' => %w[e],
|
|
26
|
+
'ë' => %w[e],
|
|
27
|
+
'î' => %w[i],
|
|
28
|
+
'ï' => %w[i],
|
|
29
|
+
'ô' => %w[o],
|
|
30
|
+
'ö' => %w[o],
|
|
31
|
+
'ù' => %w[u],
|
|
32
|
+
'û' => %w[u],
|
|
33
|
+
'ü' => %w[u],
|
|
34
|
+
'ç' => %w[c],
|
|
35
|
+
'œ' => %w[oe],
|
|
36
|
+
'æ' => %w[ae],
|
|
37
|
+
# Common French errors
|
|
38
|
+
'c' => %w[ç], # garçon vs garcon
|
|
39
|
+
'e' => %w[é è ê], # café vs caffe
|
|
40
|
+
'a' => %w[à], # à vs a
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
|
|
44
|
+
@aff_path = aff_path
|
|
45
|
+
@dic_path = dic_path
|
|
46
|
+
@script = script
|
|
47
|
+
@encoding = encoding
|
|
48
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def check(word)
|
|
52
|
+
return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
|
|
53
|
+
|
|
54
|
+
first_form = @lookuper.good_forms(word).first
|
|
55
|
+
if first_form
|
|
56
|
+
{ found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
|
|
57
|
+
else
|
|
58
|
+
{ found: false, stem: nil, flags: [] }
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def suggest(word, max_suggestions: 10)
|
|
63
|
+
return [] if word.nil? || word.empty?
|
|
64
|
+
|
|
65
|
+
first_form = @lookuper.good_forms(word).first
|
|
66
|
+
return [] if first_form
|
|
67
|
+
|
|
68
|
+
generate_suggestions(word, max_suggestions).take(max_suggestions)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def correct?(word)
|
|
72
|
+
check(word)[:found]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def lookuper
|
|
76
|
+
@lookuper
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
def calculate_distance(a, b)
|
|
82
|
+
return a.length if b.empty?
|
|
83
|
+
return b.length if a.empty?
|
|
84
|
+
|
|
85
|
+
matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
|
|
86
|
+
(1..b.length).each { |j| matrix[0][j] = j }
|
|
87
|
+
(1..a.length).each do |i|
|
|
88
|
+
(1..b.length).each do |j|
|
|
89
|
+
cost = a[i - 1] == b[j - 1] ? 0 : 1
|
|
90
|
+
matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
matrix[a.length][b.length]
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def calculate_score(original, suggestion, rank)
|
|
97
|
+
distance = calculate_distance(original, suggestion)
|
|
98
|
+
max_len = [original.length, suggestion.length].max
|
|
99
|
+
distance_score = 1.0 - (distance.to_f / max_len)
|
|
100
|
+
rank_penalty = rank * 0.05
|
|
101
|
+
[distance_score - rank_penalty, 0.0].max
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def generate_suggestions(word, _max_suggestions)
|
|
105
|
+
variations = []
|
|
106
|
+
|
|
107
|
+
# Missing accents
|
|
108
|
+
word.downcase.chars.each_with_index do |char, i|
|
|
109
|
+
FRENCH_SUBSTITUTIONS.each do |accented, unaccented_variants|
|
|
110
|
+
unaccented_variants.each do |variant|
|
|
111
|
+
if char == variant
|
|
112
|
+
unaccented_word = word.dup
|
|
113
|
+
unaccented_word[i] = accented
|
|
114
|
+
variations << unaccented_word if @lookuper.good_forms(unaccented_word).first
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Doubled letters
|
|
121
|
+
word.chars.each_with_index do |char, i|
|
|
122
|
+
next if i == 0
|
|
123
|
+
|
|
124
|
+
doubled = word.dup
|
|
125
|
+
doubled.insert(i, char)
|
|
126
|
+
variations << doubled if @lookuper.good_forms(doubled).first
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Deleted letters
|
|
130
|
+
(0...word.length).each do |i|
|
|
131
|
+
deleted = word.dup
|
|
132
|
+
deleted.slice!(i)
|
|
133
|
+
next if deleted.empty?
|
|
134
|
+
|
|
135
|
+
variations << deleted if @lookuper.good_forms(deleted).first
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Common substitutions
|
|
139
|
+
word.chars.each_with_index do |char, i|
|
|
140
|
+
next unless FRENCH_SUBSTITUTIONS.key?(char.downcase)
|
|
141
|
+
|
|
142
|
+
FRENCH_SUBSTITUTIONS[char.downcase].each do |sub|
|
|
143
|
+
substituted = word.dup
|
|
144
|
+
substituted[i] = sub
|
|
145
|
+
variations << substituted if @lookuper.good_forms(substituted).first
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
variations.uniq!
|
|
150
|
+
variations.map do |suggestion|
|
|
151
|
+
{ word: suggestion, distance: calculate_distance(word, suggestion),
|
|
152
|
+
score: calculate_score(word, suggestion, 0) }
|
|
153
|
+
end.sort_by { |s| s[:distance] }
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# French tokenizer with contraction handling.
|
|
158
|
+
class Tokenizer < Language::Tokenizer::FrenchTokenizer
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# French POS tagger.
|
|
162
|
+
#
|
|
163
|
+
# Derives POS tags from Hunspell flags using French-specific mappings.
|
|
164
|
+
class POSTagger < Components::PosTagger
|
|
165
|
+
# French POS flag mappings based on Hunspell French dictionaries
|
|
166
|
+
FLAG_TO_POS = {
|
|
167
|
+
# Nouns
|
|
168
|
+
'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
|
|
169
|
+
# Verbs
|
|
170
|
+
'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
|
|
171
|
+
'VBP' => 'VERB', 'VBZ' => 'VERB',
|
|
172
|
+
# Adjectives
|
|
173
|
+
'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
|
|
174
|
+
# Adverbs
|
|
175
|
+
'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
|
|
176
|
+
# Determiners
|
|
177
|
+
'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
|
|
178
|
+
# Pronouns
|
|
179
|
+
'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
|
|
180
|
+
'WP' => 'PRON', 'WP$' => 'PRON_POSS',
|
|
181
|
+
# Prepositions
|
|
182
|
+
'I' => 'PREP', 'IN' => 'PREP',
|
|
183
|
+
# Conjunctions
|
|
184
|
+
'C' => 'CONJ', 'CC' => 'CONJ',
|
|
185
|
+
# Particles
|
|
186
|
+
'U' => 'PART', 'RP' => 'PART',
|
|
187
|
+
# Interjections
|
|
188
|
+
'INTJ' => 'INTJ', 'UH' => 'INTJ',
|
|
189
|
+
# Numbers
|
|
190
|
+
'CD' => 'NUM',
|
|
191
|
+
# Foreign words
|
|
192
|
+
'FW' => 'X',
|
|
193
|
+
# Punctuation
|
|
194
|
+
'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
|
|
195
|
+
'?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
|
|
196
|
+
}.freeze
|
|
197
|
+
|
|
198
|
+
attr_reader :aff_path, :dic_path, :script
|
|
199
|
+
|
|
200
|
+
def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
|
|
201
|
+
@aff_path = aff_path
|
|
202
|
+
@dic_path = dic_path
|
|
203
|
+
@script = script
|
|
204
|
+
@encoding = encoding
|
|
205
|
+
@flag_mapping = flag_mapping
|
|
206
|
+
@lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
|
|
207
|
+
@lookup_cache = {}
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def tag(tokens)
|
|
211
|
+
return [] if tokens.nil? || tokens.empty?
|
|
212
|
+
|
|
213
|
+
tokens.map do |token|
|
|
214
|
+
word = token[:token]
|
|
215
|
+
if word.nil? || word.empty?
|
|
216
|
+
token.merge(pos_tag: nil, lemma: nil)
|
|
217
|
+
else
|
|
218
|
+
lookup_result = lookup_with_pos(word)
|
|
219
|
+
token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def flag_mapping
|
|
225
|
+
@flag_mapping
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def flag_mapping=(mapping)
|
|
229
|
+
@flag_mapping = mapping
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def clear_cache
|
|
233
|
+
@lookup_cache.clear
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
private
|
|
237
|
+
|
|
238
|
+
def lookup_with_pos(word)
|
|
239
|
+
return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
|
|
240
|
+
return @lookup_cache[word] if @lookup_cache.key?(word)
|
|
241
|
+
|
|
242
|
+
first_form = @lookuper.good_forms(word).first
|
|
243
|
+
pos_tag = derive_pos_tag(first_form)
|
|
244
|
+
cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
|
|
245
|
+
@lookup_cache[word] = cache_result
|
|
246
|
+
cache_result
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def derive_pos_tag(result)
|
|
250
|
+
return nil unless result
|
|
251
|
+
|
|
252
|
+
flags = result.flags&.to_a || []
|
|
253
|
+
return guess_pos_from_affix(result) if flags.empty?
|
|
254
|
+
|
|
255
|
+
flags.each do |flag|
|
|
256
|
+
pos_tag = flag_to_pos(flag)
|
|
257
|
+
return pos_tag if pos_tag
|
|
258
|
+
end
|
|
259
|
+
guess_pos_from_affix(result)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def flag_to_pos(flag)
|
|
263
|
+
return @flag_mapping[flag] if @flag_mapping.key?(flag)
|
|
264
|
+
|
|
265
|
+
first_char = flag[0]
|
|
266
|
+
@flag_mapping[first_char]
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def guess_pos_from_affix(result)
|
|
270
|
+
suffix = result.suffix
|
|
271
|
+
return guess_pos_from_suffix(suffix) if suffix
|
|
272
|
+
|
|
273
|
+
nil
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def guess_pos_from_suffix(suffix)
|
|
277
|
+
# French suffix patterns
|
|
278
|
+
return 'VERB' if suffix.match?(/^(er|ir|re|is|it|issent|issons|issez)$/)
|
|
279
|
+
return 'ADV' if suffix.end_with?('ment')
|
|
280
|
+
return 'NOUN' if suffix.match?(/^(tion|sion|ment|age|ure|ée|ée)$/)
|
|
281
|
+
return 'ADJ' if suffix.match?(/^(if|ive|eux|euse|able|ible)$/)
|
|
282
|
+
|
|
283
|
+
nil
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# French grammar rules module.
|
|
288
|
+
module GrammarRules
|
|
289
|
+
# Base class for French grammar rules.
|
|
290
|
+
class Rule
|
|
291
|
+
attr_reader :id, :name, :description
|
|
292
|
+
|
|
293
|
+
def initialize(id, name, description)
|
|
294
|
+
@id = id
|
|
295
|
+
@name = name
|
|
296
|
+
@description = description
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def check(tokens)
|
|
300
|
+
raise NotImplementedError, "#{self.class} must implement #check"
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def applies?(_tokens, _index)
|
|
304
|
+
true
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Rule: Article agreement with gender/number.
|
|
309
|
+
class ArticleAgreementRule < Rule
|
|
310
|
+
MASCULINE_SINGULAR = %w[le un].freeze
|
|
311
|
+
FEMININE_SINGULAR = %w[la une].freeze
|
|
312
|
+
PLURAL = %w[les des].freeze
|
|
313
|
+
|
|
314
|
+
def initialize
|
|
315
|
+
super('FR_ARTICLE_AGREEMENT', 'Article Agreement', 'Articles must agree with noun gender and number.')
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def check(tokens)
|
|
319
|
+
errors = []
|
|
320
|
+
tokens.each_cons(2) do |article_token, noun_token|
|
|
321
|
+
article = article_token[:token]&.downcase
|
|
322
|
+
next unless MASCULINE_SINGULAR.include?(article) ||
|
|
323
|
+
FEMININE_SINGULAR.include?(article) ||
|
|
324
|
+
PLURAL.include?(article)
|
|
325
|
+
|
|
326
|
+
# This is a simplified check - full implementation would need dictionary lookup
|
|
327
|
+
# for gender/number information
|
|
328
|
+
next unless article_token[:pos_tag] == 'DET'
|
|
329
|
+
|
|
330
|
+
noun = noun_token[:token]
|
|
331
|
+
# Check for common patterns
|
|
332
|
+
if noun&.end_with?('e') && MASCULINE_SINGULAR.include?(article)
|
|
333
|
+
# Possibly incorrect: masculine article with feminine-looking noun
|
|
334
|
+
errors << {
|
|
335
|
+
rule_id: @id,
|
|
336
|
+
position: article_token[:position],
|
|
337
|
+
message: "Article agreement: check if '#{noun}' is feminine",
|
|
338
|
+
suggestion: nil,
|
|
339
|
+
context: "#{article} #{noun}",
|
|
340
|
+
suggestions: ['la', 'une']
|
|
341
|
+
}
|
|
342
|
+
end
|
|
343
|
+
end
|
|
344
|
+
errors
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Rule: Double negation in French (correct usage).
|
|
349
|
+
class FrenchNegationRule < Rule
|
|
350
|
+
NEGATION_PARTICLES = %w[ne n'].freeze
|
|
351
|
+
SECOND_PARTICLES = %w[pas plus jamais rien personne].freeze
|
|
352
|
+
|
|
353
|
+
def initialize
|
|
354
|
+
super('FR_NEGATION', 'French Negation', 'French uses double negation (ne...pas).')
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def check(tokens)
|
|
358
|
+
errors = []
|
|
359
|
+
tokens.each_with_index do |token, idx|
|
|
360
|
+
word = token[:token]&.downcase
|
|
361
|
+
next unless NEGATION_PARTICLES.include?(word)
|
|
362
|
+
|
|
363
|
+
# Check if second negation particle exists within reasonable distance
|
|
364
|
+
found_second = false
|
|
365
|
+
((idx + 1)...[idx + 5, tokens.length].min).each do |j|
|
|
366
|
+
next_word = tokens[j][:token]&.downcase
|
|
367
|
+
if SECOND_PARTICLES.include?(next_word)
|
|
368
|
+
found_second = true
|
|
369
|
+
break
|
|
370
|
+
end
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
unless found_second
|
|
374
|
+
errors << {
|
|
375
|
+
rule_id: @id,
|
|
376
|
+
position: token[:position],
|
|
377
|
+
message: "Incomplete negation: French requires double negation (ne...pas)",
|
|
378
|
+
suggestion: 'Add pas or another negation particle',
|
|
379
|
+
context: word,
|
|
380
|
+
suggestions: ['ne...pas', 'ne...pas']
|
|
381
|
+
}
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
errors
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
# Rule registry for French.
|
|
389
|
+
class RuleRegistry
|
|
390
|
+
class << self
|
|
391
|
+
def default_rules
|
|
392
|
+
[ArticleAgreementRule.new, FrenchNegationRule.new]
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
def get_rule(id)
|
|
396
|
+
default_rules.find { |rule| rule.id == id }
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
# Registration
|
|
403
|
+
register "fr"
|
|
404
|
+
register "fr-FR"
|
|
405
|
+
register "fr-CA"
|
|
406
|
+
register "fr-BE"
|
|
407
|
+
register "fr-CH"
|
|
408
|
+
register "fr-LU"
|
|
409
|
+
register "fr-MC"
|
|
410
|
+
|
|
411
|
+
HUNSPELL_DICTIONARIES = {
|
|
412
|
+
'fr-FR' => {
|
|
413
|
+
aff: 'spec/integrational/fixtures/fr_FR.aff',
|
|
414
|
+
dic: 'spec/integrational/fixtures/fr_FR.dic'
|
|
415
|
+
},
|
|
416
|
+
'fr-CA' => {
|
|
417
|
+
aff: 'spec/integrational/fixtures/fr_CA.aff',
|
|
418
|
+
dic: 'spec/integrational/fixtures/fr_CA.dic'
|
|
419
|
+
}
|
|
420
|
+
}.freeze
|
|
421
|
+
|
|
422
|
+
VARIANT_NAMES = {
|
|
423
|
+
'FR' => 'France',
|
|
424
|
+
'CA' => 'Canadian',
|
|
425
|
+
'BE' => 'Belgian',
|
|
426
|
+
'CH' => 'Swiss',
|
|
427
|
+
'LU' => 'Luxembourgish',
|
|
428
|
+
'MC' => 'Monégasque'
|
|
429
|
+
}.freeze
|
|
430
|
+
|
|
431
|
+
def initialize(code: "fr", name: "French", variant: nil)
|
|
432
|
+
variant ||= extract_region_code(code)
|
|
433
|
+
super
|
|
434
|
+
@hunspell_paths = resolve_hunspell_paths(code)
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
def description
|
|
438
|
+
return name unless variant
|
|
439
|
+
|
|
440
|
+
variant_name = VARIANT_NAMES[variant] || variant
|
|
441
|
+
"#{name} (#{variant_name})"
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
def tokenizer
|
|
445
|
+
@tokenizer ||= Tokenizer.new
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
def normalizer
|
|
449
|
+
@normalizer ||= Language::Normalizer::Base.new
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
def dictionary_class
|
|
453
|
+
Dictionary::UnixWords
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def default_dictionary_paths
|
|
457
|
+
case code
|
|
458
|
+
when "fr-FR"
|
|
459
|
+
["/usr/share/dict/french"]
|
|
460
|
+
when "fr-CA"
|
|
461
|
+
["/usr/share/dict/french-CA"]
|
|
462
|
+
else
|
|
463
|
+
["/usr/share/dict/words"]
|
|
464
|
+
end
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
def script_type
|
|
468
|
+
:latin
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
def create_spell_checker
|
|
472
|
+
SpellChecker.new(
|
|
473
|
+
aff_path: @hunspell_paths[:aff],
|
|
474
|
+
dic_path: @hunspell_paths[:dic],
|
|
475
|
+
script: :latin
|
|
476
|
+
)
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
def create_tokenizer
|
|
480
|
+
Tokenizer.new
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def create_pos_tagger
|
|
484
|
+
POSTagger.new(
|
|
485
|
+
aff_path: @hunspell_paths[:aff],
|
|
486
|
+
dic_path: @hunspell_paths[:dic],
|
|
487
|
+
script: :latin,
|
|
488
|
+
flag_mapping: POSTagger::FLAG_TO_POS
|
|
489
|
+
)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
private
|
|
493
|
+
|
|
494
|
+
def extract_region_code(code)
|
|
495
|
+
return nil unless code.include?("-")
|
|
496
|
+
|
|
497
|
+
code.split("-", 2).last.upcase
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def resolve_hunspell_paths(code)
|
|
501
|
+
HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['fr-FR']
|
|
502
|
+
end
|
|
503
|
+
end
|
|
504
|
+
end
|
|
505
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Hebrew language implementation.
|
|
6
|
+
#
|
|
7
|
+
# Registers the Hebrew language with {Language::Registry} under the
|
|
8
|
+
# +he+ and +he-IL+ codes. Uses the {Language::Normalizer::Hebrew}
|
|
9
|
+
# normalizer (NFC + niqqud stripping + dagesh removal + maqaf
|
|
10
|
+
# normalization) so that bare dictionary forms match against
|
|
11
|
+
# user input that may include vowel points.
|
|
12
|
+
#
|
|
13
|
+
# @example
|
|
14
|
+
# lang = Kotoshu::Languages::Hebrew.new
|
|
15
|
+
# lang.script_type # => :hebrew
|
|
16
|
+
# lang.rtl? # => true
|
|
17
|
+
#
|
|
18
|
+
# Full-feature since plan 100 (batch 3): the spelling dictionary
|
|
19
|
+
# downloads from the dictionaries repo (AVAILABLE_LANGUAGES) and
|
|
20
|
+
# keyboard proximity uses the Hebrew SI-1452 grid
|
|
21
|
+
# (Keyboard::Layouts::HebrewSI1452).
|
|
22
|
+
class Hebrew < Language::Base
|
|
23
|
+
register "he"
|
|
24
|
+
register "he-IL"
|
|
25
|
+
|
|
26
|
+
def initialize(code: "he", name: "Hebrew", variant: nil)
|
|
27
|
+
super
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def description
|
|
31
|
+
name
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def tokenizer
|
|
35
|
+
@tokenizer ||= Tokenizer.new
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def normalizer
|
|
39
|
+
@normalizer ||= Language::Normalizer::Hebrew.new
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def dictionary_class
|
|
43
|
+
Dictionary::Custom
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def default_dictionary_paths
|
|
47
|
+
["/usr/share/dict/hebrew"]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def script_type
|
|
51
|
+
:hebrew
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def rtl?
|
|
55
|
+
true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Minimal Hebrew tokenizer.
|
|
59
|
+
#
|
|
60
|
+
# Splits on whitespace and standard punctuation — Hebrew text
|
|
61
|
+
# is space-delimited. Includes the Hebrew-specific geresh (׳)
|
|
62
|
+
# and gershayim (״) as word characters (they are used inside
|
|
63
|
+
# acronyms). Uses Base#skip_token? which correctly handles
|
|
64
|
+
# Unicode letters via \p{L}.
|
|
65
|
+
class Tokenizer < Language::Tokenizer::Base
|
|
66
|
+
HEBREW_SEPARATORS = /[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-·]/
|
|
67
|
+
|
|
68
|
+
def tokenize(text)
|
|
69
|
+
return [] if text.nil? || text.strip.empty?
|
|
70
|
+
|
|
71
|
+
text.split(HEBREW_SEPARATORS)
|
|
72
|
+
.map { |token| normalize(token) }
|
|
73
|
+
.reject { |token| skip_token?(token) }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Spell-check word characters (plan 91 pattern, added by
|
|
77
|
+
# plan 100): Hebrew letters plus the geresh/gershayim used
|
|
78
|
+
# inside acronyms. Niqqud are combining marks, not letters —
|
|
79
|
+
# the Hebrew normalizer strips them from input before lookup.
|
|
80
|
+
#
|
|
81
|
+
# @return [Regexp] Regex matching a single word character
|
|
82
|
+
def spellcheck_word_regex
|
|
83
|
+
/[\p{Hebrew}\u05F3\u05F4]/
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
protected
|
|
87
|
+
|
|
88
|
+
def normalize(token)
|
|
89
|
+
token.strip
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def word_separators
|
|
93
|
+
HEBREW_SEPARATORS
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Croatian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. The five Croatian digraph letters š đ č ć ž are
|
|
9
|
+
# real keys on the South-Slavic QWERTZ physical layout
|
|
10
|
+
# (Keyboard::Layouts::Croatian), shared with Slovenian.
|
|
11
|
+
class Croatian < LatinBase
|
|
12
|
+
register "hr"
|
|
13
|
+
register "hr-HR"
|
|
14
|
+
|
|
15
|
+
def initialize(code: "hr", name: "Croatian")
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def default_dictionary_paths
|
|
20
|
+
["/usr/share/dict/croatian"]
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Hungarian language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Hungarian long vowels ő ű are dead-key sequences on the QWERTZ physical layout.
|
|
9
|
+
class Hungarian < LatinBase
|
|
10
|
+
register "hu"
|
|
11
|
+
register "hu-HU"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "hu", name: "Hungarian")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/hungarian"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Indonesian language implementation (plan 100, batch 3).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition like the wave-1 members: shared Latin
|
|
8
|
+
# tokenizer, base normalizer, registry entry. Indonesian is typed
|
|
9
|
+
# on the unmodified US QWERTY grid (Keyboard::Layouts::Latin::Indonesian)
|
|
10
|
+
# — the language has no diacritics, so the Latin base covers it
|
|
11
|
+
# without any per-language care.
|
|
12
|
+
class Indonesian < LatinBase
|
|
13
|
+
register "id"
|
|
14
|
+
register "id-ID"
|
|
15
|
+
|
|
16
|
+
def initialize(code: "id", name: "Indonesian")
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def default_dictionary_paths
|
|
21
|
+
["/usr/share/dict/indonesian"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Languages
|
|
5
|
+
# Italian language implementation (plan 84, wave 1).
|
|
6
|
+
#
|
|
7
|
+
# Thin composition: shared Latin tokenizer, base normalizer,
|
|
8
|
+
# registry entry. Italian accents (à è é ì ò ù) are dead-key sequences on the US physical layout.
|
|
9
|
+
class Italian < LatinBase
|
|
10
|
+
register "it"
|
|
11
|
+
register "it-IT"
|
|
12
|
+
|
|
13
|
+
def initialize(code: "it", name: "Italian")
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def default_dictionary_paths
|
|
18
|
+
["/usr/share/dict/italian"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|