kotoshu 1.0.2-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.kotoshu-baseline.json +1 -0
- data/.pre-commit-hooks.yaml +18 -0
- data/.rspec +3 -0
- data/.rubocop.yml +351 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +813 -0
- data/CLAUDE.md +176 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/Cargo.lock +359 -0
- data/Cargo.toml +15 -0
- data/LICENSE +31 -0
- data/README.adoc +1292 -0
- data/Rakefile +150 -0
- data/SECURITY.md +93 -0
- data/TODO.impl/00-cut-0.1.md +121 -0
- data/TODO.impl/00-cut-0.2.md +175 -0
- data/TODO.impl/00-vision.md +281 -0
- data/TODO.impl/01-hunspell-correctness.md +111 -0
- data/TODO.impl/02-cli-unification.md +74 -0
- data/TODO.impl/03-dynamic-download.md +107 -0
- data/TODO.impl/04-language-modules.md +89 -0
- data/TODO.impl/05-semantic-path.md +83 -0
- data/TODO.impl/06-cjk-support.md +105 -0
- data/TODO.impl/07-rtl-support.md +71 -0
- data/TODO.impl/08-grammar-engine.md +106 -0
- data/TODO.impl/09-integrity-security.md +90 -0
- data/TODO.impl/10-testing-ci.md +98 -0
- data/TODO.impl/100-full-feature-batch3.md +28 -0
- data/TODO.impl/101-int4-tier-experiment.md +32 -0
- data/TODO.impl/102-lid-chain.md +28 -0
- data/TODO.impl/103-oov-bucket-tables.md +27 -0
- data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
- data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
- data/TODO.impl/106-native-lid-detection.md +36 -0
- data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
- data/TODO.impl/108-ko-ne-full-feature.md +37 -0
- data/TODO.impl/109-ten-audit.md +26 -0
- data/TODO.impl/11-release-v1.md +95 -0
- data/TODO.impl/110-sr-latn-and-polish.md +33 -0
- data/TODO.impl/111-embedding-bakeoff.md +33 -0
- data/TODO.impl/112-deployability-and-gating.md +26 -0
- data/TODO.impl/113-language-packs.md +26 -0
- data/TODO.impl/114-typo-biencoder-thread.md +33 -0
- data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
- data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
- data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
- data/TODO.impl/118-server-gem-floor.md +33 -0
- data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
- data/TODO.impl/120-setup-metadata-refresh.md +35 -0
- data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
- data/TODO.impl/122-pack-compression-kpk2.md +66 -0
- data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
- data/TODO.impl/124-owner-residue.md +19 -0
- data/TODO.impl/125-consumer-floor-guards.md +43 -0
- data/TODO.impl/126-tier-error-budgets.md +33 -0
- data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
- data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
- data/TODO.impl/129-pt-fr-corpora.md +23 -0
- data/TODO.impl/130-news-truth-0912.md +23 -0
- data/TODO.impl/131-hybrid-productization.md +42 -0
- data/TODO.impl/132-techniques-page-and-prose.md +27 -0
- data/TODO.impl/133-rust-first-distribution.md +68 -0
- data/TODO.impl/20-encapsulation-cleanup.md +47 -0
- data/TODO.impl/21-autoload-migration.md +64 -0
- data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
- data/TODO.impl/23-fix-url-defaults.md +57 -0
- data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
- data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
- data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
- data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
- data/TODO.impl/28-readme-quickstart-verification.md +68 -0
- data/TODO.impl/29-status-command.md +84 -0
- data/TODO.impl/30-language-auto-detection.md +74 -0
- data/TODO.impl/31-download-progress.md +58 -0
- data/TODO.impl/32-sarif-output-verification.md +72 -0
- data/TODO.impl/33-audit-log-rotation.md +54 -0
- data/TODO.impl/34-cache-eviction.md +59 -0
- data/TODO.impl/35-shell-completion.md +61 -0
- data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
- data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
- data/TODO.impl/38-onnx-semantic-gating.md +218 -0
- data/TODO.impl/39-tier3-and-beyond.md +232 -0
- data/TODO.impl/40-spec-drift-cleanup.md +134 -0
- data/TODO.impl/50-structure-aware-document-api.md +194 -0
- data/TODO.impl/51-grammar-engine-expansion.md +75 -0
- data/TODO.impl/52-multi-language-documents.md +95 -0
- data/TODO.impl/53-document-plugin-architecture.md +94 -0
- data/TODO.impl/54-cjk-support.md +117 -0
- data/TODO.impl/55-rtl-support.md +109 -0
- data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
- data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
- data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
- data/TODO.impl/59-yard-api-documentation.md +75 -0
- data/TODO.impl/60-lsp-server.md +174 -0
- data/TODO.impl/61-editor-ecosystem.md +167 -0
- data/TODO.impl/62-framework-integrations.md +159 -0
- data/TODO.impl/63-web-playground-wasm.md +186 -0
- data/TODO.impl/64-http-api-and-sdks.md +191 -0
- data/TODO.impl/65-universal-kotoshu.md +251 -0
- data/TODO.impl/66-kotoshu-core.md +169 -0
- data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
- data/TODO.impl/68-sota-adoption.md +152 -0
- data/TODO.impl/69-sota-typo-eval.md +74 -0
- data/TODO.impl/70-sota-confidence-cascade.md +59 -0
- data/TODO.impl/71-sota-nested-model.md +63 -0
- data/TODO.impl/72-model2vec-evaluation.md +49 -0
- data/TODO.impl/73-rl-reranker.md +42 -0
- data/TODO.impl/74-distilled-reranker.md +36 -0
- data/TODO.impl/75-llm-evaluator.md +35 -0
- data/TODO.impl/76-optional-llm-backend.md +37 -0
- data/TODO.impl/77-model-coverage-expansion.md +96 -0
- data/TODO.impl/78-in-browser-playground.md +69 -0
- data/TODO.impl/79-news-and-releases.md +60 -0
- data/TODO.impl/80-ecosystem-docs.md +56 -0
- data/TODO.impl/81-editor-reach.md +68 -0
- data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
- data/TODO.impl/83-models-batch-2.md +56 -0
- data/TODO.impl/84-language-modules-wave2.md +75 -0
- data/TODO.impl/85-wasm-model-api.md +48 -0
- data/TODO.impl/86-site-truth-wave2.md +45 -0
- data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
- data/TODO.impl/88-cli-directory-mode.md +54 -0
- data/TODO.impl/89-framework-integrations-exec.md +59 -0
- data/TODO.impl/90-gem-correctness-residues.md +44 -0
- data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
- data/TODO.impl/92-tier-mirror-cors.md +88 -0
- data/TODO.impl/93-python-wheels.md +32 -0
- data/TODO.impl/94-site-integrations-docs.md +35 -0
- data/TODO.impl/95-editor-action-polish.md +27 -0
- data/TODO.impl/96-server-semantic-models.md +50 -0
- data/TODO.impl/97-yard-execution.md +73 -0
- data/TODO.impl/98-campaign-cleanup.md +78 -0
- data/TODO.impl/99-site-truth-performance-docs.md +31 -0
- data/conformance/vectors.jsonl +2630 -0
- data/data/en/grammar/rules.yaml +851 -0
- data/docs/STABILITY-1.0-AUDIT.md +251 -0
- data/docs/STABILITY-1.0.md +137 -0
- data/docs/cache-architecture.md +502 -0
- data/docs/verification-evidence/correct3/pr_body.md +37 -0
- data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
- data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
- data/docs/verification-evidence/correct3/sv.aff +856 -0
- data/docs/verification-evidence/correct3/sv.dic +153717 -0
- data/examples/01_basic_word_checking.rb +38 -0
- data/examples/02_text_document_checking.rb +77 -0
- data/examples/03_dictionary_backends.rb +137 -0
- data/examples/04_trie_data_structure.rb +146 -0
- data/examples/05_suggestion_algorithms.rb +239 -0
- data/examples/06_configuration_advanced.rb +287 -0
- data/examples/07_multi_language_dictionaries.rb +278 -0
- data/exe/kotoshu +6 -0
- data/lib/kotoshu/algorithms/capitalization.rb +276 -0
- data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
- data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
- data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
- data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
- data/lib/kotoshu/algorithms/lookup.rb +841 -0
- data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
- data/lib/kotoshu/algorithms/permutations.rb +283 -0
- data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
- data/lib/kotoshu/algorithms/soundex.rb +92 -0
- data/lib/kotoshu/algorithms/suggest.rb +794 -0
- data/lib/kotoshu/algorithms.rb +23 -0
- data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
- data/lib/kotoshu/analyzers.rb +8 -0
- data/lib/kotoshu/baseline/application.rb +18 -0
- data/lib/kotoshu/baseline/entry.rb +33 -0
- data/lib/kotoshu/baseline/store.rb +175 -0
- data/lib/kotoshu/baseline.rb +20 -0
- data/lib/kotoshu/cache/base_cache.rb +778 -0
- data/lib/kotoshu/cache/cache.rb +91 -0
- data/lib/kotoshu/cache/eviction_policy.rb +66 -0
- data/lib/kotoshu/cache/frequency_cache.rb +283 -0
- data/lib/kotoshu/cache/language_cache.rb +511 -0
- data/lib/kotoshu/cache/lookup_cache.rb +166 -0
- data/lib/kotoshu/cache/model_cache.rb +1147 -0
- data/lib/kotoshu/cache/model_registry.rb +109 -0
- data/lib/kotoshu/cache/staged_languages.rb +45 -0
- data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
- data/lib/kotoshu/cache.rb +49 -0
- data/lib/kotoshu/cli/auto_setup.rb +71 -0
- data/lib/kotoshu/cli/baseline_command.rb +119 -0
- data/lib/kotoshu/cli/batch_reporter.rb +321 -0
- data/lib/kotoshu/cli/cache_command.rb +282 -0
- data/lib/kotoshu/cli/completions_command.rb +210 -0
- data/lib/kotoshu/cli/dict_command.rb +64 -0
- data/lib/kotoshu/cli/directory_check.rb +146 -0
- data/lib/kotoshu/cli/directory_walker.rb +132 -0
- data/lib/kotoshu/cli/display_formatter.rb +429 -0
- data/lib/kotoshu/cli/errors.rb +36 -0
- data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
- data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
- data/lib/kotoshu/cli/language_resolver.rb +91 -0
- data/lib/kotoshu/cli/navigation_manager.rb +273 -0
- data/lib/kotoshu/cli/personal_command.rb +113 -0
- data/lib/kotoshu/cli/progress_reporter.rb +114 -0
- data/lib/kotoshu/cli/status_report.rb +130 -0
- data/lib/kotoshu/cli.rb +880 -0
- data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
- data/lib/kotoshu/components/pos_tagger.rb +98 -0
- data/lib/kotoshu/components/spell_checker.rb +73 -0
- data/lib/kotoshu/components/synthesizer.rb +60 -0
- data/lib/kotoshu/components/tokenizer.rb +58 -0
- data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
- data/lib/kotoshu/components.rb +13 -0
- data/lib/kotoshu/configuration/builder.rb +207 -0
- data/lib/kotoshu/configuration/resolver.rb +124 -0
- data/lib/kotoshu/configuration.rb +829 -0
- data/lib/kotoshu/conformance_exporter.rb +256 -0
- data/lib/kotoshu/conformance_runner.rb +176 -0
- data/lib/kotoshu/core/exceptions.rb +186 -0
- data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
- data/lib/kotoshu/core/models/affix_rule.rb +262 -0
- data/lib/kotoshu/core/models/result/document_result.rb +132 -0
- data/lib/kotoshu/core/models/result/word_result.rb +131 -0
- data/lib/kotoshu/core/models/word.rb +131 -0
- data/lib/kotoshu/core/trie/builder.rb +119 -0
- data/lib/kotoshu/core/trie/node.rb +94 -0
- data/lib/kotoshu/core/trie/trie.rb +249 -0
- data/lib/kotoshu/core.rb +28 -0
- data/lib/kotoshu/data/common_words/de.yml +1800 -0
- data/lib/kotoshu/data/common_words/en.yml +1215 -0
- data/lib/kotoshu/data/common_words/es.yml +750 -0
- data/lib/kotoshu/data/common_words/fr.yml +1015 -0
- data/lib/kotoshu/data/common_words/pt.yml +870 -0
- data/lib/kotoshu/data/common_words/ru.yml +484 -0
- data/lib/kotoshu/data/common_words_loader.rb +141 -0
- data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
- data/lib/kotoshu/data/frozen_kelly.rb +38 -0
- data/lib/kotoshu/data.rb +8 -0
- data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
- data/lib/kotoshu/data_structures.rb +8 -0
- data/lib/kotoshu/debug_logger.rb +146 -0
- data/lib/kotoshu/debug_mode.rb +136 -0
- data/lib/kotoshu/defaults.rb +84 -0
- data/lib/kotoshu/dictionaries/catalog.rb +814 -0
- data/lib/kotoshu/dictionaries.rb +8 -0
- data/lib/kotoshu/dictionary/base.rb +324 -0
- data/lib/kotoshu/dictionary/cspell.rb +243 -0
- data/lib/kotoshu/dictionary/custom.rb +225 -0
- data/lib/kotoshu/dictionary/hunspell.rb +520 -0
- data/lib/kotoshu/dictionary/plain_text.rb +302 -0
- data/lib/kotoshu/dictionary/repository.rb +246 -0
- data/lib/kotoshu/dictionary/unified.rb +236 -0
- data/lib/kotoshu/dictionary/unix_words.rb +218 -0
- data/lib/kotoshu/dictionary.rb +15 -0
- data/lib/kotoshu/documents/document.rb +120 -0
- data/lib/kotoshu/documents/plain_text_document.rb +73 -0
- data/lib/kotoshu/documents/source_position.rb +45 -0
- data/lib/kotoshu/documents/source_range.rb +76 -0
- data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
- data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
- data/lib/kotoshu/documents/suppressions.rb +44 -0
- data/lib/kotoshu/documents/text_node.rb +49 -0
- data/lib/kotoshu/documents.rb +178 -0
- data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
- data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
- data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
- data/lib/kotoshu/embeddings/protocol.rb +96 -0
- data/lib/kotoshu/embeddings/protocols.rb +23 -0
- data/lib/kotoshu/embeddings/registry.rb +184 -0
- data/lib/kotoshu/embeddings/search.rb +204 -0
- data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
- data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
- data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
- data/lib/kotoshu/embeddings.rb +87 -0
- data/lib/kotoshu/fluent_checker.rb +94 -0
- data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
- data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
- data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
- data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
- data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
- data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
- data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
- data/lib/kotoshu/grammar/rule.rb +99 -0
- data/lib/kotoshu/grammar/rule_engine.rb +96 -0
- data/lib/kotoshu/grammar/rule_loader.rb +30 -0
- data/lib/kotoshu/grammar.rb +24 -0
- data/lib/kotoshu/integrity/audit_log.rb +167 -0
- data/lib/kotoshu/integrity/manifest.rb +116 -0
- data/lib/kotoshu/integrity/net_http.rb +51 -0
- data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
- data/lib/kotoshu/integrity.rb +26 -0
- data/lib/kotoshu/jekyll.rb +101 -0
- data/lib/kotoshu/keyboard/layout.rb +116 -0
- data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
- data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
- data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
- data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
- data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
- data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
- data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
- data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
- data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
- data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
- data/lib/kotoshu/keyboard/registry.rb +201 -0
- data/lib/kotoshu/keyboard.rb +96 -0
- data/lib/kotoshu/kotoshu_native.so +0 -0
- data/lib/kotoshu/language/detector.rb +242 -0
- data/lib/kotoshu/language/identifier.rb +376 -0
- data/lib/kotoshu/language/languages/base.rb +271 -0
- data/lib/kotoshu/language/lid_detector.rb +188 -0
- data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
- data/lib/kotoshu/language/normalizer/base.rb +133 -0
- data/lib/kotoshu/language/normalizer/greek.rb +63 -0
- data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
- data/lib/kotoshu/language/normalizer/persian.rb +36 -0
- data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
- data/lib/kotoshu/language/registry.rb +209 -0
- data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
- data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
- data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
- data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
- data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
- data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
- data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
- data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
- data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
- data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
- data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
- data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
- data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
- data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
- data/lib/kotoshu/language/script.rb +128 -0
- data/lib/kotoshu/language/segmenter.rb +122 -0
- data/lib/kotoshu/language/suika.rb +38 -0
- data/lib/kotoshu/language/tokenizer/base.rb +187 -0
- data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
- data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
- data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
- data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
- data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
- data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
- data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
- data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
- data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
- data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
- data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
- data/lib/kotoshu/language.rb +127 -0
- data/lib/kotoshu/languages/ar/language.rb +98 -0
- data/lib/kotoshu/languages/bg/language.rb +37 -0
- data/lib/kotoshu/languages/ca/language.rb +22 -0
- data/lib/kotoshu/languages/cs/language.rb +22 -0
- data/lib/kotoshu/languages/da/language.rb +22 -0
- data/lib/kotoshu/languages/de/language.rb +554 -0
- data/lib/kotoshu/languages/el/language.rb +50 -0
- data/lib/kotoshu/languages/en/language.rb +470 -0
- data/lib/kotoshu/languages/es/language.rb +476 -0
- data/lib/kotoshu/languages/et/language.rb +24 -0
- data/lib/kotoshu/languages/fa/language.rb +91 -0
- data/lib/kotoshu/languages/fr/language.rb +505 -0
- data/lib/kotoshu/languages/he/language.rb +98 -0
- data/lib/kotoshu/languages/hr/language.rb +24 -0
- data/lib/kotoshu/languages/hu/language.rb +22 -0
- data/lib/kotoshu/languages/id/language.rb +25 -0
- data/lib/kotoshu/languages/it/language.rb +22 -0
- data/lib/kotoshu/languages/ja/language.rb +476 -0
- data/lib/kotoshu/languages/ko/language.rb +56 -0
- data/lib/kotoshu/languages/latin_base.rb +53 -0
- data/lib/kotoshu/languages/lt/language.rb +24 -0
- data/lib/kotoshu/languages/lv/language.rb +24 -0
- data/lib/kotoshu/languages/nb/language.rb +25 -0
- data/lib/kotoshu/languages/ne/language.rb +60 -0
- data/lib/kotoshu/languages/nl/language.rb +22 -0
- data/lib/kotoshu/languages/nn/language.rb +26 -0
- data/lib/kotoshu/languages/pl/language.rb +22 -0
- data/lib/kotoshu/languages/pt/language.rb +439 -0
- data/lib/kotoshu/languages/ro/language.rb +22 -0
- data/lib/kotoshu/languages/ru/language.rb +420 -0
- data/lib/kotoshu/languages/sk/language.rb +25 -0
- data/lib/kotoshu/languages/sl/language.rb +25 -0
- data/lib/kotoshu/languages/sr/language.rb +39 -0
- data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
- data/lib/kotoshu/languages/sv/language.rb +22 -0
- data/lib/kotoshu/languages/tr/language.rb +36 -0
- data/lib/kotoshu/languages/uk/language.rb +38 -0
- data/lib/kotoshu/languages/vi/language.rb +22 -0
- data/lib/kotoshu/languages.rb +64 -0
- data/lib/kotoshu/metrics_collector.rb +238 -0
- data/lib/kotoshu/metrics_module.rb +121 -0
- data/lib/kotoshu/models/context.rb +119 -0
- data/lib/kotoshu/models/embedding_model.rb +181 -0
- data/lib/kotoshu/models/fasttext_model.rb +212 -0
- data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
- data/lib/kotoshu/models/onnx_model.rb +330 -0
- data/lib/kotoshu/models/semantic_error.rb +204 -0
- data/lib/kotoshu/models/suggestion.rb +106 -0
- data/lib/kotoshu/models/word_embedding.rb +97 -0
- data/lib/kotoshu/models.rb +27 -0
- data/lib/kotoshu/multi_language_checker.rb +127 -0
- data/lib/kotoshu/native.rb +83 -0
- data/lib/kotoshu/native_backend.rb +131 -0
- data/lib/kotoshu/paths.rb +79 -0
- data/lib/kotoshu/personal_dictionary.rb +105 -0
- data/lib/kotoshu/plugins/plugin.rb +61 -0
- data/lib/kotoshu/plugins/registry.rb +118 -0
- data/lib/kotoshu/plugins.rb +9 -0
- data/lib/kotoshu/project_config.rb +76 -0
- data/lib/kotoshu/readers/aff_data.rb +530 -0
- data/lib/kotoshu/readers/aff_reader.rb +393 -0
- data/lib/kotoshu/readers/condition_checker.rb +125 -0
- data/lib/kotoshu/readers/dic_reader.rb +208 -0
- data/lib/kotoshu/readers/file_reader.rb +377 -0
- data/lib/kotoshu/readers/lookup_builder.rb +584 -0
- data/lib/kotoshu/readers/readers.rb +17 -0
- data/lib/kotoshu/readers.rb +25 -0
- data/lib/kotoshu/resource_bundle.rb +55 -0
- data/lib/kotoshu/resource_manager.rb +600 -0
- data/lib/kotoshu/results/result.rb +165 -0
- data/lib/kotoshu/results.rb +8 -0
- data/lib/kotoshu/rspec.rb +163 -0
- data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
- data/lib/kotoshu/source_registry.rb +104 -0
- data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
- data/lib/kotoshu/spellchecker.rb +406 -0
- data/lib/kotoshu/string_metrics.rb +153 -0
- data/lib/kotoshu/suggestions/context.rb +43 -0
- data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
- data/lib/kotoshu/suggestions/generator.rb +169 -0
- data/lib/kotoshu/suggestions/pipeline.rb +135 -0
- data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
- data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
- data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
- data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
- data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
- data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
- data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
- data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
- data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
- data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
- data/lib/kotoshu/suggestions/suggestion.rb +154 -0
- data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
- data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
- data/lib/kotoshu/suggestions.rb +29 -0
- data/lib/kotoshu/tasks/check_task.rb +104 -0
- data/lib/kotoshu/tasks.rb +11 -0
- data/lib/kotoshu/validators/spelling_validator.rb +101 -0
- data/lib/kotoshu/validators.rb +9 -0
- data/lib/kotoshu/version.rb +5 -0
- data/lib/kotoshu.rb +538 -0
- data/rakelib/compile.rake +10 -0
- data/script/validate_all_dictionaries.rb +444 -0
- data/scripts/build_native_gem.rb +45 -0
- data/scripts/extract_eval_grids.rb +37 -0
- data/scripts/generate_frozen_tiers.rb +44 -0
- data/sig/kotoshu.rbs +435 -0
- data/test_oop.rb +79 -0
- metadata +546 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kotoshu
|
|
4
|
+
module Readers
|
|
5
|
+
# Word entry from the dictionary file.
|
|
6
|
+
#
|
|
7
|
+
# @attr stem [String] The word stem
|
|
8
|
+
# @attr flags [Set<String>] Morphological flags
|
|
9
|
+
# @attr morph_data [Array<String>] Morphological data fields (e.g. "ph:wich")
|
|
10
|
+
Word = Struct.new(:stem, :flags, :morph_data, keyword_init: true) do
|
|
11
|
+
# Create a word from a dictionary line.
|
|
12
|
+
#
|
|
13
|
+
# @param line [String] The dictionary line
|
|
14
|
+
# @param context [Hash] The reading context (for flag parsing)
|
|
15
|
+
# @return [Word] The parsed word
|
|
16
|
+
def self.from_line(line, context = {})
|
|
17
|
+
# Format: stem[/flags][<SEP><morph_data>]
|
|
18
|
+
# SEP is a tab per spec, but real-world fixtures (including the
|
|
19
|
+
# Hunspell ph.* tests) sometimes use spaces. We split on the first
|
|
20
|
+
# morphological token (key:value) regardless of separator.
|
|
21
|
+
head, morph = split_stem_and_morph(line)
|
|
22
|
+
head = head.strip
|
|
23
|
+
|
|
24
|
+
# Find the first UNESCAPED slash to split stem from flags.
|
|
25
|
+
# Hunspell allows `\/` to represent a literal `/` inside a word —
|
|
26
|
+
# so we can't just use String#index('/'). Mirrors Spylls's
|
|
27
|
+
# SLASH_REGEXP (a slash not preceded by a backslash).
|
|
28
|
+
#
|
|
29
|
+
# Special case: a word that STARTS with `/` is not an empty stem +
|
|
30
|
+
# flags — it's a word whose first character is `/`. Without this,
|
|
31
|
+
# dic entry `/` would be parsed as stem="" + flags="".
|
|
32
|
+
if head.start_with?('/')
|
|
33
|
+
stem = head
|
|
34
|
+
flags_str = nil
|
|
35
|
+
else
|
|
36
|
+
slash_idx = unescaped_slash_index(head)
|
|
37
|
+
if slash_idx
|
|
38
|
+
stem = head[0...slash_idx]
|
|
39
|
+
flags_str = head[(slash_idx + 1)..]
|
|
40
|
+
else
|
|
41
|
+
stem = head
|
|
42
|
+
flags_str = nil
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Replace escaped slashes in the stem: `\/` → `/`
|
|
46
|
+
stem = stem.gsub('\/', '/') if stem.include?('\/')
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
flags = if flags_str && !flags_str.empty? && context[:flag_format]
|
|
50
|
+
parse_flags(flags_str, context[:flag_format], context[:flag_synonyms])
|
|
51
|
+
elsif flags_str && !flags_str.empty?
|
|
52
|
+
flags_str.chars.to_set
|
|
53
|
+
else
|
|
54
|
+
Set.new
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
morph_data = parse_morph_data(morph)
|
|
58
|
+
new(stem:, flags:, morph_data:)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Split a dictionary line into stem and morphological-data portions.
|
|
62
|
+
#
|
|
63
|
+
# Hunspell specifies tab-separated morph data, but the ph.* test
|
|
64
|
+
# fixtures use spaces. We honor both: if there's a tab, split there;
|
|
65
|
+
# otherwise split before the first `key:value` token (which is the
|
|
66
|
+
# universal signature of morphological data).
|
|
67
|
+
#
|
|
68
|
+
# @param line [String] The raw dictionary line
|
|
69
|
+
# @return [Array(String, String)] [stem portion, morph portion]
|
|
70
|
+
def self.split_stem_and_morph(line)
|
|
71
|
+
tab_idx = line.index("\t")
|
|
72
|
+
return line.split("\t", 2) if tab_idx
|
|
73
|
+
|
|
74
|
+
match = line.match(/(.*?)(\s+[a-zA-Z]+:[^\s].*)$/)
|
|
75
|
+
return [line, ''] unless match
|
|
76
|
+
|
|
77
|
+
[match[1], match[2]]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Parse morphological data from the post-tab portion of a dic line.
|
|
81
|
+
#
|
|
82
|
+
# Hunspell morphological data is whitespace-separated key:value tokens
|
|
83
|
+
# such as `ph:wich` (phonetic), `st:foo` (stem), `po:noun` (part of
|
|
84
|
+
# speech). We preserve them as a list of raw strings — downstream
|
|
85
|
+
# consumers (e.g. PhonetSuggest via alt_spellings) extract what they
|
|
86
|
+
# need.
|
|
87
|
+
#
|
|
88
|
+
# @param morph [String, nil] The morphological portion
|
|
89
|
+
# @return [Array<String>] List of morphological tokens
|
|
90
|
+
def self.parse_morph_data(morph)
|
|
91
|
+
return [] if morph.nil?
|
|
92
|
+
|
|
93
|
+
morph.split(/\s+/).reject(&:empty?)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Parse flags from string.
|
|
97
|
+
#
|
|
98
|
+
# @param string [String] Flag string
|
|
99
|
+
# @param flag_format [String] Flag format ('short', 'long', 'num', 'UTF-8')
|
|
100
|
+
# @param flag_synonyms [Hash] Flag synonyms map
|
|
101
|
+
# @return [Set<String>] Parsed flags
|
|
102
|
+
def self.parse_flags(string, flag_format, flag_synonyms = {})
|
|
103
|
+
return Set.new if string.nil? || string.empty?
|
|
104
|
+
|
|
105
|
+
# AF (flag aliases) only applies when aliases are actually defined
|
|
106
|
+
# and the string is a positional alias index (pure digits). Without
|
|
107
|
+
# the !empty? guard, a `FLAG num` dictionary with no AF would have
|
|
108
|
+
# every numeric flag collapsed to the empty set.
|
|
109
|
+
if flag_synonyms.is_a?(Hash) && !flag_synonyms.empty? && string =~ /^\d+$/
|
|
110
|
+
return flag_synonyms[string] || Set.new
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
case flag_format
|
|
114
|
+
when 'short'
|
|
115
|
+
string.chars.to_set
|
|
116
|
+
when 'long'
|
|
117
|
+
string.scan(/../).to_set
|
|
118
|
+
when 'num'
|
|
119
|
+
string.scan(/\d+/).to_set
|
|
120
|
+
when 'UTF-8'
|
|
121
|
+
string.chars.to_set
|
|
122
|
+
else
|
|
123
|
+
string.chars.to_set
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Find the index of the first unescaped slash in the string.
|
|
128
|
+
#
|
|
129
|
+
# @param str [String] Input string
|
|
130
|
+
# @return [Integer, nil] Index of first unescaped `/`, or nil
|
|
131
|
+
def self.unescaped_slash_index(str)
|
|
132
|
+
i = 0
|
|
133
|
+
while i < str.length
|
|
134
|
+
c = str[i]
|
|
135
|
+
return i if c == '/' && (i.zero? || str[i - 1] != '\\')
|
|
136
|
+
|
|
137
|
+
i += 1
|
|
138
|
+
end
|
|
139
|
+
nil
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# DIC file reader for Hunspell dictionary files.
|
|
144
|
+
#
|
|
145
|
+
# This class reads .dic files and creates a list of Word entries.
|
|
146
|
+
#
|
|
147
|
+
# @example Reading a dic file
|
|
148
|
+
# reader = DicReader.new('en_US.dic', flag_format: 'short')
|
|
149
|
+
# words = reader.read
|
|
150
|
+
class DicReader
|
|
151
|
+
attr_reader :path, :encoding, :flag_format, :flag_synonyms
|
|
152
|
+
|
|
153
|
+
# Create a new DIC reader.
|
|
154
|
+
#
|
|
155
|
+
# @param path [String] Path to the .dic file
|
|
156
|
+
# @param encoding [String] File encoding (default: 'UTF-8')
|
|
157
|
+
# @param flag_format [String] Flag format ('short', 'long', 'num', 'UTF-8')
|
|
158
|
+
# @param flag_synonyms [Hash] Flag synonyms map
|
|
159
|
+
def initialize(path, encoding: 'UTF-8', flag_format: 'short', flag_synonyms: {})
|
|
160
|
+
@path = path
|
|
161
|
+
@encoding = encoding
|
|
162
|
+
@flag_format = flag_format
|
|
163
|
+
@flag_synonyms = flag_synonyms
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Read the dic file and return a list of Word entries.
|
|
167
|
+
#
|
|
168
|
+
# @param source [FileReader, nil] Optional file reader to use instead of creating a new one
|
|
169
|
+
# @return [Array<Word>] List of word entries
|
|
170
|
+
def read(source = nil)
|
|
171
|
+
owned = source.nil?
|
|
172
|
+
reader = source || FileReader.new(@path, @encoding)
|
|
173
|
+
|
|
174
|
+
begin
|
|
175
|
+
words = []
|
|
176
|
+
first_line = true
|
|
177
|
+
expected_count = 0
|
|
178
|
+
|
|
179
|
+
reader.each do |_line_no, line|
|
|
180
|
+
if first_line
|
|
181
|
+
# First line is word count
|
|
182
|
+
expected_count = line.to_i
|
|
183
|
+
first_line = false
|
|
184
|
+
next
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Skip empty lines
|
|
188
|
+
next if line.empty?
|
|
189
|
+
|
|
190
|
+
# Parse word
|
|
191
|
+
word = Word.from_line(line, flag_format: @flag_format, flag_synonyms: @flag_synonyms)
|
|
192
|
+
words << word
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Verify word count
|
|
196
|
+
# Note: We don't raise an error if count doesn't match, as some dictionaries have different formats
|
|
197
|
+
|
|
198
|
+
words
|
|
199
|
+
ensure
|
|
200
|
+
# Close the reader only when we created it. If the caller passed
|
|
201
|
+
# one in, they own its lifecycle. Without this, the underlying
|
|
202
|
+
# File handle leaks (on Windows it also blocks tempfile cleanup).
|
|
203
|
+
reader.close if owned
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rubygems'
|
|
4
|
+
|
|
5
|
+
begin
|
|
6
|
+
require 'zip'
|
|
7
|
+
rescue LoadError
|
|
8
|
+
# rubyzip is optional — only needed for ZipReader (.oxt dictionaries).
|
|
9
|
+
# The plain FileReader and StreamReader work without it.
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
module Kotoshu
|
|
13
|
+
module Readers
|
|
14
|
+
# Base reader class for reading files line by line.
|
|
15
|
+
#
|
|
16
|
+
# This class provides:
|
|
17
|
+
# - Line-by-line reading with line numbers
|
|
18
|
+
# - BOM (byte-order mark) handling
|
|
19
|
+
# - Comment stripping
|
|
20
|
+
# - Empty line filtering
|
|
21
|
+
#
|
|
22
|
+
# @example Basic usage
|
|
23
|
+
# reader = FileReader.new('file.aff', 'UTF-8')
|
|
24
|
+
# reader.each do |line_no, line|
|
|
25
|
+
# puts "#{line_no}: #{line}"
|
|
26
|
+
# end
|
|
27
|
+
class FileReader
|
|
28
|
+
# @return [String] The file path
|
|
29
|
+
attr_reader :path
|
|
30
|
+
|
|
31
|
+
# @return [String] The encoding
|
|
32
|
+
attr_reader :encoding
|
|
33
|
+
|
|
34
|
+
# @return [Integer] Current line number
|
|
35
|
+
attr_reader :line_no
|
|
36
|
+
|
|
37
|
+
# BOM (byte-order mark) for UTF-8
|
|
38
|
+
UTF8_BOM = "\xEF\xBB\xBF"
|
|
39
|
+
|
|
40
|
+
# Create a new file reader.
|
|
41
|
+
#
|
|
42
|
+
# @param path [String] Path to the file
|
|
43
|
+
# @param encoding [String] File encoding (default: 'UTF-8')
|
|
44
|
+
def initialize(path, encoding = 'UTF-8')
|
|
45
|
+
@path = path
|
|
46
|
+
@encoding = encoding
|
|
47
|
+
@line_no = 0
|
|
48
|
+
@file = nil
|
|
49
|
+
@iterator = nil
|
|
50
|
+
reset_io
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Reset encoding and reopen file.
|
|
54
|
+
#
|
|
55
|
+
# @param new_encoding [String] New encoding
|
|
56
|
+
def reset_encoding(new_encoding)
|
|
57
|
+
@encoding = new_encoding
|
|
58
|
+
@line_no = 0
|
|
59
|
+
@file&.close
|
|
60
|
+
reset_io
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Iterate over lines.
|
|
64
|
+
#
|
|
65
|
+
# @yield [Integer, String] Line number and line content
|
|
66
|
+
# @return [Enumerator] If no block given
|
|
67
|
+
def each(&)
|
|
68
|
+
return enum_for(:each) unless block_given?
|
|
69
|
+
|
|
70
|
+
@iterator.each(&)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Get all lines as an array.
|
|
74
|
+
#
|
|
75
|
+
# @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
|
|
76
|
+
def to_a
|
|
77
|
+
@iterator.to_a
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Check if there are more lines.
|
|
81
|
+
#
|
|
82
|
+
# @return [Boolean] True if there are more lines
|
|
83
|
+
def has_next?
|
|
84
|
+
peek
|
|
85
|
+
true
|
|
86
|
+
rescue StopIteration
|
|
87
|
+
false
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Peek at next line without consuming it.
|
|
91
|
+
#
|
|
92
|
+
# @return [Array<Integer, String>] Next line number and content
|
|
93
|
+
def peek
|
|
94
|
+
@iterator.peek
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Get next line.
|
|
98
|
+
#
|
|
99
|
+
# @return [Array<Integer, String>] Line number and content
|
|
100
|
+
def next
|
|
101
|
+
@iterator.next
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Reset the reader to the beginning.
|
|
105
|
+
def reset
|
|
106
|
+
@line_no = 0
|
|
107
|
+
reset_io
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Close the file.
|
|
111
|
+
def close
|
|
112
|
+
@file&.close
|
|
113
|
+
@file = nil
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# Reset the IO object.
|
|
119
|
+
def reset_io
|
|
120
|
+
@file = File.open(@path, "rb")
|
|
121
|
+
@iterator = read_lines.lazy
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Read lines from the file.
|
|
125
|
+
#
|
|
126
|
+
# Reads as binary and normalizes each line to UTF-8. The
|
|
127
|
+
# `@encoding` attribute holds the *source* encoding declared via the
|
|
128
|
+
# SET directive or auto-detected (Hunspell historically defaulted to
|
|
129
|
+
# ISO-8859-1 when no SET line was present); every line is transcoded
|
|
130
|
+
# to UTF-8 so downstream code (string metrics, regexps, suggestions)
|
|
131
|
+
# never has to think about encoding.
|
|
132
|
+
#
|
|
133
|
+
# @return [Enumerator] Enumerator of [line_no, line] pairs
|
|
134
|
+
def read_lines
|
|
135
|
+
return enum_for(:read_lines) unless block_given?
|
|
136
|
+
|
|
137
|
+
@file.each_line do |raw|
|
|
138
|
+
@line_no += 1
|
|
139
|
+
|
|
140
|
+
line = decode_line(raw)
|
|
141
|
+
line = line.strip
|
|
142
|
+
|
|
143
|
+
# Skip empty lines
|
|
144
|
+
next if line.empty?
|
|
145
|
+
|
|
146
|
+
# Handle UTF-8 BOM on first line
|
|
147
|
+
if @line_no == 1 && line.start_with?(UTF8_BOM)
|
|
148
|
+
line = line[UTF8_BOM.length..]
|
|
149
|
+
line = line.strip if line
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Skip if line is now empty after processing
|
|
153
|
+
next if line.nil? || line.empty?
|
|
154
|
+
|
|
155
|
+
yield [@line_no, line]
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Decode a raw line of bytes to a UTF-8 String.
|
|
160
|
+
#
|
|
161
|
+
# Strategy:
|
|
162
|
+
# 1. Treat the bytes as the declared @encoding (typically UTF-8 or
|
|
163
|
+
# ISO-8859-1).
|
|
164
|
+
# 2. Transcode to UTF-8. ISO-8859-1 → UTF-8 is lossless (every byte
|
|
165
|
+
# 0x00..0xFF maps to a codepoint), so legacy Latin-1 files round-trip
|
|
166
|
+
# cleanly. For UTF-8 source data this is a no-op.
|
|
167
|
+
# 3. If the declared encoding doesn't fit the bytes (e.g., a `.dic` file
|
|
168
|
+
# is actually Latin-1 even though its sibling `.aff` declared UTF-8),
|
|
169
|
+
# reinterpret as ISO-8859-1. This matches Hunspell's tolerant
|
|
170
|
+
# behaviour: every byte 0x00..0xFF is a valid Latin-1 codepoint, so
|
|
171
|
+
# this never fails. As a last resort, scrub invalid bytes.
|
|
172
|
+
def decode_line(raw)
|
|
173
|
+
line = raw.dup.force_encoding(@encoding)
|
|
174
|
+
return line.encode("UTF-8") if line.valid_encoding?
|
|
175
|
+
|
|
176
|
+
latin1 = raw.dup.force_encoding("ISO-8859-1")
|
|
177
|
+
latin1.encode("UTF-8")
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# String reader for reading from a string.
|
|
182
|
+
#
|
|
183
|
+
# Useful for testing or when content is already in memory.
|
|
184
|
+
class StringReader < FileReader
|
|
185
|
+
# Create a new string reader.
|
|
186
|
+
#
|
|
187
|
+
# @param content [String] The content to read
|
|
188
|
+
# @param encoding [String] Encoding (default: 'UTF-8')
|
|
189
|
+
def initialize(content, encoding = 'UTF-8')
|
|
190
|
+
@content = content
|
|
191
|
+
@lines = content.split("\n", -1)
|
|
192
|
+
@index = 0
|
|
193
|
+
super(nil, encoding)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
private
|
|
197
|
+
|
|
198
|
+
def reset_io
|
|
199
|
+
@line_no = 0
|
|
200
|
+
@index = 0
|
|
201
|
+
@iterator = read_lines_iterator
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def read_lines_iterator
|
|
205
|
+
Enumerator.new do |yielder|
|
|
206
|
+
while @index < @lines.length
|
|
207
|
+
@line_no += 1
|
|
208
|
+
line = @lines[@index].strip
|
|
209
|
+
@index += 1
|
|
210
|
+
|
|
211
|
+
# Skip empty lines
|
|
212
|
+
next if line.empty?
|
|
213
|
+
|
|
214
|
+
# Handle UTF-8 BOM on first line
|
|
215
|
+
if @line_no == 1 && line.start_with?(UTF8_BOM)
|
|
216
|
+
line = line[UTF8_BOM.length..]
|
|
217
|
+
line = line.strip if line
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Skip if line is now empty after processing
|
|
221
|
+
next if line.nil? || line.empty?
|
|
222
|
+
|
|
223
|
+
yielder << [@line_no, line]
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Zip reader for reading files from zip archives.
|
|
230
|
+
#
|
|
231
|
+
# This class reads files from within zip archives, such as
|
|
232
|
+
# OpenOffice/LibreOffice extensions (.odt, .oxt).
|
|
233
|
+
#
|
|
234
|
+
# @example Reading from a zip archive
|
|
235
|
+
# zip = Zip::File.open('dictionary.oxt')
|
|
236
|
+
# reader = ZipReader.new(zip, 'en_US.aff', 'UTF-8')
|
|
237
|
+
# reader.each do |line_no, line|
|
|
238
|
+
# puts "#{line_no}: #{line}"
|
|
239
|
+
# end
|
|
240
|
+
class ZipReader
|
|
241
|
+
# @return [Zip::File] The zip file object
|
|
242
|
+
attr_reader :zipfile
|
|
243
|
+
|
|
244
|
+
# @return [String] The entry path within the zip
|
|
245
|
+
attr_reader :entry_path
|
|
246
|
+
|
|
247
|
+
# @return [String] The encoding
|
|
248
|
+
attr_reader :encoding
|
|
249
|
+
|
|
250
|
+
# @return [Integer] Current line number
|
|
251
|
+
attr_reader :line_no
|
|
252
|
+
|
|
253
|
+
# BOM (byte-order mark) for UTF-8
|
|
254
|
+
UTF8_BOM = "\xEF\xBB\xBF"
|
|
255
|
+
|
|
256
|
+
# Create a new zip reader.
|
|
257
|
+
#
|
|
258
|
+
# @param zipfile [Zip::File] The zip file object
|
|
259
|
+
# @param entry_path [String] Path to the entry within the zip
|
|
260
|
+
# @param encoding [String] File encoding (default: 'UTF-8')
|
|
261
|
+
def initialize(zipfile, entry_path, encoding = 'UTF-8')
|
|
262
|
+
@zipfile = zipfile
|
|
263
|
+
@entry_path = entry_path
|
|
264
|
+
@encoding = encoding
|
|
265
|
+
@line_no = 0
|
|
266
|
+
@entry = nil
|
|
267
|
+
@iterator = nil
|
|
268
|
+
reset_io
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Reset encoding and reopen zip entry.
|
|
272
|
+
#
|
|
273
|
+
# @param new_encoding [String] New encoding
|
|
274
|
+
def reset_encoding(new_encoding)
|
|
275
|
+
@encoding = new_encoding
|
|
276
|
+
@line_no = 0
|
|
277
|
+
@entry&.close
|
|
278
|
+
reset_io
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# Iterate over lines.
|
|
282
|
+
#
|
|
283
|
+
# @yield [Integer, String] Line number and line content
|
|
284
|
+
# @return [Enumerator] If no block given
|
|
285
|
+
def each(&)
|
|
286
|
+
return enum_for(:each) unless block_given?
|
|
287
|
+
|
|
288
|
+
@iterator.each(&)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Get all lines as an array.
|
|
292
|
+
#
|
|
293
|
+
# @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
|
|
294
|
+
def to_a
|
|
295
|
+
@iterator.to_a
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# Check if there are more lines.
|
|
299
|
+
#
|
|
300
|
+
# @return [Boolean] True if there are more lines
|
|
301
|
+
def has_next?
|
|
302
|
+
peek
|
|
303
|
+
true
|
|
304
|
+
rescue StopIteration
|
|
305
|
+
false
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Peek at next line without consuming it.
|
|
309
|
+
#
|
|
310
|
+
# @return [Array<Integer, String>] Next line number and content
|
|
311
|
+
def peek
|
|
312
|
+
@iterator.peek
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# Get next line.
|
|
316
|
+
#
|
|
317
|
+
# @return [Array<Integer, String>] Line number and content
|
|
318
|
+
def next
|
|
319
|
+
@iterator.next
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Reset the reader to the beginning.
|
|
323
|
+
def reset
|
|
324
|
+
@line_no = 0
|
|
325
|
+
reset_io
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# Close the zip entry.
|
|
329
|
+
def close
|
|
330
|
+
@entry&.close
|
|
331
|
+
@entry = nil
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
private
|
|
335
|
+
|
|
336
|
+
# Reset the IO object.
|
|
337
|
+
def reset_io
|
|
338
|
+
@entry = @zipfile.find_entry(@entry_path)
|
|
339
|
+
raise IOError, "Entry not found in zip: #{@entry_path}" unless @entry
|
|
340
|
+
|
|
341
|
+
# Read the entire entry content and decode it
|
|
342
|
+
content = @entry.get_input_stream.read
|
|
343
|
+
content = content.encode(@encoding, invalid: :replace, undef: :replace)
|
|
344
|
+
|
|
345
|
+
@lines = content.split("\n", -1)
|
|
346
|
+
@line_no = 0
|
|
347
|
+
@iterator = read_lines_from_zip.lazy
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Read lines from the zip entry.
|
|
351
|
+
#
|
|
352
|
+
# @return [Enumerator] Enumerator of [line_no, line] pairs
|
|
353
|
+
def read_lines_from_zip
|
|
354
|
+
return enum_for(:read_lines_from_zip) unless block_given?
|
|
355
|
+
|
|
356
|
+
@lines.each do |line|
|
|
357
|
+
@line_no += 1
|
|
358
|
+
line = line.strip
|
|
359
|
+
|
|
360
|
+
# Skip empty lines
|
|
361
|
+
next if line.empty?
|
|
362
|
+
|
|
363
|
+
# Handle UTF-8 BOM on first line
|
|
364
|
+
if @line_no == 1 && line.start_with?(UTF8_BOM)
|
|
365
|
+
line = line[UTF8_BOM.length..]
|
|
366
|
+
line = line.strip if line
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# Skip if line is now empty after processing
|
|
370
|
+
next if line.nil? || line.empty?
|
|
371
|
+
|
|
372
|
+
yield [@line_no, line]
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
end
|