kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,208 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Readers
5
+ # Word entry from the dictionary file.
6
+ #
7
+ # @attr stem [String] The word stem
8
+ # @attr flags [Set<String>] Morphological flags
9
+ # @attr morph_data [Array<String>] Morphological data fields (e.g. "ph:wich")
10
+ Word = Struct.new(:stem, :flags, :morph_data, keyword_init: true) do
11
+ # Create a word from a dictionary line.
12
+ #
13
+ # @param line [String] The dictionary line
14
+ # @param context [Hash] The reading context (for flag parsing)
15
+ # @return [Word] The parsed word
16
+ def self.from_line(line, context = {})
17
+ # Format: stem[/flags][<SEP><morph_data>]
18
+ # SEP is a tab per spec, but real-world fixtures (including the
19
+ # Hunspell ph.* tests) sometimes use spaces. We split on the first
20
+ # morphological token (key:value) regardless of separator.
21
+ head, morph = split_stem_and_morph(line)
22
+ head = head.strip
23
+
24
+ # Find the first UNESCAPED slash to split stem from flags.
25
+ # Hunspell allows `\/` to represent a literal `/` inside a word —
26
+ # so we can't just use String#index('/'). Mirrors Spylls's
27
+ # SLASH_REGEXP (a slash not preceded by a backslash).
28
+ #
29
+ # Special case: a word that STARTS with `/` is not an empty stem +
30
+ # flags — it's a word whose first character is `/`. Without this,
31
+ # dic entry `/` would be parsed as stem="" + flags="".
32
+ if head.start_with?('/')
33
+ stem = head
34
+ flags_str = nil
35
+ else
36
+ slash_idx = unescaped_slash_index(head)
37
+ if slash_idx
38
+ stem = head[0...slash_idx]
39
+ flags_str = head[(slash_idx + 1)..]
40
+ else
41
+ stem = head
42
+ flags_str = nil
43
+ end
44
+
45
+ # Replace escaped slashes in the stem: `\/` → `/`
46
+ stem = stem.gsub('\/', '/') if stem.include?('\/')
47
+ end
48
+
49
+ flags = if flags_str && !flags_str.empty? && context[:flag_format]
50
+ parse_flags(flags_str, context[:flag_format], context[:flag_synonyms])
51
+ elsif flags_str && !flags_str.empty?
52
+ flags_str.chars.to_set
53
+ else
54
+ Set.new
55
+ end
56
+
57
+ morph_data = parse_morph_data(morph)
58
+ new(stem:, flags:, morph_data:)
59
+ end
60
+
61
+ # Split a dictionary line into stem and morphological-data portions.
62
+ #
63
+ # Hunspell specifies tab-separated morph data, but the ph.* test
64
+ # fixtures use spaces. We honor both: if there's a tab, split there;
65
+ # otherwise split before the first `key:value` token (which is the
66
+ # universal signature of morphological data).
67
+ #
68
+ # @param line [String] The raw dictionary line
69
+ # @return [Array(String, String)] [stem portion, morph portion]
70
+ def self.split_stem_and_morph(line)
71
+ tab_idx = line.index("\t")
72
+ return line.split("\t", 2) if tab_idx
73
+
74
+ match = line.match(/(.*?)(\s+[a-zA-Z]+:[^\s].*)$/)
75
+ return [line, ''] unless match
76
+
77
+ [match[1], match[2]]
78
+ end
79
+
80
+ # Parse morphological data from the post-tab portion of a dic line.
81
+ #
82
+ # Hunspell morphological data is whitespace-separated key:value tokens
83
+ # such as `ph:wich` (phonetic), `st:foo` (stem), `po:noun` (part of
84
+ # speech). We preserve them as a list of raw strings — downstream
85
+ # consumers (e.g. PhonetSuggest via alt_spellings) extract what they
86
+ # need.
87
+ #
88
+ # @param morph [String, nil] The morphological portion
89
+ # @return [Array<String>] List of morphological tokens
90
+ def self.parse_morph_data(morph)
91
+ return [] if morph.nil?
92
+
93
+ morph.split(/\s+/).reject(&:empty?)
94
+ end
95
+
96
+ # Parse flags from string.
97
+ #
98
+ # @param string [String] Flag string
99
+ # @param flag_format [String] Flag format ('short', 'long', 'num', 'UTF-8')
100
+ # @param flag_synonyms [Hash] Flag synonyms map
101
+ # @return [Set<String>] Parsed flags
102
+ def self.parse_flags(string, flag_format, flag_synonyms = {})
103
+ return Set.new if string.nil? || string.empty?
104
+
105
+ # AF (flag aliases) only applies when aliases are actually defined
106
+ # and the string is a positional alias index (pure digits). Without
107
+ # the !empty? guard, a `FLAG num` dictionary with no AF would have
108
+ # every numeric flag collapsed to the empty set.
109
+ if flag_synonyms.is_a?(Hash) && !flag_synonyms.empty? && string =~ /^\d+$/
110
+ return flag_synonyms[string] || Set.new
111
+ end
112
+
113
+ case flag_format
114
+ when 'short'
115
+ string.chars.to_set
116
+ when 'long'
117
+ string.scan(/../).to_set
118
+ when 'num'
119
+ string.scan(/\d+/).to_set
120
+ when 'UTF-8'
121
+ string.chars.to_set
122
+ else
123
+ string.chars.to_set
124
+ end
125
+ end
126
+
127
+ # Find the index of the first unescaped slash in the string.
128
+ #
129
+ # @param str [String] Input string
130
+ # @return [Integer, nil] Index of first unescaped `/`, or nil
131
+ def self.unescaped_slash_index(str)
132
+ i = 0
133
+ while i < str.length
134
+ c = str[i]
135
+ return i if c == '/' && (i.zero? || str[i - 1] != '\\')
136
+
137
+ i += 1
138
+ end
139
+ nil
140
+ end
141
+ end
142
+
143
+ # DIC file reader for Hunspell dictionary files.
144
+ #
145
+ # This class reads .dic files and creates a list of Word entries.
146
+ #
147
+ # @example Reading a dic file
148
+ # reader = DicReader.new('en_US.dic', flag_format: 'short')
149
+ # words = reader.read
150
+ class DicReader
151
+ attr_reader :path, :encoding, :flag_format, :flag_synonyms
152
+
153
+ # Create a new DIC reader.
154
+ #
155
+ # @param path [String] Path to the .dic file
156
+ # @param encoding [String] File encoding (default: 'UTF-8')
157
+ # @param flag_format [String] Flag format ('short', 'long', 'num', 'UTF-8')
158
+ # @param flag_synonyms [Hash] Flag synonyms map
159
+ def initialize(path, encoding: 'UTF-8', flag_format: 'short', flag_synonyms: {})
160
+ @path = path
161
+ @encoding = encoding
162
+ @flag_format = flag_format
163
+ @flag_synonyms = flag_synonyms
164
+ end
165
+
166
+ # Read the dic file and return a list of Word entries.
167
+ #
168
+ # @param source [FileReader, nil] Optional file reader to use instead of creating a new one
169
+ # @return [Array<Word>] List of word entries
170
+ def read(source = nil)
171
+ owned = source.nil?
172
+ reader = source || FileReader.new(@path, @encoding)
173
+
174
+ begin
175
+ words = []
176
+ first_line = true
177
+ expected_count = 0
178
+
179
+ reader.each do |_line_no, line|
180
+ if first_line
181
+ # First line is word count
182
+ expected_count = line.to_i
183
+ first_line = false
184
+ next
185
+ end
186
+
187
+ # Skip empty lines
188
+ next if line.empty?
189
+
190
+ # Parse word
191
+ word = Word.from_line(line, flag_format: @flag_format, flag_synonyms: @flag_synonyms)
192
+ words << word
193
+ end
194
+
195
+ # Verify word count
196
+ # Note: We don't raise an error if count doesn't match, as some dictionaries have different formats
197
+
198
+ words
199
+ ensure
200
+ # Close the reader only when we created it. If the caller passed
201
+ # one in, they own its lifecycle. Without this, the underlying
202
+ # File handle leaks (on Windows it also blocks tempfile cleanup).
203
+ reader.close if owned
204
+ end
205
+ end
206
+ end
207
+ end
208
+ end
@@ -0,0 +1,377 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rubygems'
4
+
5
+ begin
6
+ require 'zip'
7
+ rescue LoadError
8
+ # rubyzip is optional — only needed for ZipReader (.oxt dictionaries).
9
+ # The plain FileReader and StreamReader work without it.
10
+ end
11
+
12
+ module Kotoshu
13
+ module Readers
14
+ # Base reader class for reading files line by line.
15
+ #
16
+ # This class provides:
17
+ # - Line-by-line reading with line numbers
18
+ # - BOM (byte-order mark) handling
19
+ # - Comment stripping
20
+ # - Empty line filtering
21
+ #
22
+ # @example Basic usage
23
+ # reader = FileReader.new('file.aff', 'UTF-8')
24
+ # reader.each do |line_no, line|
25
+ # puts "#{line_no}: #{line}"
26
+ # end
27
+ class FileReader
28
+ # @return [String] The file path
29
+ attr_reader :path
30
+
31
+ # @return [String] The encoding
32
+ attr_reader :encoding
33
+
34
+ # @return [Integer] Current line number
35
+ attr_reader :line_no
36
+
37
+ # BOM (byte-order mark) for UTF-8
38
+ UTF8_BOM = "\xEF\xBB\xBF"
39
+
40
+ # Create a new file reader.
41
+ #
42
+ # @param path [String] Path to the file
43
+ # @param encoding [String] File encoding (default: 'UTF-8')
44
+ def initialize(path, encoding = 'UTF-8')
45
+ @path = path
46
+ @encoding = encoding
47
+ @line_no = 0
48
+ @file = nil
49
+ @iterator = nil
50
+ reset_io
51
+ end
52
+
53
+ # Reset encoding and reopen file.
54
+ #
55
+ # @param new_encoding [String] New encoding
56
+ def reset_encoding(new_encoding)
57
+ @encoding = new_encoding
58
+ @line_no = 0
59
+ @file&.close
60
+ reset_io
61
+ end
62
+
63
+ # Iterate over lines.
64
+ #
65
+ # @yield [Integer, String] Line number and line content
66
+ # @return [Enumerator] If no block given
67
+ def each(&)
68
+ return enum_for(:each) unless block_given?
69
+
70
+ @iterator.each(&)
71
+ end
72
+
73
+ # Get all lines as an array.
74
+ #
75
+ # @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
76
+ def to_a
77
+ @iterator.to_a
78
+ end
79
+
80
+ # Check if there are more lines.
81
+ #
82
+ # @return [Boolean] True if there are more lines
83
+ def has_next?
84
+ peek
85
+ true
86
+ rescue StopIteration
87
+ false
88
+ end
89
+
90
+ # Peek at next line without consuming it.
91
+ #
92
+ # @return [Array<Integer, String>] Next line number and content
93
+ def peek
94
+ @iterator.peek
95
+ end
96
+
97
+ # Get next line.
98
+ #
99
+ # @return [Array<Integer, String>] Line number and content
100
+ def next
101
+ @iterator.next
102
+ end
103
+
104
+ # Reset the reader to the beginning.
105
+ def reset
106
+ @line_no = 0
107
+ reset_io
108
+ end
109
+
110
+ # Close the file.
111
+ def close
112
+ @file&.close
113
+ @file = nil
114
+ end
115
+
116
+ private
117
+
118
+ # Reset the IO object.
119
+ def reset_io
120
+ @file = File.open(@path, "rb")
121
+ @iterator = read_lines.lazy
122
+ end
123
+
124
+ # Read lines from the file.
125
+ #
126
+ # Reads as binary and normalizes each line to UTF-8. The
127
+ # `@encoding` attribute holds the *source* encoding declared via the
128
+ # SET directive or auto-detected (Hunspell historically defaulted to
129
+ # ISO-8859-1 when no SET line was present); every line is transcoded
130
+ # to UTF-8 so downstream code (string metrics, regexps, suggestions)
131
+ # never has to think about encoding.
132
+ #
133
+ # @return [Enumerator] Enumerator of [line_no, line] pairs
134
+ def read_lines
135
+ return enum_for(:read_lines) unless block_given?
136
+
137
+ @file.each_line do |raw|
138
+ @line_no += 1
139
+
140
+ line = decode_line(raw)
141
+ line = line.strip
142
+
143
+ # Skip empty lines
144
+ next if line.empty?
145
+
146
+ # Handle UTF-8 BOM on first line
147
+ if @line_no == 1 && line.start_with?(UTF8_BOM)
148
+ line = line[UTF8_BOM.length..]
149
+ line = line.strip if line
150
+ end
151
+
152
+ # Skip if line is now empty after processing
153
+ next if line.nil? || line.empty?
154
+
155
+ yield [@line_no, line]
156
+ end
157
+ end
158
+
159
+ # Decode a raw line of bytes to a UTF-8 String.
160
+ #
161
+ # Strategy:
162
+ # 1. Treat the bytes as the declared @encoding (typically UTF-8 or
163
+ # ISO-8859-1).
164
+ # 2. Transcode to UTF-8. ISO-8859-1 → UTF-8 is lossless (every byte
165
+ # 0x00..0xFF maps to a codepoint), so legacy Latin-1 files round-trip
166
+ # cleanly. For UTF-8 source data this is a no-op.
167
+ # 3. If the declared encoding doesn't fit the bytes (e.g., a `.dic` file
168
+ # is actually Latin-1 even though its sibling `.aff` declared UTF-8),
169
+ # reinterpret as ISO-8859-1. This matches Hunspell's tolerant
170
+ # behaviour: every byte 0x00..0xFF is a valid Latin-1 codepoint, so
171
+ # this never fails. As a last resort, scrub invalid bytes.
172
+ def decode_line(raw)
173
+ line = raw.dup.force_encoding(@encoding)
174
+ return line.encode("UTF-8") if line.valid_encoding?
175
+
176
+ latin1 = raw.dup.force_encoding("ISO-8859-1")
177
+ latin1.encode("UTF-8")
178
+ end
179
+ end
180
+
181
+ # String reader for reading from a string.
182
+ #
183
+ # Useful for testing or when content is already in memory.
184
+ class StringReader < FileReader
185
+ # Create a new string reader.
186
+ #
187
+ # @param content [String] The content to read
188
+ # @param encoding [String] Encoding (default: 'UTF-8')
189
+ def initialize(content, encoding = 'UTF-8')
190
+ @content = content
191
+ @lines = content.split("\n", -1)
192
+ @index = 0
193
+ super(nil, encoding)
194
+ end
195
+
196
+ private
197
+
198
+ def reset_io
199
+ @line_no = 0
200
+ @index = 0
201
+ @iterator = read_lines_iterator
202
+ end
203
+
204
+ def read_lines_iterator
205
+ Enumerator.new do |yielder|
206
+ while @index < @lines.length
207
+ @line_no += 1
208
+ line = @lines[@index].strip
209
+ @index += 1
210
+
211
+ # Skip empty lines
212
+ next if line.empty?
213
+
214
+ # Handle UTF-8 BOM on first line
215
+ if @line_no == 1 && line.start_with?(UTF8_BOM)
216
+ line = line[UTF8_BOM.length..]
217
+ line = line.strip if line
218
+ end
219
+
220
+ # Skip if line is now empty after processing
221
+ next if line.nil? || line.empty?
222
+
223
+ yielder << [@line_no, line]
224
+ end
225
+ end
226
+ end
227
+ end
228
+
229
+ # Zip reader for reading files from zip archives.
230
+ #
231
+ # This class reads files from within zip archives, such as
232
+ # OpenOffice/LibreOffice extensions (.odt, .oxt).
233
+ #
234
+ # @example Reading from a zip archive
235
+ # zip = Zip::File.open('dictionary.oxt')
236
+ # reader = ZipReader.new(zip, 'en_US.aff', 'UTF-8')
237
+ # reader.each do |line_no, line|
238
+ # puts "#{line_no}: #{line}"
239
+ # end
240
+ class ZipReader
241
+ # @return [Zip::File] The zip file object
242
+ attr_reader :zipfile
243
+
244
+ # @return [String] The entry path within the zip
245
+ attr_reader :entry_path
246
+
247
+ # @return [String] The encoding
248
+ attr_reader :encoding
249
+
250
+ # @return [Integer] Current line number
251
+ attr_reader :line_no
252
+
253
+ # BOM (byte-order mark) for UTF-8
254
+ UTF8_BOM = "\xEF\xBB\xBF"
255
+
256
+ # Create a new zip reader.
257
+ #
258
+ # @param zipfile [Zip::File] The zip file object
259
+ # @param entry_path [String] Path to the entry within the zip
260
+ # @param encoding [String] File encoding (default: 'UTF-8')
261
+ def initialize(zipfile, entry_path, encoding = 'UTF-8')
262
+ @zipfile = zipfile
263
+ @entry_path = entry_path
264
+ @encoding = encoding
265
+ @line_no = 0
266
+ @entry = nil
267
+ @iterator = nil
268
+ reset_io
269
+ end
270
+
271
+ # Reset encoding and reopen zip entry.
272
+ #
273
+ # @param new_encoding [String] New encoding
274
+ def reset_encoding(new_encoding)
275
+ @encoding = new_encoding
276
+ @line_no = 0
277
+ @entry&.close
278
+ reset_io
279
+ end
280
+
281
+ # Iterate over lines.
282
+ #
283
+ # @yield [Integer, String] Line number and line content
284
+ # @return [Enumerator] If no block given
285
+ def each(&)
286
+ return enum_for(:each) unless block_given?
287
+
288
+ @iterator.each(&)
289
+ end
290
+
291
+ # Get all lines as an array.
292
+ #
293
+ # @return [Array<Array<Integer, String>>] Array of [line_no, line] pairs
294
+ def to_a
295
+ @iterator.to_a
296
+ end
297
+
298
+ # Check if there are more lines.
299
+ #
300
+ # @return [Boolean] True if there are more lines
301
+ def has_next?
302
+ peek
303
+ true
304
+ rescue StopIteration
305
+ false
306
+ end
307
+
308
+ # Peek at next line without consuming it.
309
+ #
310
+ # @return [Array<Integer, String>] Next line number and content
311
+ def peek
312
+ @iterator.peek
313
+ end
314
+
315
+ # Get next line.
316
+ #
317
+ # @return [Array<Integer, String>] Line number and content
318
+ def next
319
+ @iterator.next
320
+ end
321
+
322
+ # Reset the reader to the beginning.
323
+ def reset
324
+ @line_no = 0
325
+ reset_io
326
+ end
327
+
328
+ # Close the zip entry.
329
+ def close
330
+ @entry&.close
331
+ @entry = nil
332
+ end
333
+
334
+ private
335
+
336
+ # Reset the IO object.
337
+ def reset_io
338
+ @entry = @zipfile.find_entry(@entry_path)
339
+ raise IOError, "Entry not found in zip: #{@entry_path}" unless @entry
340
+
341
+ # Read the entire entry content and decode it
342
+ content = @entry.get_input_stream.read
343
+ content = content.encode(@encoding, invalid: :replace, undef: :replace)
344
+
345
+ @lines = content.split("\n", -1)
346
+ @line_no = 0
347
+ @iterator = read_lines_from_zip.lazy
348
+ end
349
+
350
+ # Read lines from the zip entry.
351
+ #
352
+ # @return [Enumerator] Enumerator of [line_no, line] pairs
353
+ def read_lines_from_zip
354
+ return enum_for(:read_lines_from_zip) unless block_given?
355
+
356
+ @lines.each do |line|
357
+ @line_no += 1
358
+ line = line.strip
359
+
360
+ # Skip empty lines
361
+ next if line.empty?
362
+
363
+ # Handle UTF-8 BOM on first line
364
+ if @line_no == 1 && line.start_with?(UTF8_BOM)
365
+ line = line[UTF8_BOM.length..]
366
+ line = line.strip if line
367
+ end
368
+
369
+ # Skip if line is now empty after processing
370
+ next if line.nil? || line.empty?
371
+
372
+ yield [@line_no, line]
373
+ end
374
+ end
375
+ end
376
+ end
377
+ end