kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,60 @@
1
+ # frozen_string: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Nepali language implementation (plan 108).
6
+ #
7
+ # Registers +ne+ and +ne-NP+. The staged ne dictionary (LibreOffice
8
+ # ne_NP) serves checking through the standard Hunspell engine;
9
+ # this module contributes the language wiring the basic tier
10
+ # lacks: the Devanagari tokenizer that keeps matras and virama
11
+ # conjuncts attached to their base consonant during word
12
+ # extraction, and the Devanagari InScript grid (the Indian
13
+ # national standard layout, mirrored from the models repo eval
14
+ # harness) for typo-proximity suggestion ranking. The Nepali
15
+ # Traditional Romanized layout is more common on the ground, but
16
+ # it is not a documented standard the way InScript is — the same
17
+ # choice the eval harness made.
18
+ #
19
+ # @example
20
+ # lang = Kotoshu::Languages::Nepali.new
21
+ # lang.script_type # => :devanagari
22
+ # lang.rtl? # => false
23
+ class Nepali < Language::Base
24
+ register "ne"
25
+ register "ne-NP"
26
+
27
+ def initialize(code: "ne", name: "Nepali", variant: nil)
28
+ super
29
+ end
30
+
31
+ def description
32
+ name
33
+ end
34
+
35
+ def tokenizer
36
+ @tokenizer ||= Language::Tokenizer::DevanagariTokenizer.new
37
+ end
38
+
39
+ def normalizer
40
+ @normalizer ||= Language::Normalizer::Base.new
41
+ end
42
+
43
+ def dictionary_class
44
+ Dictionary::Custom
45
+ end
46
+
47
+ def default_dictionary_paths
48
+ ["/usr/share/dict/nepali"]
49
+ end
50
+
51
+ def script_type
52
+ :devanagari
53
+ end
54
+
55
+ def rtl?
56
+ false
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Dutch language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Dutch loanword accents (é ï ö ü) are dead-key sequences on the US physical layout.
9
+ class Dutch < LatinBase
10
+ register "nl"
11
+ register "nl-NL"
12
+
13
+ def initialize(code: "nl", name: "Dutch")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/dutch"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Norwegian Nynorsk (plan 107): the second Norwegian written
6
+ # standard, wired while basic support opened the staged manifest —
7
+ # dictionary and model are both staged (`nn` in the dictionaries
8
+ # repo, `nn` in the models registry), so Nynorsk users get the
9
+ # full-feature tier rather than the script fallback. Thin
10
+ # composition like its Bokmål sibling — shared Latin tokenizer,
11
+ # base normalizer, registry entry; å æ ø are real keys on the
12
+ # Nordic layout the nb module resolves to.
13
+ class NorwegianNynorsk < LatinBase
14
+ register "nn"
15
+ register "nn-NO"
16
+
17
+ def initialize(code: "nn", name: "Norwegian Nynorsk")
18
+ super
19
+ end
20
+
21
+ def default_dictionary_paths
22
+ ["/usr/share/dict/norwegian"]
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Polish language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Polish diacritics (ą ć ę ł ń ó ś ź ż) are AltGr sequences on the programmers layout.
9
+ class Polish < LatinBase
10
+ register "pl"
11
+ register "pl-PL"
12
+
13
+ def initialize(code: "pl", name: "Polish")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/polish"]
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,439 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Portuguese language implementation.
6
+ #
7
+ # Supports multiple dialects: pt-BR, pt-PT, pt-AO, pt-MZ, pt-GW, pt-CV
8
+ #
9
+ # Full Hunspell integration with spell checking, POS tagging, and grammar rules
10
+ # specifically handling Portuguese accents and Brazilian vs European differences.
11
+ class Portuguese < Language::Base
12
+ # Portuguese spell checker with Hunspell integration.
13
+ class SpellChecker < Components::SpellChecker
14
+ attr_reader :aff_path, :dic_path, :script
15
+
16
+ # Portuguese-specific character substitutions
17
+ PORTUGUESE_SUBSTITUTIONS = {
18
+ 'á' => %w[a],
19
+ 'â' => %w[a],
20
+ 'ã' => %w[a],
21
+ 'à' => %w[a],
22
+ 'é' => %w[e],
23
+ 'ê' => %w[e],
24
+ 'í' => %w[i],
25
+ 'ó' => %w[o],
26
+ 'ô' => %w[o],
27
+ 'õ' => %w[o],
28
+ 'ú' => %w[u],
29
+ 'ü' => %w[u],
30
+ 'ç' => %w[c],
31
+ }.freeze
32
+
33
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8')
34
+ @aff_path = aff_path
35
+ @dic_path = dic_path
36
+ @script = script
37
+ @encoding = encoding
38
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
39
+ end
40
+
41
+ def check(word)
42
+ return { found: false, stem: nil, flags: [] } if word.nil? || word.empty?
43
+
44
+ first_form = @lookuper.good_forms(word).first
45
+ if first_form
46
+ { found: true, stem: first_form.stem || word, flags: first_form.flags&.to_a || [] }
47
+ else
48
+ { found: false, stem: nil, flags: [] }
49
+ end
50
+ end
51
+
52
+ def suggest(word, max_suggestions: 10)
53
+ return [] if word.nil? || word.empty?
54
+
55
+ first_form = @lookuper.good_forms(word).first
56
+ return [] if first_form
57
+
58
+ generate_suggestions(word, max_suggestions).take(max_suggestions)
59
+ end
60
+
61
+ def correct?(word)
62
+ check(word)[:found]
63
+ end
64
+
65
+ def lookuper
66
+ @lookuper
67
+ end
68
+
69
+ private
70
+
71
+ def calculate_distance(a, b)
72
+ return a.length if b.empty?
73
+ return b.length if a.empty?
74
+
75
+ matrix = Array.new(a.length + 1) { |i| [i] + ([0] * b.length) }
76
+ (1..b.length).each { |j| matrix[0][j] = j }
77
+ (1..a.length).each do |i|
78
+ (1..b.length).each do |j|
79
+ cost = a[i - 1] == b[j - 1] ? 0 : 1
80
+ matrix[i][j] = [matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost].min
81
+ end
82
+ end
83
+ matrix[a.length][b.length]
84
+ end
85
+
86
+ def calculate_score(original, suggestion, rank)
87
+ distance = calculate_distance(original, suggestion)
88
+ max_len = [original.length, suggestion.length].max
89
+ distance_score = 1.0 - (distance.to_f / max_len)
90
+ rank_penalty = rank * 0.05
91
+ [distance_score - rank_penalty, 0.0].max
92
+ end
93
+
94
+ def generate_suggestions(word, _max_suggestions)
95
+ variations = []
96
+
97
+ # Missing accents
98
+ word.downcase.chars.each_with_index do |char, i|
99
+ PORTUGUESE_SUBSTITUTIONS.each do |accented, variants|
100
+ variants.each do |variant|
101
+ if char == variant
102
+ accented_word = word.dup
103
+ accented_word[i] = accented
104
+ variations << accented_word if @lookuper.good_forms(accented_word).first
105
+ end
106
+ end
107
+ end
108
+ end
109
+
110
+ # Common substitutions
111
+ word.chars.each_with_index do |char, i|
112
+ next unless PORTUGUESE_SUBSTITUTIONS.key?(char.downcase)
113
+
114
+ PORTUGUESE_SUBSTITUTIONS[char.downcase].each do |sub|
115
+ substituted = word.dup
116
+ substituted[i] = sub
117
+ variations << substituted if @lookuper.good_forms(substituted).first
118
+ end
119
+ end
120
+
121
+ # Doubled and deleted letters
122
+ word.chars.each_with_index do |char, i|
123
+ next if i == 0
124
+
125
+ doubled = word.dup
126
+ doubled.insert(i, char)
127
+ variations << doubled if @lookuper.good_forms(doubled).first
128
+ end
129
+
130
+ (0...word.length).each do |i|
131
+ deleted = word.dup
132
+ deleted.slice!(i)
133
+ next if deleted.empty?
134
+
135
+ variations << deleted if @lookuper.good_forms(deleted).first
136
+ end
137
+
138
+ variations.uniq!
139
+ variations.map do |suggestion|
140
+ { word: suggestion, distance: calculate_distance(word, suggestion),
141
+ score: calculate_score(word, suggestion, 0) }
142
+ end.sort_by { |s| s[:distance] }
143
+ end
144
+ end
145
+
146
+ # Portuguese tokenizer with number and date handling.
147
+ class Tokenizer < Language::Tokenizer::PortugueseTokenizer
148
+ end
149
+
150
+ # Portuguese POS tagger.
151
+ class POSTagger < Components::PosTagger
152
+ FLAG_TO_POS = {
153
+ 'N' => 'NOUN', 'NN' => 'NOUN', 'NNS' => 'NOUN', 'NNP' => 'NOUN_PROPER',
154
+ 'V' => 'VERB', 'VB' => 'VERB', 'VBD' => 'VERB', 'VBG' => 'VERB', 'VBN' => 'VERB',
155
+ 'VBP' => 'VERB', 'VBZ' => 'VERB',
156
+ 'A' => 'ADJ', 'JJ' => 'ADJ', 'JJR' => 'ADJ', 'JJS' => 'ADJ',
157
+ 'R' => 'ADV', 'RB' => 'ADV', 'RBR' => 'ADV', 'RBS' => 'ADV',
158
+ 'D' => 'DET', 'DT' => 'DET', 'PDT' => 'DET',
159
+ 'P' => 'PRON', 'PP' => 'PRON', 'PRP' => 'PRON', 'PRP$' => 'PRON_POSS',
160
+ 'WP' => 'PRON', 'WP$' => 'PRON_POSS',
161
+ 'I' => 'PREP', 'IN' => 'PREP',
162
+ 'C' => 'CONJ', 'CC' => 'CONJ',
163
+ 'U' => 'PART', 'RP' => 'PART',
164
+ 'INTJ' => 'INTJ', 'UH' => 'INTJ',
165
+ 'CD' => 'NUM',
166
+ 'FW' => 'X',
167
+ 'PUNCT' => 'PUNCT', '.' => 'PUNCT', ',' => 'PUNCT', '!' => 'PUNCT',
168
+ '?' => 'PUNCT', ';' => 'PUNCT', ':' => 'PUNCT'
169
+ }.freeze
170
+
171
+ attr_reader :aff_path, :dic_path, :script
172
+
173
+ def initialize(aff_path:, dic_path:, script: :latin, encoding: 'UTF-8', flag_mapping: FLAG_TO_POS)
174
+ @aff_path = aff_path
175
+ @dic_path = dic_path
176
+ @script = script
177
+ @encoding = encoding
178
+ @flag_mapping = flag_mapping
179
+ @lookuper = Readers::LookupBuilder.new(aff_path, dic_path, encoding: encoding, script: script).build
180
+ @lookup_cache = {}
181
+ end
182
+
183
+ def tag(tokens)
184
+ return [] if tokens.nil? || tokens.empty?
185
+
186
+ tokens.map do |token|
187
+ word = token[:token]
188
+ if word.nil? || word.empty?
189
+ token.merge(pos_tag: nil, lemma: nil)
190
+ else
191
+ lookup_result = lookup_with_pos(word)
192
+ token.merge(pos_tag: lookup_result[:pos_tag], lemma: lookup_result[:lemma] || word)
193
+ end
194
+ end
195
+ end
196
+
197
+ def flag_mapping
198
+ @flag_mapping
199
+ end
200
+
201
+ def flag_mapping=(mapping)
202
+ @flag_mapping = mapping
203
+ end
204
+
205
+ def clear_cache
206
+ @lookup_cache.clear
207
+ end
208
+
209
+ def cache_size
210
+ @lookup_cache.size
211
+ end
212
+
213
+ private
214
+
215
+ def lookup_with_pos(word)
216
+ return { pos_tag: nil, lemma: nil } if word.nil? || word.empty?
217
+ return @lookup_cache[word] if @lookup_cache.key?(word)
218
+
219
+ first_form = @lookuper.good_forms(word).first
220
+ pos_tag = derive_pos_tag(first_form)
221
+ cache_result = { pos_tag: pos_tag, lemma: first_form&.stem }
222
+ @lookup_cache[word] = cache_result
223
+ cache_result
224
+ end
225
+
226
+ def derive_pos_tag(result)
227
+ return nil unless result
228
+
229
+ flags = result.flags&.to_a || []
230
+ return guess_pos_from_affix(result) if flags.empty?
231
+
232
+ flags.each do |flag|
233
+ pos_tag = flag_to_pos(flag)
234
+ return pos_tag if pos_tag
235
+ end
236
+ guess_pos_from_affix(result)
237
+ end
238
+
239
+ def flag_to_pos(flag)
240
+ return @flag_mapping[flag] if @flag_mapping.key?(flag)
241
+
242
+ first_char = flag[0]
243
+ @flag_mapping[first_char]
244
+ end
245
+
246
+ def guess_pos_from_affix(result)
247
+ suffix = result.suffix
248
+ return guess_pos_from_suffix(suffix) if suffix
249
+
250
+ nil
251
+ end
252
+
253
+ def guess_pos_from_suffix(suffix)
254
+ return 'VERB' if suffix.match?(/^(ar|er|ir|ando|endo|indo|ado|ido)$/)
255
+ return 'ADV' if suffix.end_with?('mente')
256
+ return 'NOUN' if suffix.match?(/^(ção|são|mento|dade|eza|ismo|ista|or|nte)$/)
257
+ return 'ADJ' if suffix.match?(/%(oso|ável|ível|ico|ica|ante)$/)
258
+
259
+ nil
260
+ end
261
+ end
262
+
263
+ # Portuguese grammar rules module.
264
+ module GrammarRules
265
+ class Rule
266
+ attr_reader :id, :name, :description
267
+
268
+ def initialize(id, name, description)
269
+ @id = id
270
+ @name = name
271
+ @description = description
272
+ end
273
+
274
+ def check(tokens)
275
+ raise NotImplementedError, "#{self.class} must implement #check"
276
+ end
277
+ end
278
+
279
+ # Rule: Personal infinitive agreement
280
+ class PersonalInfinitiveRule < Rule
281
+ def initialize
282
+ super('PT_PERSONAL_INFINITIVE', 'Personal Infinitive', 'Personal infinitive must agree with the subject.')
283
+ end
284
+
285
+ def check(_tokens)
286
+ # Simplified implementation
287
+ []
288
+ end
289
+ end
290
+
291
+ # Rule: Crase (à vs a)
292
+ class CraseRule < Rule
293
+ def initialize
294
+ super('PT_CRASE', 'Crase Usage', 'Use crase (à) before feminine nouns indicating place/time.')
295
+ end
296
+
297
+ def check(tokens)
298
+ errors = []
299
+ tokens.each_cons(2) do |prev_token, current_token|
300
+ prev_word = prev_token[:token]&.downcase
301
+ next unless %w[a ema].include?(prev_word)
302
+
303
+ # Check if next word starts with 'a' sound and is feminine
304
+ next_word = current_token[:token]
305
+ next if next_word.nil? || next_word.empty?
306
+
307
+ if next_word&.match?(/^[aáãâä]/i)
308
+ # Suggest using crase
309
+ errors << {
310
+ rule_id: @id,
311
+ position: prev_token[:position],
312
+ message: "Possible crase usage needed: '#{prev_word}' -> 'à'",
313
+ suggestion: 'à',
314
+ context: "#{prev_word} #{next_word}",
315
+ suggestions: ['à']
316
+ }
317
+ end
318
+ end
319
+ errors
320
+ end
321
+ end
322
+
323
+ class RuleRegistry
324
+ class << self
325
+ def default_rules
326
+ [PersonalInfinitiveRule.new, CraseRule.new]
327
+ end
328
+
329
+ def get_rule(id)
330
+ default_rules.find { |rule| rule.id == id }
331
+ end
332
+ end
333
+ end
334
+ end
335
+
336
+ # Registration
337
+ register "pt"
338
+ register "pt-BR"
339
+ register "pt-PT"
340
+ register "pt-AO"
341
+ register "pt-MZ"
342
+ register "pt-GW"
343
+ register "pt-CV"
344
+
345
+ HUNSPELL_DICTIONARIES = {
346
+ 'pt-BR' => {
347
+ aff: 'spec/integrational/fixtures/pt_BR.aff',
348
+ dic: 'spec/integrational/fixtures/pt_BR.dic'
349
+ },
350
+ 'pt-PT' => {
351
+ aff: 'spec/integrational/fixtures/pt_PT.aff',
352
+ dic: 'spec/integrational/fixtures/pt_PT.dic'
353
+ }
354
+ }.freeze
355
+
356
+ VARIANT_NAMES = {
357
+ 'BR' => 'Brazilian',
358
+ 'PT' => 'European',
359
+ 'AO' => 'Angolan',
360
+ 'MZ' => 'Mozambican',
361
+ 'GW' => 'Guinea-Bissau',
362
+ 'CV' => 'Cape Verdean'
363
+ }.freeze
364
+
365
+ def initialize(code: "pt", name: "Portuguese", variant: nil)
366
+ variant ||= extract_region_code(code)
367
+ super
368
+ @hunspell_paths = resolve_hunspell_paths(code)
369
+ end
370
+
371
+ def description
372
+ return name unless variant
373
+
374
+ variant_name = VARIANT_NAMES[variant] || variant
375
+ "#{name} (#{variant_name})"
376
+ end
377
+
378
+ def tokenizer
379
+ @tokenizer ||= Tokenizer.new
380
+ end
381
+
382
+ def normalizer
383
+ @normalizer ||= Language::Normalizer::Base.new
384
+ end
385
+
386
+ def dictionary_class
387
+ Dictionary::UnixWords
388
+ end
389
+
390
+ def default_dictionary_paths
391
+ case code
392
+ when "pt-BR"
393
+ ["/usr/share/dict/brazilian"]
394
+ when "pt-PT"
395
+ ["/usr/share/dict/portuguese"]
396
+ else
397
+ ["/usr/share/dict/words"]
398
+ end
399
+ end
400
+
401
+ def script_type
402
+ :latin
403
+ end
404
+
405
+ def create_spell_checker
406
+ SpellChecker.new(
407
+ aff_path: @hunspell_paths[:aff],
408
+ dic_path: @hunspell_paths[:dic],
409
+ script: :latin
410
+ )
411
+ end
412
+
413
+ def create_tokenizer
414
+ Tokenizer.new
415
+ end
416
+
417
+ def create_pos_tagger
418
+ POSTagger.new(
419
+ aff_path: @hunspell_paths[:aff],
420
+ dic_path: @hunspell_paths[:dic],
421
+ script: :latin,
422
+ flag_mapping: POSTagger::FLAG_TO_POS
423
+ )
424
+ end
425
+
426
+ private
427
+
428
+ def extract_region_code(code)
429
+ return nil unless code.include?("-")
430
+
431
+ code.split("-", 2).last.upcase
432
+ end
433
+
434
+ def resolve_hunspell_paths(code)
435
+ HUNSPELL_DICTIONARIES[code] || HUNSPELL_DICTIONARIES['pt-BR']
436
+ end
437
+ end
438
+ end
439
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Languages
5
+ # Romanian language implementation (plan 84, wave 1).
6
+ #
7
+ # Thin composition: shared Latin tokenizer, base normalizer,
8
+ # registry entry. Romanian diacritics (ă â î ș ț) are AltGr sequences on the US physical layout.
9
+ class Romanian < LatinBase
10
+ register "ro"
11
+ register "ro-RO"
12
+
13
+ def initialize(code: "ro", name: "Romanian")
14
+ super
15
+ end
16
+
17
+ def default_dictionary_paths
18
+ ["/usr/share/dict/romanian"]
19
+ end
20
+ end
21
+ end
22
+ end