kotoshu 1.0.2-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Passthrough spell checker for languages that don't use spell checking.
6
+ #
7
+ # This checker always returns that words are "found" (correct). It's used
8
+ # for languages that don't have traditional spell checking, such as:
9
+ # - CJK languages (Japanese, Chinese) - use confusion rules instead
10
+ # - Languages with purely rule-based checking
11
+ #
12
+ # @example
13
+ # checker = PassthroughSpellChecker.new
14
+ # result = checker.check('任意のテキスト')
15
+ # # => { found: true, stem: nil, flags: [] }
16
+ #
17
+ # @example Getting suggestions (always empty)
18
+ # suggestions = checker.suggest('テキスト')
19
+ # # => []
20
+ class PassthroughSpellChecker < SpellChecker
21
+ # Create a new passthrough spell checker.
22
+ #
23
+ # @param reason [String] Optional reason why spell checking is not used
24
+ def initialize(reason: nil)
25
+ @reason = reason || "Language does not use spell checking"
26
+ end
27
+
28
+ # Always returns that the word is "found" (correct).
29
+ #
30
+ # @param _word [String] The word to check (ignored)
31
+ # @return [Hash] Always returns { found: true, stem: nil, flags: [] }
32
+ def check(_word)
33
+ { found: true, stem: nil, flags: [] }
34
+ end
35
+
36
+ # Returns no suggestions.
37
+ #
38
+ # Passthrough spell checkers don't provide suggestions.
39
+ #
40
+ # @param _word [String] The word (ignored)
41
+ # @param _max_suggestions [Integer] Max suggestions (ignored)
42
+ # @return [Array<Hash>] Always returns empty array
43
+ def suggest(_word, _max_suggestions: 10)
44
+ []
45
+ end
46
+
47
+ # Always returns true (all words are "correct").
48
+ #
49
+ # @param _word [String] The word to check (ignored)
50
+ # @return [Boolean] Always true
51
+ def correct?(_word)
52
+ true
53
+ end
54
+
55
+ # Get the reason why spell checking is not used.
56
+ #
57
+ # @return [String] Reason text
58
+ def reason
59
+ @reason
60
+ end
61
+
62
+ # Check if this is a passthrough checker.
63
+ #
64
+ # @return [Boolean] Always true for this class
65
+ def passthrough?
66
+ true
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Base class for POS (Part-of-Speech) taggers.
6
+ #
7
+ # POS taggers assign grammatical categories (NOUN, VERB, ADJ, etc.) to tokens.
8
+ # Different languages use different POS tagging strategies:
9
+ # - Latin scripts: Dictionary-based (Hunspell flags → POS tags)
10
+ # - CJK: Integrated with morphological analysis (tokenizer provides POS)
11
+ # - German: Compound word decomposition affects tagging
12
+ #
13
+ # Common POS tags (Penn Treebank style):
14
+ # - CC: Coordinating conjunction
15
+ # - CD: Cardinal number
16
+ # - DT: Determiner
17
+ # - EX: Existential there
18
+ # - FW: Foreign word
19
+ # - IN: Preposition or subordinating conjunction
20
+ # - JJ: Adjective
21
+ # - JJR: Adjective, comparative
22
+ # - JJS: Adjective, superlative
23
+ # - LS: List item marker
24
+ # - MD: Modal
25
+ # - NN: Noun, singular or mass
26
+ # - NNS: Noun, plural
27
+ # - NNP: Proper noun, singular
28
+ # - NNPS: Proper noun, plural
29
+ # - PDT: Predeterminer
30
+ # - POS: Possessive ending
31
+ # - PRP: Personal pronoun
32
+ # - PRP$: Possessive pronoun
33
+ # - RB: Adverb
34
+ # - RBR: Adverb, comparative
35
+ # - RBS: Adverb, superlative
36
+ # - RP: Particle
37
+ # - SYM: Symbol
38
+ # - TO: to
39
+ # - UH: Interjection
40
+ # - VB: Verb, base form
41
+ # - VBD: Verb, past tense
42
+ # - VBG: Verb, gerund or present participle
43
+ # - VBN: Verb, past participle
44
+ # - VBP: Verb, non-3rd person singular present
45
+ # - VBZ: Verb, 3rd person singular present
46
+ # - WDT: Wh-determiner
47
+ # - WP: Wh-pronoun
48
+ # - WP$: Possessive wh-pronoun
49
+ # - WRB: Wh-adverb
50
+ #
51
+ # Language-specific tags:
52
+ # - CJK uses its own tagset (e.g., Japanese: 名詞, 動詞, etc.)
53
+ # - German uses STTS tagset
54
+ #
55
+ # @abstract Subclasses must implement #tag
56
+ #
57
+ # @example Tagging tokens
58
+ # tagger = EnglishPosTagger.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
59
+ # tokens = [
60
+ # { token: "The", position: 0, length: 3 },
61
+ # { token: "dog", position: 4, length: 3 },
62
+ # { token: "runs", position: 8, length: 4 }
63
+ # ]
64
+ # tagged = tagger.tag(tokens)
65
+ # # => [
66
+ # # { token: "The", position: 0, length: 3, pos_tag: "DET", lemma: "the" },
67
+ # # { token: "dog", position: 4, length: 3, pos_tag: "NOUN", lemma: "dog" },
68
+ # # { token: "runs", position: 8, length: 4, pos_tag: "VERB", lemma: "run" }
69
+ # # ]
70
+ class PosTagger
71
+ # Tag tokens with POS information.
72
+ #
73
+ # Takes an array of token hashes (from Tokenizer#tokenize) and adds:
74
+ # - :pos_tag (String, nil) - POS category (NOUN, VERB, etc.) or nil if unknown
75
+ # - :lemma (String, nil) - Lemma/base form or nil if unknown
76
+ #
77
+ # @abstract Subclasses must implement
78
+ # @param tokens [Array<Hash>] Array of token hashes from Tokenizer
79
+ # @return [Array<Hash>] Token hashes with added :pos_tag and :lemma keys
80
+ # @raise [NotImplementedError] if not implemented by subclass
81
+ def tag(tokens)
82
+ raise NotImplementedError, "#{self.class} must implement #tag"
83
+ end
84
+
85
+ # Tag a single word.
86
+ #
87
+ # Convenience method for single-word tagging.
88
+ #
89
+ # @param word [String] The word to tag
90
+ # @return [Hash] Hash with :pos_tag and :lemma keys (may be nil)
91
+ def tag_word(word)
92
+ token = { token: word, position: 0, length: word.length }
93
+ result = tag([token])
94
+ result.first || { pos_tag: nil, lemma: nil }
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Base class for spell checkers.
6
+ #
7
+ # Spell checkers validate words and provide suggestions for misspelled words.
8
+ # Different languages use different spell checking strategies:
9
+ # - Latin scripts: Dictionary lookup (Hunspell, Morfologik)
10
+ # - CJK: Confusion rule checking (no dictionary)
11
+ # - RTL: Dictionary lookup with bidirectional text handling
12
+ #
13
+ # @abstract Subclasses must implement #check and #suggest
14
+ #
15
+ # @example Checking a word
16
+ # checker = EnglishSpellChecker.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
17
+ # result = checker.check("hello")
18
+ # # => { found: true, stem: "hello", flags: [] }
19
+ #
20
+ # @example Getting suggestions
21
+ # result = checker.check("helo")
22
+ # # => { found: false, stem: nil, flags: [] }
23
+ # suggestions = checker.suggest("helo")
24
+ # # => [
25
+ # # { word: "hello", distance: 1, score: 0.9 },
26
+ # # { word: "help", distance: 2, score: 0.7 }
27
+ # # ]
28
+ class SpellChecker
29
+ # Check if a word is spelled correctly.
30
+ #
31
+ # Returns a hash with:
32
+ # - :found (Boolean) - true if word is in dictionary
33
+ # - :stem (String, nil) - The stem/lemma if found
34
+ # - :flags (Array<String>) - Morphological flags
35
+ #
36
+ # @abstract Subclasses must implement
37
+ # @param word [String] The word to check
38
+ # @return [Hash] Result with :found, :stem, :flags
39
+ # @raise [NotImplementedError] if not implemented by subclass
40
+ def check(word)
41
+ raise NotImplementedError, "#{self.class} must implement #check"
42
+ end
43
+
44
+ # Get spelling suggestions for a misspelled word.
45
+ #
46
+ # Returns an array of suggestion hashes with:
47
+ # - :word (String) - The suggested word
48
+ # - :distance (Integer) - Edit distance from original word
49
+ # - :score (Float) - Confidence score (0-1, higher is better)
50
+ #
51
+ # Suggestions are sorted by relevance (highest score first).
52
+ #
53
+ # @abstract Subclasses must implement
54
+ # @param word [String] The misspelled word
55
+ # @param max_suggestions [Integer] Maximum number of suggestions to return
56
+ # @return [Array<Hash>] Array of suggestion hashes
57
+ # @raise [NotImplementedError] if not implemented by subclass
58
+ def suggest(word, max_suggestions: 10)
59
+ raise NotImplementedError, "#{self.class} must implement #suggest"
60
+ end
61
+
62
+ # Check if a word is spelled correctly.
63
+ #
64
+ # Convenience method that returns a boolean.
65
+ #
66
+ # @param word [String] The word to check
67
+ # @return [Boolean] true if word is correct
68
+ def correct?(word)
69
+ check(word)[:found]
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Base class for word form synthesizers.
6
+ #
7
+ # Synthesizers generate inflected forms from a lemma (base form).
8
+ # This is the inverse of lemmatization:
9
+ # - Lemmatization: "runs" → "run"
10
+ # - Synthesis: "run" → ["run", "runs", "running", "ran"]
11
+ #
12
+ # Different languages use different synthesis strategies:
13
+ # - Latin scripts: Hunspell affix rules
14
+ # - CJK: Not applicable (no inflection)
15
+ # - German: Compound word + affix synthesis
16
+ # - Finnish: Complex agglutinative patterns
17
+ #
18
+ # @abstract Subclasses must implement #synthesize
19
+ #
20
+ # @example Synthesizing English verb forms
21
+ # synthesizer = EnglishSynthesizer.new(aff_path: "en_US.aff", dic_path: "en_US.dic")
22
+ # forms = synthesizer.synthesize("run", "VERB")
23
+ # # => ["run", "runs", "running", "ran"]
24
+ #
25
+ # @example Synthesizing with POS constraint
26
+ # forms = synthesizer.synthesize("happy", "ADJ")
27
+ # # => ["happy", "happier", "happiest"]
28
+ class Synthesizer
29
+ # Generate inflected forms of a word.
30
+ #
31
+ # Given a lemma (base form) and a POS tag, returns all possible
32
+ # inflected forms of that word.
33
+ #
34
+ # @abstract Subclasses must implement
35
+ # @param lemma [String] The base form (lemma)
36
+ # @param pos_tag [String] The POS tag to constrain generation
37
+ # @return [Array<String>] Array of inflected forms
38
+ # @raise [NotImplementedError] if not implemented by subclass
39
+ def synthesize(lemma, pos_tag)
40
+ raise NotImplementedError, "#{self.class} must implement #synthesize"
41
+ end
42
+
43
+ # Generate all inflected forms (all POS tags).
44
+ #
45
+ # Convenience method that generates forms for all possible POS tags.
46
+ #
47
+ # @param lemma [String] The base form (lemma)
48
+ # @return [Hash] Hash mapping POS tags to arrays of forms
49
+ def synthesize_all(lemma)
50
+ # Default implementation - subclasses can optimize
51
+ {
52
+ 'NOUN' => synthesize(lemma, 'NOUN'),
53
+ 'VERB' => synthesize(lemma, 'VERB'),
54
+ 'ADJ' => synthesize(lemma, 'ADJ'),
55
+ 'ADV' => synthesize(lemma, 'ADV')
56
+ }
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Base class for tokenizers.
6
+ #
7
+ # Tokenizers split text into individual tokens (words, punctuation).
8
+ # Different languages use different tokenization strategies:
9
+ # - Latin scripts: Whitespace + punctuation
10
+ # - CJK: Morphological analysis
11
+ # - German: Compound word splitting
12
+ # - RTL: Right-to-left text handling
13
+ #
14
+ # @abstract Subclasses must implement #tokenize
15
+ #
16
+ # @example Tokenizing English text
17
+ # tokenizer = WhitespaceTokenizer.new
18
+ # tokens = tokenizer.tokenize("Hello, world!")
19
+ # # => [
20
+ # # { token: "Hello", position: 0, length: 5 },
21
+ # # { token: ",", position: 5, length: 1 },
22
+ # # { token: "world", position: 7, length: 5 },
23
+ # # { token: "!", position: 12, length: 1 }
24
+ # # ]
25
+ class Tokenizer
26
+ # Split text into tokens.
27
+ #
28
+ # Each token is a hash with:
29
+ # - :token (String) - The token text
30
+ # - :position (Integer) - Character position in original text
31
+ # - :length (Integer) - Token length in characters
32
+ #
33
+ # Additional keys may be added by subclasses:
34
+ # - :pos_tag (String) - Part of speech tag
35
+ # - :lemma (String) - Base form / lemma
36
+ # - :compound_part (Boolean) - Whether this is a compound word part
37
+ # - :script (Symbol) - Script type for multilingual text
38
+ #
39
+ # @abstract Subclasses must implement
40
+ # @param text [String] The input text
41
+ # @return [Array<Hash>] Array of token hashes
42
+ # @raise [NotImplementedError] if not implemented by subclass
43
+ def tokenize(text)
44
+ raise NotImplementedError, "#{self.class} must implement #tokenize"
45
+ end
46
+
47
+ # Tokenize and return just the token strings.
48
+ #
49
+ # Convenience method for when you only need the text content.
50
+ #
51
+ # @param text [String] The input text
52
+ # @return [Array<String>] Array of token strings
53
+ def tokenize_to_strings(text)
54
+ tokenize(text).map { |t| t[:token] }
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ module Components
5
+ # Whitespace-based tokenizer for Latin-script languages.
6
+ #
7
+ # Splits text on whitespace and separates punctuation.
8
+ # Suitable for languages with space-separated words (English, French, German, etc.).
9
+ #
10
+ # This is a simple tokenizer that works well for most Latin-script languages.
11
+ # For more advanced tokenization (contractions, compounds), use language-specific
12
+ # tokenizers.
13
+ #
14
+ # @example Basic tokenization
15
+ # tokenizer = WhitespaceTokenizer.new
16
+ # tokens = tokenizer.tokenize("Hello, world!")
17
+ # # => [
18
+ # # { token: "Hello", position: 0, length: 5 },
19
+ # # { token: ",", position: 5, length: 1 },
20
+ # # { token: "world", position: 7, length: 5 },
21
+ # # { token: "!", position: 12, length: 1 }
22
+ # # ]
23
+ #
24
+ # @example Tokenizing to strings
25
+ # tokenizer.tokenize_to_strings("Hello, world!")
26
+ # # => ["Hello", ",", "world", "!"]
27
+ class WhitespaceTokenizer < Tokenizer
28
+ # Regex pattern for matching tokens (words or punctuation).
29
+ TOKEN_PATTERN = /[\w']+|[^\w\s]/
30
+
31
+ # Create a new whitespace tokenizer.
32
+ #
33
+ # @param pattern [Regexp] Optional custom token pattern
34
+ def initialize(pattern: TOKEN_PATTERN)
35
+ @pattern = pattern
36
+ end
37
+
38
+ # Split text into tokens.
39
+ #
40
+ # Each token is a hash with:
41
+ # - :token (String) - The token text
42
+ # - :position (Integer) - Character position in original text
43
+ # - :length (Integer) - Token length in characters
44
+ #
45
+ # @param text [String] The input text
46
+ # @return [Array<Hash>] Array of token hashes
47
+ def tokenize(text)
48
+ return [] if text.nil? || text.empty?
49
+
50
+ tokens = []
51
+ position = 0
52
+
53
+ # Find all matches
54
+ text.scan(@pattern) do |match|
55
+ match_str = match.is_a?(Array) ? match.first : match
56
+ start_pos = text.index(match_str, position)
57
+
58
+ tokens << {
59
+ token: match_str,
60
+ position: start_pos,
61
+ length: match_str.length
62
+ }
63
+
64
+ position = start_pos + match_str.length
65
+ end
66
+
67
+ tokens
68
+ end
69
+
70
+ # Get the token pattern used by this tokenizer.
71
+ #
72
+ # @return [Regexp] The token pattern
73
+ def pattern
74
+ @pattern
75
+ end
76
+
77
+ # Check if a character is a word character.
78
+ #
79
+ # @param char [String] Single character
80
+ # @return [Boolean] True if word character
81
+ def word_char?(char)
82
+ char.match?(/\w/)
83
+ end
84
+
85
+ # Check if a character is punctuation.
86
+ #
87
+ # @param char [String] Single character
88
+ # @return [Boolean] True if punctuation
89
+ def punctuation?(char)
90
+ char.match?(/[^\w\s]/)
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ # Pluggable linguistic components (tokenizer, POS tagger, spell checker, synthesizer).
5
+ module Components
6
+ autoload :SpellChecker, "kotoshu/components/spell_checker"
7
+ autoload :PassthroughSpellChecker, "kotoshu/components/passthrough_spell_checker"
8
+ autoload :PosTagger, "kotoshu/components/pos_tagger"
9
+ autoload :Synthesizer, "kotoshu/components/synthesizer"
10
+ autoload :Tokenizer, "kotoshu/components/tokenizer"
11
+ autoload :WhitespaceTokenizer, "kotoshu/components/whitespace_tokenizer"
12
+ end
13
+ end
@@ -0,0 +1,207 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kotoshu
4
+ class Configuration
5
+ # Builder for creating immutable Configuration objects.
6
+ #
7
+ # Provides a fluent interface for building configuration objects
8
+ # that are frozen after creation, ensuring thread-safety and immutability.
9
+ #
10
+ # @example Building with block
11
+ # config = Configuration::Builder.build do |b|
12
+ # b.dictionary_path = "words.txt"
13
+ # b.language = "en-GB"
14
+ # end
15
+ #
16
+ # @example Building with fluent methods
17
+ # config = Configuration::Builder.build
18
+ # .with_dictionary_path("words.txt")
19
+ # .with_language("en-GB")
20
+ class Builder
21
+ # Build an immutable configuration.
22
+ #
23
+ # @yield [builder] Optional block for configuration
24
+ # @return [Configuration] Frozen configuration object
25
+ #
26
+ # @example With block
27
+ # config = Builder.build do |b|
28
+ # b.dictionary_path = "words.txt"
29
+ # end
30
+ #
31
+ # @example Without block (uses defaults)
32
+ # config = Builder.build
33
+ def self.build
34
+ builder_instance = new
35
+ yield(builder_instance) if block_given?
36
+ builder_instance.to_config
37
+ end
38
+
39
+ # Create a new builder.
40
+ def initialize
41
+ @settings = DEFAULTS.dup
42
+ end
43
+
44
+ # Set dictionary path.
45
+ #
46
+ # @param path [String] Path to dictionary file
47
+ # @return [self] Self for chaining
48
+ def dictionary_path=(path)
49
+ @settings[:dictionary_path] = path
50
+ self
51
+ end
52
+
53
+ # Set dictionary type.
54
+ #
55
+ # @param type [Symbol] Dictionary type
56
+ # @return [self] Self for chaining
57
+ def dictionary_type=(type)
58
+ @settings[:dictionary_type] = type
59
+ self
60
+ end
61
+
62
+ # Set language code.
63
+ #
64
+ # @param lang [String] Language code
65
+ # @return [self] Self for chaining
66
+ def language=(lang)
67
+ @settings[:language] = lang
68
+ self
69
+ end
70
+
71
+ # Set locale.
72
+ #
73
+ # @param locale [String, nil] Locale
74
+ # @return [self] Self for chaining
75
+ def locale=(locale)
76
+ @settings[:locale] = locale
77
+ self
78
+ end
79
+
80
+ # Set max suggestions.
81
+ #
82
+ # @param max [Integer] Maximum suggestions
83
+ # @return [self] Self for chaining
84
+ def max_suggestions=(max)
85
+ @settings[:max_suggestions] = max
86
+ self
87
+ end
88
+
89
+ # Set case sensitivity.
90
+ #
91
+ # @param sensitive [Boolean] Case sensitive flag
92
+ # @return [self] Self for chaining
93
+ def case_sensitive=(sensitive)
94
+ @settings[:case_sensitive] = sensitive
95
+ self
96
+ end
97
+
98
+ # Set verbose mode.
99
+ #
100
+ # @param verbose [Boolean] Verbose flag
101
+ # @return [self] Self for chaining
102
+ def verbose=(verbose)
103
+ @settings[:verbose] = verbose
104
+ self
105
+ end
106
+
107
+ # Set suggestion algorithms.
108
+ #
109
+ # @param algorithms [Array<Class>, nil] Suggestion algorithms
110
+ # @return [self] Self for chaining
111
+ def suggestion_algorithms=(algorithms)
112
+ @settings[:suggestion_algorithms] = algorithms
113
+ self
114
+ end
115
+
116
+ # Set custom words.
117
+ #
118
+ # @param words [Array<String>] Custom words
119
+ # @return [self] Self for chaining
120
+ def custom_words=(words)
121
+ @settings[:custom_words] = words.dup.freeze
122
+ self
123
+ end
124
+
125
+ # Set encoding.
126
+ #
127
+ # @param encoding [String] Character encoding
128
+ # @return [self] Self for chaining
129
+ def encoding=(encoding)
130
+ @settings[:encoding] = encoding
131
+ self
132
+ end
133
+
134
+ # Fluent method to set dictionary path.
135
+ #
136
+ # @param path [String] Path to dictionary file
137
+ # @return [Configuration] New configuration
138
+ def with_dictionary_path(path)
139
+ @settings[:dictionary_path] = path
140
+ self
141
+ end
142
+
143
+ # Fluent method to set dictionary type.
144
+ #
145
+ # @param type [Symbol] Dictionary type
146
+ # @return [Configuration] New configuration
147
+ def with_dictionary_type(type)
148
+ @settings[:dictionary_type] = type
149
+ self
150
+ end
151
+
152
+ # Fluent method to set language.
153
+ #
154
+ # @param lang [String] Language code
155
+ # @return [Configuration] New configuration
156
+ def with_language(lang)
157
+ @settings[:language] = lang
158
+ self
159
+ end
160
+
161
+ # Fluent method to set locale.
162
+ #
163
+ # @param locale [String, nil] Locale
164
+ # @return [Configuration] New configuration
165
+ def with_locale(locale)
166
+ @settings[:locale] = locale
167
+ self
168
+ end
169
+
170
+ # Fluent method to set max suggestions.
171
+ #
172
+ # @param max [Integer] Maximum suggestions
173
+ # @return [Configuration] New configuration
174
+ def with_max_suggestions(max)
175
+ @settings[:max_suggestions] = max
176
+ self
177
+ end
178
+
179
+ # Fluent method to set case sensitivity.
180
+ #
181
+ # @param sensitive [Boolean] Case sensitive flag
182
+ # @return [Configuration] New configuration
183
+ def with_case_sensitive(sensitive)
184
+ @settings[:case_sensitive] = sensitive
185
+ self
186
+ end
187
+
188
+ # Fluent method to set verbose mode.
189
+ #
190
+ # @param verbose [Boolean] Verbose flag
191
+ # @return [Configuration] New configuration
192
+ def with_verbose(verbose)
193
+ @settings[:verbose] = verbose
194
+ self
195
+ end
196
+
197
+ # Convert builder to frozen Configuration.
198
+ #
199
+ # @return [Configuration] Frozen configuration object
200
+ def to_config
201
+ config = Configuration.new(@settings.dup)
202
+ config.freeze
203
+ config
204
+ end
205
+ end
206
+ end
207
+ end