kotoshu 1.0.2-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (458) hide show
  1. checksums.yaml +7 -0
  2. data/.kotoshu-baseline.json +1 -0
  3. data/.pre-commit-hooks.yaml +18 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +351 -0
  6. data/.yardopts +8 -0
  7. data/CHANGELOG.md +813 -0
  8. data/CLAUDE.md +176 -0
  9. data/CODE_OF_CONDUCT.md +132 -0
  10. data/Cargo.lock +359 -0
  11. data/Cargo.toml +15 -0
  12. data/LICENSE +31 -0
  13. data/README.adoc +1292 -0
  14. data/Rakefile +150 -0
  15. data/SECURITY.md +93 -0
  16. data/TODO.impl/00-cut-0.1.md +121 -0
  17. data/TODO.impl/00-cut-0.2.md +175 -0
  18. data/TODO.impl/00-vision.md +281 -0
  19. data/TODO.impl/01-hunspell-correctness.md +111 -0
  20. data/TODO.impl/02-cli-unification.md +74 -0
  21. data/TODO.impl/03-dynamic-download.md +107 -0
  22. data/TODO.impl/04-language-modules.md +89 -0
  23. data/TODO.impl/05-semantic-path.md +83 -0
  24. data/TODO.impl/06-cjk-support.md +105 -0
  25. data/TODO.impl/07-rtl-support.md +71 -0
  26. data/TODO.impl/08-grammar-engine.md +106 -0
  27. data/TODO.impl/09-integrity-security.md +90 -0
  28. data/TODO.impl/10-testing-ci.md +98 -0
  29. data/TODO.impl/100-full-feature-batch3.md +28 -0
  30. data/TODO.impl/101-int4-tier-experiment.md +32 -0
  31. data/TODO.impl/102-lid-chain.md +28 -0
  32. data/TODO.impl/103-oov-bucket-tables.md +27 -0
  33. data/TODO.impl/104-vscode-marketplace-readiness.md +32 -0
  34. data/TODO.impl/105-personal-dictionary-in-check.md +34 -0
  35. data/TODO.impl/106-native-lid-detection.md +36 -0
  36. data/TODO.impl/107-basic-support-every-staged-language.md +42 -0
  37. data/TODO.impl/108-ko-ne-full-feature.md +37 -0
  38. data/TODO.impl/109-ten-audit.md +26 -0
  39. data/TODO.impl/11-release-v1.md +95 -0
  40. data/TODO.impl/110-sr-latn-and-polish.md +33 -0
  41. data/TODO.impl/111-embedding-bakeoff.md +33 -0
  42. data/TODO.impl/112-deployability-and-gating.md +26 -0
  43. data/TODO.impl/113-language-packs.md +26 -0
  44. data/TODO.impl/114-typo-biencoder-thread.md +33 -0
  45. data/TODO.impl/115-hybrid-retrieval-pricing.md +22 -0
  46. data/TODO.impl/116-baseline-suggestion-skip.md +81 -0
  47. data/TODO.impl/117-deterministic-frequency-tiers.md +110 -0
  48. data/TODO.impl/118-server-gem-floor.md +33 -0
  49. data/TODO.impl/119-embedded-kelly-tiers.md +56 -0
  50. data/TODO.impl/120-setup-metadata-refresh.md +35 -0
  51. data/TODO.impl/121-jekyll-baseline-skip.md +33 -0
  52. data/TODO.impl/122-pack-compression-kpk2.md +66 -0
  53. data/TODO.impl/123-typo-corpus-synthesis.md +55 -0
  54. data/TODO.impl/124-owner-residue.md +19 -0
  55. data/TODO.impl/125-consumer-floor-guards.md +43 -0
  56. data/TODO.impl/126-tier-error-budgets.md +33 -0
  57. data/TODO.impl/127-suggestion-feedback-loop.md +34 -0
  58. data/TODO.impl/128-hybrid-pricing-verdict.md +47 -0
  59. data/TODO.impl/129-pt-fr-corpora.md +23 -0
  60. data/TODO.impl/130-news-truth-0912.md +23 -0
  61. data/TODO.impl/131-hybrid-productization.md +42 -0
  62. data/TODO.impl/132-techniques-page-and-prose.md +27 -0
  63. data/TODO.impl/133-rust-first-distribution.md +68 -0
  64. data/TODO.impl/20-encapsulation-cleanup.md +47 -0
  65. data/TODO.impl/21-autoload-migration.md +64 -0
  66. data/TODO.impl/22-url-pin-single-source-of-truth.md +120 -0
  67. data/TODO.impl/23-fix-url-defaults.md +57 -0
  68. data/TODO.impl/24-cli-auto-setup-prompt.md +76 -0
  69. data/TODO.impl/25-soft-onnxruntime-require.md +75 -0
  70. data/TODO.impl/26-onnx-vocab-json-generation.md +96 -0
  71. data/TODO.impl/27-end-to-end-smoke-spec.md +113 -0
  72. data/TODO.impl/28-readme-quickstart-verification.md +68 -0
  73. data/TODO.impl/29-status-command.md +84 -0
  74. data/TODO.impl/30-language-auto-detection.md +74 -0
  75. data/TODO.impl/31-download-progress.md +58 -0
  76. data/TODO.impl/32-sarif-output-verification.md +72 -0
  77. data/TODO.impl/33-audit-log-rotation.md +54 -0
  78. data/TODO.impl/34-cache-eviction.md +59 -0
  79. data/TODO.impl/35-shell-completion.md +61 -0
  80. data/TODO.impl/36-cleanup-regressions-0.3.1.md +134 -0
  81. data/TODO.impl/37-hunspell-correctness-tier2.md +203 -0
  82. data/TODO.impl/38-onnx-semantic-gating.md +218 -0
  83. data/TODO.impl/39-tier3-and-beyond.md +232 -0
  84. data/TODO.impl/40-spec-drift-cleanup.md +134 -0
  85. data/TODO.impl/50-structure-aware-document-api.md +194 -0
  86. data/TODO.impl/51-grammar-engine-expansion.md +75 -0
  87. data/TODO.impl/52-multi-language-documents.md +95 -0
  88. data/TODO.impl/53-document-plugin-architecture.md +94 -0
  89. data/TODO.impl/54-cjk-support.md +117 -0
  90. data/TODO.impl/55-rtl-support.md +109 -0
  91. data/TODO.impl/56-t4-t5-t6-quality-architecture-ecosystem.md +126 -0
  92. data/TODO.impl/57-t5.1-step3-pure-function-strategies.md +138 -0
  93. data/TODO.impl/58-t5.2-configuration-as-data.md +113 -0
  94. data/TODO.impl/59-yard-api-documentation.md +75 -0
  95. data/TODO.impl/60-lsp-server.md +174 -0
  96. data/TODO.impl/61-editor-ecosystem.md +167 -0
  97. data/TODO.impl/62-framework-integrations.md +159 -0
  98. data/TODO.impl/63-web-playground-wasm.md +186 -0
  99. data/TODO.impl/64-http-api-and-sdks.md +191 -0
  100. data/TODO.impl/65-universal-kotoshu.md +251 -0
  101. data/TODO.impl/66-kotoshu-core.md +169 -0
  102. data/TODO.impl/67-kotoshu-rs-and-access-libraries.md +137 -0
  103. data/TODO.impl/68-sota-adoption.md +152 -0
  104. data/TODO.impl/69-sota-typo-eval.md +74 -0
  105. data/TODO.impl/70-sota-confidence-cascade.md +59 -0
  106. data/TODO.impl/71-sota-nested-model.md +63 -0
  107. data/TODO.impl/72-model2vec-evaluation.md +49 -0
  108. data/TODO.impl/73-rl-reranker.md +42 -0
  109. data/TODO.impl/74-distilled-reranker.md +36 -0
  110. data/TODO.impl/75-llm-evaluator.md +35 -0
  111. data/TODO.impl/76-optional-llm-backend.md +37 -0
  112. data/TODO.impl/77-model-coverage-expansion.md +96 -0
  113. data/TODO.impl/78-in-browser-playground.md +69 -0
  114. data/TODO.impl/79-news-and-releases.md +60 -0
  115. data/TODO.impl/80-ecosystem-docs.md +56 -0
  116. data/TODO.impl/81-editor-reach.md +68 -0
  117. data/TODO.impl/82-doc-formats-and-ignores.md +81 -0
  118. data/TODO.impl/83-models-batch-2.md +56 -0
  119. data/TODO.impl/84-language-modules-wave2.md +75 -0
  120. data/TODO.impl/85-wasm-model-api.md +48 -0
  121. data/TODO.impl/86-site-truth-wave2.md +45 -0
  122. data/TODO.impl/87-lsp-personal-dictionary.md +36 -0
  123. data/TODO.impl/88-cli-directory-mode.md +54 -0
  124. data/TODO.impl/89-framework-integrations-exec.md +59 -0
  125. data/TODO.impl/90-gem-correctness-residues.md +44 -0
  126. data/TODO.impl/91-unicode-and-sv-aff.md +47 -0
  127. data/TODO.impl/92-tier-mirror-cors.md +88 -0
  128. data/TODO.impl/93-python-wheels.md +32 -0
  129. data/TODO.impl/94-site-integrations-docs.md +35 -0
  130. data/TODO.impl/95-editor-action-polish.md +27 -0
  131. data/TODO.impl/96-server-semantic-models.md +50 -0
  132. data/TODO.impl/97-yard-execution.md +73 -0
  133. data/TODO.impl/98-campaign-cleanup.md +78 -0
  134. data/TODO.impl/99-site-truth-performance-docs.md +31 -0
  135. data/conformance/vectors.jsonl +2630 -0
  136. data/data/en/grammar/rules.yaml +851 -0
  137. data/docs/STABILITY-1.0-AUDIT.md +251 -0
  138. data/docs/STABILITY-1.0.md +137 -0
  139. data/docs/cache-architecture.md +502 -0
  140. data/docs/verification-evidence/correct3/pr_body.md +37 -0
  141. data/docs/verification-evidence/correct3/rspec_final.log +5588 -0
  142. data/docs/verification-evidence/correct3/rspec_full.log +4504 -0
  143. data/docs/verification-evidence/correct3/sv.aff +856 -0
  144. data/docs/verification-evidence/correct3/sv.dic +153717 -0
  145. data/examples/01_basic_word_checking.rb +38 -0
  146. data/examples/02_text_document_checking.rb +77 -0
  147. data/examples/03_dictionary_backends.rb +137 -0
  148. data/examples/04_trie_data_structure.rb +146 -0
  149. data/examples/05_suggestion_algorithms.rb +239 -0
  150. data/examples/06_configuration_advanced.rb +287 -0
  151. data/examples/07_multi_language_dictionaries.rb +278 -0
  152. data/exe/kotoshu +6 -0
  153. data/lib/kotoshu/algorithms/capitalization.rb +276 -0
  154. data/lib/kotoshu/algorithms/edit_distance.rb +128 -0
  155. data/lib/kotoshu/algorithms/lookup/affix_form.rb +132 -0
  156. data/lib/kotoshu/algorithms/lookup/compound_checks.rb +209 -0
  157. data/lib/kotoshu/algorithms/lookup/compound_form.rb +50 -0
  158. data/lib/kotoshu/algorithms/lookup.rb +841 -0
  159. data/lib/kotoshu/algorithms/ngram_suggest.rb +367 -0
  160. data/lib/kotoshu/algorithms/permutations.rb +283 -0
  161. data/lib/kotoshu/algorithms/phonet_suggest.rb +162 -0
  162. data/lib/kotoshu/algorithms/soundex.rb +92 -0
  163. data/lib/kotoshu/algorithms/suggest.rb +794 -0
  164. data/lib/kotoshu/algorithms.rb +23 -0
  165. data/lib/kotoshu/analyzers/semantic_analyzer.rb +405 -0
  166. data/lib/kotoshu/analyzers.rb +8 -0
  167. data/lib/kotoshu/baseline/application.rb +18 -0
  168. data/lib/kotoshu/baseline/entry.rb +33 -0
  169. data/lib/kotoshu/baseline/store.rb +175 -0
  170. data/lib/kotoshu/baseline.rb +20 -0
  171. data/lib/kotoshu/cache/base_cache.rb +778 -0
  172. data/lib/kotoshu/cache/cache.rb +91 -0
  173. data/lib/kotoshu/cache/eviction_policy.rb +66 -0
  174. data/lib/kotoshu/cache/frequency_cache.rb +283 -0
  175. data/lib/kotoshu/cache/language_cache.rb +511 -0
  176. data/lib/kotoshu/cache/lookup_cache.rb +166 -0
  177. data/lib/kotoshu/cache/model_cache.rb +1147 -0
  178. data/lib/kotoshu/cache/model_registry.rb +109 -0
  179. data/lib/kotoshu/cache/staged_languages.rb +45 -0
  180. data/lib/kotoshu/cache/suggestion_cache.rb +113 -0
  181. data/lib/kotoshu/cache.rb +49 -0
  182. data/lib/kotoshu/cli/auto_setup.rb +71 -0
  183. data/lib/kotoshu/cli/baseline_command.rb +119 -0
  184. data/lib/kotoshu/cli/batch_reporter.rb +321 -0
  185. data/lib/kotoshu/cli/cache_command.rb +282 -0
  186. data/lib/kotoshu/cli/completions_command.rb +210 -0
  187. data/lib/kotoshu/cli/dict_command.rb +64 -0
  188. data/lib/kotoshu/cli/directory_check.rb +146 -0
  189. data/lib/kotoshu/cli/directory_walker.rb +132 -0
  190. data/lib/kotoshu/cli/display_formatter.rb +429 -0
  191. data/lib/kotoshu/cli/errors.rb +36 -0
  192. data/lib/kotoshu/cli/ignore_matcher.rb +205 -0
  193. data/lib/kotoshu/cli/interactive_reviewer.rb +314 -0
  194. data/lib/kotoshu/cli/language_resolver.rb +91 -0
  195. data/lib/kotoshu/cli/navigation_manager.rb +273 -0
  196. data/lib/kotoshu/cli/personal_command.rb +113 -0
  197. data/lib/kotoshu/cli/progress_reporter.rb +114 -0
  198. data/lib/kotoshu/cli/status_report.rb +130 -0
  199. data/lib/kotoshu/cli.rb +880 -0
  200. data/lib/kotoshu/components/passthrough_spell_checker.rb +70 -0
  201. data/lib/kotoshu/components/pos_tagger.rb +98 -0
  202. data/lib/kotoshu/components/spell_checker.rb +73 -0
  203. data/lib/kotoshu/components/synthesizer.rb +60 -0
  204. data/lib/kotoshu/components/tokenizer.rb +58 -0
  205. data/lib/kotoshu/components/whitespace_tokenizer.rb +94 -0
  206. data/lib/kotoshu/components.rb +13 -0
  207. data/lib/kotoshu/configuration/builder.rb +207 -0
  208. data/lib/kotoshu/configuration/resolver.rb +124 -0
  209. data/lib/kotoshu/configuration.rb +829 -0
  210. data/lib/kotoshu/conformance_exporter.rb +256 -0
  211. data/lib/kotoshu/conformance_runner.rb +176 -0
  212. data/lib/kotoshu/core/exceptions.rb +186 -0
  213. data/lib/kotoshu/core/indexed_dictionary.rb +293 -0
  214. data/lib/kotoshu/core/models/affix_rule.rb +262 -0
  215. data/lib/kotoshu/core/models/result/document_result.rb +132 -0
  216. data/lib/kotoshu/core/models/result/word_result.rb +131 -0
  217. data/lib/kotoshu/core/models/word.rb +131 -0
  218. data/lib/kotoshu/core/trie/builder.rb +119 -0
  219. data/lib/kotoshu/core/trie/node.rb +94 -0
  220. data/lib/kotoshu/core/trie/trie.rb +249 -0
  221. data/lib/kotoshu/core.rb +28 -0
  222. data/lib/kotoshu/data/common_words/de.yml +1800 -0
  223. data/lib/kotoshu/data/common_words/en.yml +1215 -0
  224. data/lib/kotoshu/data/common_words/es.yml +750 -0
  225. data/lib/kotoshu/data/common_words/fr.yml +1015 -0
  226. data/lib/kotoshu/data/common_words/pt.yml +870 -0
  227. data/lib/kotoshu/data/common_words/ru.yml +484 -0
  228. data/lib/kotoshu/data/common_words_loader.rb +141 -0
  229. data/lib/kotoshu/data/frozen_kelly/en.json +1154 -0
  230. data/lib/kotoshu/data/frozen_kelly.rb +38 -0
  231. data/lib/kotoshu/data.rb +8 -0
  232. data/lib/kotoshu/data_structures/bloom_filter.rb +188 -0
  233. data/lib/kotoshu/data_structures.rb +8 -0
  234. data/lib/kotoshu/debug_logger.rb +146 -0
  235. data/lib/kotoshu/debug_mode.rb +136 -0
  236. data/lib/kotoshu/defaults.rb +84 -0
  237. data/lib/kotoshu/dictionaries/catalog.rb +814 -0
  238. data/lib/kotoshu/dictionaries.rb +8 -0
  239. data/lib/kotoshu/dictionary/base.rb +324 -0
  240. data/lib/kotoshu/dictionary/cspell.rb +243 -0
  241. data/lib/kotoshu/dictionary/custom.rb +225 -0
  242. data/lib/kotoshu/dictionary/hunspell.rb +520 -0
  243. data/lib/kotoshu/dictionary/plain_text.rb +302 -0
  244. data/lib/kotoshu/dictionary/repository.rb +246 -0
  245. data/lib/kotoshu/dictionary/unified.rb +236 -0
  246. data/lib/kotoshu/dictionary/unix_words.rb +218 -0
  247. data/lib/kotoshu/dictionary.rb +15 -0
  248. data/lib/kotoshu/documents/document.rb +120 -0
  249. data/lib/kotoshu/documents/plain_text_document.rb +73 -0
  250. data/lib/kotoshu/documents/source_position.rb +45 -0
  251. data/lib/kotoshu/documents/source_range.rb +76 -0
  252. data/lib/kotoshu/documents/suppressions/scanner.rb +149 -0
  253. data/lib/kotoshu/documents/suppressions/suppression.rb +75 -0
  254. data/lib/kotoshu/documents/suppressions.rb +44 -0
  255. data/lib/kotoshu/documents/text_node.rb +49 -0
  256. data/lib/kotoshu/documents.rb +178 -0
  257. data/lib/kotoshu/embeddings/embedding_pipeline.rb +238 -0
  258. data/lib/kotoshu/embeddings/lru_cache.rb +244 -0
  259. data/lib/kotoshu/embeddings/onnx_runtime_model.rb +411 -0
  260. data/lib/kotoshu/embeddings/protocol.rb +96 -0
  261. data/lib/kotoshu/embeddings/protocols.rb +23 -0
  262. data/lib/kotoshu/embeddings/registry.rb +184 -0
  263. data/lib/kotoshu/embeddings/search.rb +204 -0
  264. data/lib/kotoshu/embeddings/similarity_engine.rb +278 -0
  265. data/lib/kotoshu/embeddings/similarity_search.rb +328 -0
  266. data/lib/kotoshu/embeddings/vocabulary.rb +306 -0
  267. data/lib/kotoshu/embeddings.rb +87 -0
  268. data/lib/kotoshu/fluent_checker.rb +94 -0
  269. data/lib/kotoshu/grammar/pattern_matchers/base_matcher.rb +48 -0
  270. data/lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb +179 -0
  271. data/lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb +86 -0
  272. data/lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb +75 -0
  273. data/lib/kotoshu/grammar/pattern_matchers/possessive_contraction_matcher.rb +87 -0
  274. data/lib/kotoshu/grammar/pattern_matchers/sentence_start_matcher.rb +84 -0
  275. data/lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb +81 -0
  276. data/lib/kotoshu/grammar/pattern_matchers/word_list_matcher.rb +66 -0
  277. data/lib/kotoshu/grammar/rule.rb +99 -0
  278. data/lib/kotoshu/grammar/rule_engine.rb +96 -0
  279. data/lib/kotoshu/grammar/rule_loader.rb +30 -0
  280. data/lib/kotoshu/grammar.rb +24 -0
  281. data/lib/kotoshu/integrity/audit_log.rb +167 -0
  282. data/lib/kotoshu/integrity/manifest.rb +116 -0
  283. data/lib/kotoshu/integrity/net_http.rb +51 -0
  284. data/lib/kotoshu/integrity/rotation_policy.rb +88 -0
  285. data/lib/kotoshu/integrity.rb +26 -0
  286. data/lib/kotoshu/jekyll.rb +101 -0
  287. data/lib/kotoshu/keyboard/layout.rb +116 -0
  288. data/lib/kotoshu/keyboard/layouts/arabic.rb +59 -0
  289. data/lib/kotoshu/keyboard/layouts/azerty.rb +55 -0
  290. data/lib/kotoshu/keyboard/layouts/bulgarian.rb +57 -0
  291. data/lib/kotoshu/keyboard/layouts/devanagari_inscript.rb +52 -0
  292. data/lib/kotoshu/keyboard/layouts/dubeolsik.rb +56 -0
  293. data/lib/kotoshu/keyboard/layouts/dvorak.rb +54 -0
  294. data/lib/kotoshu/keyboard/layouts/greek.rb +52 -0
  295. data/lib/kotoshu/keyboard/layouts/hebrew.rb +57 -0
  296. data/lib/kotoshu/keyboard/layouts/hrsl.rb +94 -0
  297. data/lib/kotoshu/keyboard/layouts/jcuken.rb +57 -0
  298. data/lib/kotoshu/keyboard/layouts/latin.rb +266 -0
  299. data/lib/kotoshu/keyboard/layouts/persian.rb +58 -0
  300. data/lib/kotoshu/keyboard/layouts/qwerty.rb +52 -0
  301. data/lib/kotoshu/keyboard/layouts/qwertz.rb +55 -0
  302. data/lib/kotoshu/keyboard/layouts/serbian.rb +59 -0
  303. data/lib/kotoshu/keyboard/layouts/turkish_q.rb +52 -0
  304. data/lib/kotoshu/keyboard/layouts/ukrainian.rb +51 -0
  305. data/lib/kotoshu/keyboard/registry.rb +201 -0
  306. data/lib/kotoshu/keyboard.rb +96 -0
  307. data/lib/kotoshu/kotoshu_native.so +0 -0
  308. data/lib/kotoshu/language/detector.rb +242 -0
  309. data/lib/kotoshu/language/identifier.rb +376 -0
  310. data/lib/kotoshu/language/languages/base.rb +271 -0
  311. data/lib/kotoshu/language/lid_detector.rb +188 -0
  312. data/lib/kotoshu/language/normalizer/arabic.rb +113 -0
  313. data/lib/kotoshu/language/normalizer/base.rb +133 -0
  314. data/lib/kotoshu/language/normalizer/greek.rb +63 -0
  315. data/lib/kotoshu/language/normalizer/hebrew.rb +72 -0
  316. data/lib/kotoshu/language/normalizer/persian.rb +36 -0
  317. data/lib/kotoshu/language/normalizer/turkish.rb +38 -0
  318. data/lib/kotoshu/language/registry.rb +209 -0
  319. data/lib/kotoshu/language/resources/ar/common_words.txt +6753 -0
  320. data/lib/kotoshu/language/resources/ar/confusion_sets.txt +11 -0
  321. data/lib/kotoshu/language/resources/de/common_words.txt +10003 -0
  322. data/lib/kotoshu/language/resources/de/confusion_sets.txt +246 -0
  323. data/lib/kotoshu/language/resources/en/common_words.txt +9979 -0
  324. data/lib/kotoshu/language/resources/en/confusion_sets.txt +871 -0
  325. data/lib/kotoshu/language/resources/es/common_words.txt +9992 -0
  326. data/lib/kotoshu/language/resources/es/confusion_sets.txt +17 -0
  327. data/lib/kotoshu/language/resources/fr/common_words.txt +9993 -0
  328. data/lib/kotoshu/language/resources/fr/confusion_sets.txt +76 -0
  329. data/lib/kotoshu/language/resources/pt/common_words.txt +9977 -0
  330. data/lib/kotoshu/language/resources/pt/confusion_sets.txt +18 -0
  331. data/lib/kotoshu/language/resources/ru/common_words.txt +9951 -0
  332. data/lib/kotoshu/language/resources/ru/confusion_sets.txt +5 -0
  333. data/lib/kotoshu/language/script.rb +128 -0
  334. data/lib/kotoshu/language/segmenter.rb +122 -0
  335. data/lib/kotoshu/language/suika.rb +38 -0
  336. data/lib/kotoshu/language/tokenizer/base.rb +187 -0
  337. data/lib/kotoshu/language/tokenizer/cyrillic_tokenizer.rb +76 -0
  338. data/lib/kotoshu/language/tokenizer/devanagari_tokenizer.rb +70 -0
  339. data/lib/kotoshu/language/tokenizer/french_tokenizer.rb +172 -0
  340. data/lib/kotoshu/language/tokenizer/german_tokenizer.rb +41 -0
  341. data/lib/kotoshu/language/tokenizer/greek_tokenizer.rb +72 -0
  342. data/lib/kotoshu/language/tokenizer/hangul_tokenizer.rb +70 -0
  343. data/lib/kotoshu/language/tokenizer/japanese_tokenizer.rb +55 -0
  344. data/lib/kotoshu/language/tokenizer/latin_tokenizer.rb +153 -0
  345. data/lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb +160 -0
  346. data/lib/kotoshu/language/tokenizer/russian_tokenizer.rb +93 -0
  347. data/lib/kotoshu/language/tokenizer/script_tokenizer.rb +62 -0
  348. data/lib/kotoshu/language/tokenizer/spanish_tokenizer.rb +122 -0
  349. data/lib/kotoshu/language.rb +127 -0
  350. data/lib/kotoshu/languages/ar/language.rb +98 -0
  351. data/lib/kotoshu/languages/bg/language.rb +37 -0
  352. data/lib/kotoshu/languages/ca/language.rb +22 -0
  353. data/lib/kotoshu/languages/cs/language.rb +22 -0
  354. data/lib/kotoshu/languages/da/language.rb +22 -0
  355. data/lib/kotoshu/languages/de/language.rb +554 -0
  356. data/lib/kotoshu/languages/el/language.rb +50 -0
  357. data/lib/kotoshu/languages/en/language.rb +470 -0
  358. data/lib/kotoshu/languages/es/language.rb +476 -0
  359. data/lib/kotoshu/languages/et/language.rb +24 -0
  360. data/lib/kotoshu/languages/fa/language.rb +91 -0
  361. data/lib/kotoshu/languages/fr/language.rb +505 -0
  362. data/lib/kotoshu/languages/he/language.rb +98 -0
  363. data/lib/kotoshu/languages/hr/language.rb +24 -0
  364. data/lib/kotoshu/languages/hu/language.rb +22 -0
  365. data/lib/kotoshu/languages/id/language.rb +25 -0
  366. data/lib/kotoshu/languages/it/language.rb +22 -0
  367. data/lib/kotoshu/languages/ja/language.rb +476 -0
  368. data/lib/kotoshu/languages/ko/language.rb +56 -0
  369. data/lib/kotoshu/languages/latin_base.rb +53 -0
  370. data/lib/kotoshu/languages/lt/language.rb +24 -0
  371. data/lib/kotoshu/languages/lv/language.rb +24 -0
  372. data/lib/kotoshu/languages/nb/language.rb +25 -0
  373. data/lib/kotoshu/languages/ne/language.rb +60 -0
  374. data/lib/kotoshu/languages/nl/language.rb +22 -0
  375. data/lib/kotoshu/languages/nn/language.rb +26 -0
  376. data/lib/kotoshu/languages/pl/language.rb +22 -0
  377. data/lib/kotoshu/languages/pt/language.rb +439 -0
  378. data/lib/kotoshu/languages/ro/language.rb +22 -0
  379. data/lib/kotoshu/languages/ru/language.rb +420 -0
  380. data/lib/kotoshu/languages/sk/language.rb +25 -0
  381. data/lib/kotoshu/languages/sl/language.rb +25 -0
  382. data/lib/kotoshu/languages/sr/language.rb +39 -0
  383. data/lib/kotoshu/languages/sr-Latn/language.rb +35 -0
  384. data/lib/kotoshu/languages/sv/language.rb +22 -0
  385. data/lib/kotoshu/languages/tr/language.rb +36 -0
  386. data/lib/kotoshu/languages/uk/language.rb +38 -0
  387. data/lib/kotoshu/languages/vi/language.rb +22 -0
  388. data/lib/kotoshu/languages.rb +64 -0
  389. data/lib/kotoshu/metrics_collector.rb +238 -0
  390. data/lib/kotoshu/metrics_module.rb +121 -0
  391. data/lib/kotoshu/models/context.rb +119 -0
  392. data/lib/kotoshu/models/embedding_model.rb +181 -0
  393. data/lib/kotoshu/models/fasttext_model.rb +212 -0
  394. data/lib/kotoshu/models/nearest_neighbor.rb +96 -0
  395. data/lib/kotoshu/models/onnx_model.rb +330 -0
  396. data/lib/kotoshu/models/semantic_error.rb +204 -0
  397. data/lib/kotoshu/models/suggestion.rb +106 -0
  398. data/lib/kotoshu/models/word_embedding.rb +97 -0
  399. data/lib/kotoshu/models.rb +27 -0
  400. data/lib/kotoshu/multi_language_checker.rb +127 -0
  401. data/lib/kotoshu/native.rb +83 -0
  402. data/lib/kotoshu/native_backend.rb +131 -0
  403. data/lib/kotoshu/paths.rb +79 -0
  404. data/lib/kotoshu/personal_dictionary.rb +105 -0
  405. data/lib/kotoshu/plugins/plugin.rb +61 -0
  406. data/lib/kotoshu/plugins/registry.rb +118 -0
  407. data/lib/kotoshu/plugins.rb +9 -0
  408. data/lib/kotoshu/project_config.rb +76 -0
  409. data/lib/kotoshu/readers/aff_data.rb +530 -0
  410. data/lib/kotoshu/readers/aff_reader.rb +393 -0
  411. data/lib/kotoshu/readers/condition_checker.rb +125 -0
  412. data/lib/kotoshu/readers/dic_reader.rb +208 -0
  413. data/lib/kotoshu/readers/file_reader.rb +377 -0
  414. data/lib/kotoshu/readers/lookup_builder.rb +584 -0
  415. data/lib/kotoshu/readers/readers.rb +17 -0
  416. data/lib/kotoshu/readers.rb +25 -0
  417. data/lib/kotoshu/resource_bundle.rb +55 -0
  418. data/lib/kotoshu/resource_manager.rb +600 -0
  419. data/lib/kotoshu/results/result.rb +165 -0
  420. data/lib/kotoshu/results.rb +8 -0
  421. data/lib/kotoshu/rspec.rb +163 -0
  422. data/lib/kotoshu/scripts/fasttext_to_onnx.py +275 -0
  423. data/lib/kotoshu/source_registry.rb +104 -0
  424. data/lib/kotoshu/spellchecker/parallel_checker.rb +90 -0
  425. data/lib/kotoshu/spellchecker.rb +406 -0
  426. data/lib/kotoshu/string_metrics.rb +153 -0
  427. data/lib/kotoshu/suggestions/context.rb +43 -0
  428. data/lib/kotoshu/suggestions/frequency_provider.rb +80 -0
  429. data/lib/kotoshu/suggestions/generator.rb +169 -0
  430. data/lib/kotoshu/suggestions/pipeline.rb +135 -0
  431. data/lib/kotoshu/suggestions/semantic_cascade.rb +73 -0
  432. data/lib/kotoshu/suggestions/strategies/base_strategy.rb +315 -0
  433. data/lib/kotoshu/suggestions/strategies/composite_strategy.rb +197 -0
  434. data/lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb +631 -0
  435. data/lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb +207 -0
  436. data/lib/kotoshu/suggestions/strategies/ngram_strategy.rb +165 -0
  437. data/lib/kotoshu/suggestions/strategies/phonetic_strategy.rb +284 -0
  438. data/lib/kotoshu/suggestions/strategies/semantic_strategy.rb +317 -0
  439. data/lib/kotoshu/suggestions/strategies/symspell_strategy.rb +270 -0
  440. data/lib/kotoshu/suggestions/strategies/word_index.rb +78 -0
  441. data/lib/kotoshu/suggestions/suggestion.rb +154 -0
  442. data/lib/kotoshu/suggestions/suggestion_set.rb +270 -0
  443. data/lib/kotoshu/suggestions/sweep_index.rb +137 -0
  444. data/lib/kotoshu/suggestions.rb +29 -0
  445. data/lib/kotoshu/tasks/check_task.rb +104 -0
  446. data/lib/kotoshu/tasks.rb +11 -0
  447. data/lib/kotoshu/validators/spelling_validator.rb +101 -0
  448. data/lib/kotoshu/validators.rb +9 -0
  449. data/lib/kotoshu/version.rb +5 -0
  450. data/lib/kotoshu.rb +538 -0
  451. data/rakelib/compile.rake +10 -0
  452. data/script/validate_all_dictionaries.rb +444 -0
  453. data/scripts/build_native_gem.rb +45 -0
  454. data/scripts/extract_eval_grids.rb +37 -0
  455. data/scripts/generate_frozen_tiers.rb +44 -0
  456. data/sig/kotoshu.rbs +435 -0
  457. data/test_oop.rb +79 -0
  458. metadata +546 -0
@@ -0,0 +1,1154 @@
1
+ {
2
+ "provenance": {
3
+ "source": "kotoshu-rs kotoshu/src/suggest/frequency_data.rs",
4
+ "upstream": "kotoshu/frequency-list-kelly data/en.json",
5
+ "upstream_sha256": "97535823f41aa3f06c2087dfc162aec0a591412b8b95c0594fcbb2cea79b0e61"
6
+ },
7
+ "tiers": {
8
+ "top_50": [
9
+ "British",
10
+ "all",
11
+ "also",
12
+ "an",
13
+ "and",
14
+ "as",
15
+ "at",
16
+ "be",
17
+ "but",
18
+ "by",
19
+ "can",
20
+ "do",
21
+ "for",
22
+ "from",
23
+ "have",
24
+ "he",
25
+ "his",
26
+ "if",
27
+ "in",
28
+ "it",
29
+ "make",
30
+ "not",
31
+ "of",
32
+ "on",
33
+ "one",
34
+ "or",
35
+ "other",
36
+ "see",
37
+ "take",
38
+ "that",
39
+ "the",
40
+ "their",
41
+ "there",
42
+ "they",
43
+ "this",
44
+ "time",
45
+ "to",
46
+ "use",
47
+ "we",
48
+ "which",
49
+ "who",
50
+ "will",
51
+ "with",
52
+ "would",
53
+ "year",
54
+ "you",
55
+ "your"
56
+ ],
57
+ "top_200": [
58
+ "British",
59
+ "England",
60
+ "English",
61
+ "European",
62
+ "London",
63
+ "Scotland",
64
+ "able",
65
+ "about",
66
+ "after",
67
+ "again",
68
+ "against",
69
+ "all",
70
+ "also",
71
+ "always",
72
+ "an",
73
+ "and",
74
+ "another",
75
+ "any",
76
+ "around",
77
+ "as",
78
+ "ask",
79
+ "at",
80
+ "back",
81
+ "be",
82
+ "because",
83
+ "before",
84
+ "between",
85
+ "both",
86
+ "bring",
87
+ "but",
88
+ "by",
89
+ "call",
90
+ "can",
91
+ "child",
92
+ "come",
93
+ "could",
94
+ "country",
95
+ "day",
96
+ "do",
97
+ "each",
98
+ "end",
99
+ "even",
100
+ "every",
101
+ "feel",
102
+ "find",
103
+ "first",
104
+ "for",
105
+ "from",
106
+ "full",
107
+ "get",
108
+ "give",
109
+ "go",
110
+ "good",
111
+ "great",
112
+ "have",
113
+ "he",
114
+ "help",
115
+ "her",
116
+ "here",
117
+ "high",
118
+ "his",
119
+ "hold",
120
+ "home",
121
+ "how",
122
+ "hundred",
123
+ "if",
124
+ "important",
125
+ "in",
126
+ "include",
127
+ "information",
128
+ "into",
129
+ "it",
130
+ "its",
131
+ "just",
132
+ "keep",
133
+ "know",
134
+ "last",
135
+ "leave",
136
+ "life",
137
+ "like",
138
+ "long",
139
+ "look",
140
+ "make",
141
+ "man",
142
+ "many",
143
+ "me",
144
+ "mean",
145
+ "meet",
146
+ "might",
147
+ "month",
148
+ "more",
149
+ "most",
150
+ "much",
151
+ "must",
152
+ "my",
153
+ "need",
154
+ "new",
155
+ "next",
156
+ "no",
157
+ "not",
158
+ "now",
159
+ "number",
160
+ "of",
161
+ "old",
162
+ "on",
163
+ "one",
164
+ "only",
165
+ "or",
166
+ "other",
167
+ "our",
168
+ "out",
169
+ "over",
170
+ "own",
171
+ "part",
172
+ "people",
173
+ "place",
174
+ "play",
175
+ "please",
176
+ "point",
177
+ "possible",
178
+ "problem",
179
+ "provide",
180
+ "put",
181
+ "run",
182
+ "same",
183
+ "say",
184
+ "see",
185
+ "seem",
186
+ "she",
187
+ "should",
188
+ "show",
189
+ "small",
190
+ "so",
191
+ "some",
192
+ "start",
193
+ "still",
194
+ "such",
195
+ "take",
196
+ "tell",
197
+ "than",
198
+ "that",
199
+ "the",
200
+ "their",
201
+ "then",
202
+ "there",
203
+ "these",
204
+ "they",
205
+ "thing",
206
+ "think",
207
+ "this",
208
+ "those",
209
+ "three",
210
+ "through",
211
+ "time",
212
+ "to",
213
+ "too",
214
+ "try",
215
+ "two",
216
+ "under",
217
+ "up",
218
+ "us",
219
+ "use",
220
+ "very",
221
+ "want",
222
+ "way",
223
+ "we",
224
+ "week",
225
+ "well",
226
+ "what",
227
+ "when",
228
+ "where",
229
+ "which",
230
+ "while",
231
+ "who",
232
+ "why",
233
+ "will",
234
+ "with",
235
+ "without",
236
+ "work",
237
+ "would",
238
+ "write",
239
+ "year",
240
+ "yes",
241
+ "you",
242
+ "your"
243
+ ],
244
+ "top_1000": [
245
+ "African",
246
+ "America",
247
+ "American",
248
+ "April",
249
+ "August",
250
+ "Australia",
251
+ "Britain",
252
+ "British",
253
+ "Canada",
254
+ "China",
255
+ "Christ",
256
+ "Christian",
257
+ "Christmas",
258
+ "December",
259
+ "England",
260
+ "English",
261
+ "Europe",
262
+ "European",
263
+ "February",
264
+ "French",
265
+ "Friday",
266
+ "German",
267
+ "Germany",
268
+ "I",
269
+ "Internet",
270
+ "Ireland",
271
+ "Irish",
272
+ "Israel",
273
+ "January",
274
+ "Jesus",
275
+ "July",
276
+ "June",
277
+ "London",
278
+ "March",
279
+ "May",
280
+ "Monday",
281
+ "November",
282
+ "October",
283
+ "Saturday",
284
+ "Scotland",
285
+ "Scottish",
286
+ "September",
287
+ "Sunday",
288
+ "Thursday",
289
+ "Tuesday",
290
+ "United Kingdom",
291
+ "United States",
292
+ "Wales",
293
+ "Wednesday",
294
+ "a",
295
+ "ability",
296
+ "able",
297
+ "about",
298
+ "above",
299
+ "accept",
300
+ "accord",
301
+ "account",
302
+ "across",
303
+ "act",
304
+ "action",
305
+ "actually",
306
+ "add",
307
+ "addition",
308
+ "additional",
309
+ "address",
310
+ "advice",
311
+ "affect",
312
+ "after",
313
+ "again",
314
+ "against",
315
+ "age",
316
+ "ago",
317
+ "agree",
318
+ "air",
319
+ "all",
320
+ "allow",
321
+ "almost",
322
+ "alone",
323
+ "along",
324
+ "already",
325
+ "also",
326
+ "although",
327
+ "always",
328
+ "among",
329
+ "amount",
330
+ "an",
331
+ "analysis",
332
+ "and",
333
+ "animal",
334
+ "another",
335
+ "answer",
336
+ "any",
337
+ "anyone",
338
+ "anything",
339
+ "apply",
340
+ "area",
341
+ "around",
342
+ "arrange",
343
+ "arrive",
344
+ "art",
345
+ "as",
346
+ "ask",
347
+ "assessment",
348
+ "assist",
349
+ "associate",
350
+ "at",
351
+ "attention",
352
+ "author",
353
+ "aware",
354
+ "away",
355
+ "back",
356
+ "background",
357
+ "bad",
358
+ "bag",
359
+ "ball",
360
+ "band",
361
+ "bar",
362
+ "basic",
363
+ "bath",
364
+ "bathroom",
365
+ "be",
366
+ "bear",
367
+ "beautiful",
368
+ "because",
369
+ "become",
370
+ "bed",
371
+ "bedroom",
372
+ "before",
373
+ "begin",
374
+ "behind",
375
+ "believe",
376
+ "below",
377
+ "between",
378
+ "beyond",
379
+ "big",
380
+ "bird",
381
+ "birthday",
382
+ "bit",
383
+ "black",
384
+ "blue",
385
+ "board",
386
+ "body",
387
+ "book",
388
+ "both",
389
+ "bottle",
390
+ "box",
391
+ "boy",
392
+ "break",
393
+ "breakfast",
394
+ "bring",
395
+ "brother",
396
+ "brown",
397
+ "build",
398
+ "building",
399
+ "bus",
400
+ "business",
401
+ "but",
402
+ "buy",
403
+ "by",
404
+ "bye",
405
+ "call",
406
+ "can",
407
+ "car",
408
+ "card",
409
+ "care",
410
+ "carry",
411
+ "case",
412
+ "cat",
413
+ "catch",
414
+ "cause",
415
+ "central",
416
+ "centre",
417
+ "certain",
418
+ "certainly",
419
+ "chair",
420
+ "challenge",
421
+ "chance",
422
+ "change",
423
+ "character",
424
+ "charge",
425
+ "check",
426
+ "child",
427
+ "chip",
428
+ "choice",
429
+ "choose",
430
+ "church",
431
+ "circumstances",
432
+ "city",
433
+ "claim",
434
+ "clean",
435
+ "clear",
436
+ "client",
437
+ "close",
438
+ "clothes",
439
+ "club",
440
+ "coffee",
441
+ "collect",
442
+ "colour",
443
+ "combine",
444
+ "come",
445
+ "comment",
446
+ "common",
447
+ "communication",
448
+ "company",
449
+ "compare",
450
+ "complete",
451
+ "completely",
452
+ "computer",
453
+ "contain",
454
+ "content",
455
+ "continue",
456
+ "contract",
457
+ "contribute",
458
+ "control",
459
+ "conversation",
460
+ "cook",
461
+ "copy",
462
+ "cost",
463
+ "could",
464
+ "country",
465
+ "couple",
466
+ "course",
467
+ "cover",
468
+ "cup",
469
+ "currently",
470
+ "customer",
471
+ "cut",
472
+ "daddy",
473
+ "darling",
474
+ "data",
475
+ "date",
476
+ "day",
477
+ "dead",
478
+ "deal",
479
+ "death",
480
+ "decide",
481
+ "decision",
482
+ "deep",
483
+ "define",
484
+ "degree",
485
+ "deliver",
486
+ "demonstrate",
487
+ "department",
488
+ "depend",
489
+ "describe",
490
+ "despite",
491
+ "detailed",
492
+ "die",
493
+ "difference",
494
+ "difficult",
495
+ "difficulty",
496
+ "dinner",
497
+ "direction",
498
+ "directly",
499
+ "discover",
500
+ "discussion",
501
+ "do",
502
+ "doctor",
503
+ "door",
504
+ "down",
505
+ "draw",
506
+ "drink",
507
+ "drive",
508
+ "dry",
509
+ "during",
510
+ "each",
511
+ "ear",
512
+ "early",
513
+ "easily",
514
+ "easy",
515
+ "economic",
516
+ "education",
517
+ "effort",
518
+ "egg",
519
+ "eight",
520
+ "either",
521
+ "else",
522
+ "email",
523
+ "enable",
524
+ "end",
525
+ "energy",
526
+ "enjoy",
527
+ "enough",
528
+ "ensure",
529
+ "enter",
530
+ "entry",
531
+ "equipment",
532
+ "especially",
533
+ "essential",
534
+ "establish",
535
+ "even",
536
+ "evening",
537
+ "ever",
538
+ "every",
539
+ "everybody",
540
+ "everyone",
541
+ "everything",
542
+ "everywhere",
543
+ "exactly",
544
+ "example",
545
+ "excellent",
546
+ "existing",
547
+ "expect",
548
+ "expensive",
549
+ "explain",
550
+ "explore",
551
+ "express",
552
+ "extra",
553
+ "extremely",
554
+ "eye",
555
+ "face",
556
+ "fact",
557
+ "fail",
558
+ "fall",
559
+ "family",
560
+ "father",
561
+ "feel",
562
+ "few",
563
+ "field",
564
+ "figure",
565
+ "file",
566
+ "fill",
567
+ "film",
568
+ "find",
569
+ "fine",
570
+ "finger",
571
+ "finish",
572
+ "first",
573
+ "fish",
574
+ "fit",
575
+ "five",
576
+ "floor",
577
+ "flower",
578
+ "focus",
579
+ "follow",
580
+ "food",
581
+ "foot",
582
+ "football",
583
+ "for",
584
+ "forget",
585
+ "former",
586
+ "forward",
587
+ "four",
588
+ "free",
589
+ "friend",
590
+ "from",
591
+ "front",
592
+ "fruit",
593
+ "full",
594
+ "fun",
595
+ "funding",
596
+ "funny",
597
+ "game",
598
+ "garden",
599
+ "general",
600
+ "generally",
601
+ "get",
602
+ "girl",
603
+ "give",
604
+ "glass",
605
+ "go",
606
+ "goal",
607
+ "good",
608
+ "great",
609
+ "green",
610
+ "ground",
611
+ "group",
612
+ "grow",
613
+ "growth",
614
+ "hair",
615
+ "half",
616
+ "hand",
617
+ "happen",
618
+ "happy",
619
+ "hard",
620
+ "hate",
621
+ "have",
622
+ "he",
623
+ "head",
624
+ "health",
625
+ "hear",
626
+ "heart",
627
+ "hello",
628
+ "help",
629
+ "her",
630
+ "here",
631
+ "hers",
632
+ "high",
633
+ "highly",
634
+ "him",
635
+ "himself",
636
+ "his",
637
+ "history",
638
+ "hold",
639
+ "holiday",
640
+ "home",
641
+ "hope",
642
+ "horse",
643
+ "hot",
644
+ "hour",
645
+ "house",
646
+ "how",
647
+ "huge",
648
+ "human",
649
+ "hundred",
650
+ "husband",
651
+ "idea",
652
+ "if",
653
+ "immediately",
654
+ "impact",
655
+ "importance",
656
+ "important",
657
+ "improve",
658
+ "improvement",
659
+ "in",
660
+ "include",
661
+ "indeed",
662
+ "information",
663
+ "initiative",
664
+ "instead",
665
+ "interest",
666
+ "interested",
667
+ "interesting",
668
+ "into",
669
+ "invite",
670
+ "it",
671
+ "its",
672
+ "itself",
673
+ "job",
674
+ "join",
675
+ "just",
676
+ "keep",
677
+ "key",
678
+ "kind",
679
+ "kitchen",
680
+ "know",
681
+ "land",
682
+ "language",
683
+ "large",
684
+ "last",
685
+ "late",
686
+ "laugh",
687
+ "launch",
688
+ "law",
689
+ "leader",
690
+ "learn",
691
+ "least",
692
+ "leave",
693
+ "leg",
694
+ "less",
695
+ "let",
696
+ "letter",
697
+ "lie",
698
+ "life",
699
+ "light",
700
+ "like",
701
+ "likely",
702
+ "line",
703
+ "list",
704
+ "listen",
705
+ "little",
706
+ "live",
707
+ "locate",
708
+ "location",
709
+ "long",
710
+ "longer",
711
+ "look",
712
+ "lose",
713
+ "lot",
714
+ "love",
715
+ "lovely",
716
+ "low",
717
+ "lucky",
718
+ "lunch",
719
+ "machine",
720
+ "main",
721
+ "maintain",
722
+ "make",
723
+ "man",
724
+ "manage",
725
+ "manager",
726
+ "many",
727
+ "market",
728
+ "matter",
729
+ "may",
730
+ "maybe",
731
+ "me",
732
+ "meal",
733
+ "mean",
734
+ "means",
735
+ "measure",
736
+ "meat",
737
+ "media",
738
+ "meet",
739
+ "meeting",
740
+ "member",
741
+ "mention",
742
+ "message",
743
+ "middle",
744
+ "might",
745
+ "mile",
746
+ "milk",
747
+ "million",
748
+ "mind",
749
+ "minute",
750
+ "miss",
751
+ "model",
752
+ "modern",
753
+ "moment",
754
+ "money",
755
+ "month",
756
+ "more",
757
+ "morning",
758
+ "most",
759
+ "mother",
760
+ "mouth",
761
+ "move",
762
+ "much",
763
+ "mummy",
764
+ "music",
765
+ "must",
766
+ "my",
767
+ "myself",
768
+ "name",
769
+ "national",
770
+ "natural",
771
+ "near",
772
+ "nearly",
773
+ "necessary",
774
+ "neck",
775
+ "need",
776
+ "network",
777
+ "never",
778
+ "new",
779
+ "news",
780
+ "next",
781
+ "nice",
782
+ "night",
783
+ "nine",
784
+ "no",
785
+ "nobody",
786
+ "normal",
787
+ "not",
788
+ "note",
789
+ "nothing",
790
+ "now",
791
+ "number",
792
+ "o'clock",
793
+ "objective",
794
+ "obtain",
795
+ "of",
796
+ "off",
797
+ "offer",
798
+ "office",
799
+ "officer",
800
+ "often",
801
+ "old",
802
+ "on",
803
+ "once",
804
+ "one",
805
+ "online",
806
+ "only",
807
+ "onto",
808
+ "open",
809
+ "operation",
810
+ "opportunity",
811
+ "or",
812
+ "order",
813
+ "organisation",
814
+ "organise",
815
+ "original",
816
+ "other",
817
+ "our",
818
+ "out",
819
+ "outside",
820
+ "over",
821
+ "own",
822
+ "page",
823
+ "paper",
824
+ "pardon",
825
+ "parent",
826
+ "park",
827
+ "part",
828
+ "particularly",
829
+ "partner",
830
+ "partnership",
831
+ "party",
832
+ "pass",
833
+ "past",
834
+ "pay",
835
+ "people",
836
+ "per",
837
+ "percent",
838
+ "perhaps",
839
+ "person",
840
+ "phone",
841
+ "pick",
842
+ "picture",
843
+ "piece",
844
+ "place",
845
+ "plan",
846
+ "plate",
847
+ "play",
848
+ "player",
849
+ "please",
850
+ "point",
851
+ "poor",
852
+ "popular",
853
+ "position",
854
+ "possible",
855
+ "post",
856
+ "potential",
857
+ "pound",
858
+ "power",
859
+ "prepare",
860
+ "present",
861
+ "pressure",
862
+ "pretty",
863
+ "price",
864
+ "principle",
865
+ "private",
866
+ "probably",
867
+ "problem",
868
+ "programme",
869
+ "progress",
870
+ "promote",
871
+ "property",
872
+ "proposal",
873
+ "propose",
874
+ "prove",
875
+ "provide",
876
+ "provision",
877
+ "publication",
878
+ "publish",
879
+ "pull",
880
+ "put",
881
+ "question",
882
+ "quickly",
883
+ "quiet",
884
+ "quite",
885
+ "rain",
886
+ "rate",
887
+ "rather",
888
+ "reach",
889
+ "read",
890
+ "reading",
891
+ "ready",
892
+ "real",
893
+ "realise",
894
+ "really",
895
+ "reason",
896
+ "recently",
897
+ "record",
898
+ "red",
899
+ "reflect",
900
+ "regular",
901
+ "relationship",
902
+ "relevant",
903
+ "remember",
904
+ "replace",
905
+ "requirement",
906
+ "respect",
907
+ "response",
908
+ "rest",
909
+ "right",
910
+ "ring",
911
+ "road",
912
+ "room",
913
+ "round",
914
+ "route",
915
+ "rule",
916
+ "run",
917
+ "sad",
918
+ "safe",
919
+ "sale",
920
+ "same",
921
+ "save",
922
+ "say",
923
+ "school",
924
+ "science",
925
+ "season",
926
+ "second",
927
+ "security",
928
+ "see",
929
+ "seek",
930
+ "seem",
931
+ "sell",
932
+ "send",
933
+ "sense",
934
+ "serious",
935
+ "serve",
936
+ "session",
937
+ "set",
938
+ "seven",
939
+ "shall",
940
+ "share",
941
+ "she",
942
+ "shoe",
943
+ "shop",
944
+ "short",
945
+ "should",
946
+ "show",
947
+ "side",
948
+ "sign",
949
+ "significant",
950
+ "similar",
951
+ "simple",
952
+ "since",
953
+ "sing",
954
+ "single",
955
+ "sister",
956
+ "sit",
957
+ "site",
958
+ "situation",
959
+ "six",
960
+ "size",
961
+ "sleep",
962
+ "small",
963
+ "so",
964
+ "society",
965
+ "software",
966
+ "solution",
967
+ "some",
968
+ "someone",
969
+ "something",
970
+ "sometimes",
971
+ "somewhere",
972
+ "soon",
973
+ "sorry",
974
+ "sort",
975
+ "sound",
976
+ "space",
977
+ "speak",
978
+ "special",
979
+ "spend",
980
+ "stage",
981
+ "stand",
982
+ "start",
983
+ "station",
984
+ "stay",
985
+ "step",
986
+ "still",
987
+ "stop",
988
+ "story",
989
+ "strategy",
990
+ "street",
991
+ "strong",
992
+ "student",
993
+ "study",
994
+ "stuff",
995
+ "style",
996
+ "success",
997
+ "successful",
998
+ "such",
999
+ "suffer",
1000
+ "suggest",
1001
+ "suitable",
1002
+ "summer",
1003
+ "sun",
1004
+ "sure",
1005
+ "survey",
1006
+ "system",
1007
+ "table",
1008
+ "take",
1009
+ "talk",
1010
+ "tall",
1011
+ "task",
1012
+ "taxi",
1013
+ "tea",
1014
+ "teach",
1015
+ "teacher",
1016
+ "teaching",
1017
+ "team",
1018
+ "technical",
1019
+ "technique",
1020
+ "technology",
1021
+ "tell",
1022
+ "ten",
1023
+ "test",
1024
+ "text",
1025
+ "than",
1026
+ "thank",
1027
+ "thanks",
1028
+ "that",
1029
+ "the",
1030
+ "their",
1031
+ "them",
1032
+ "themselves",
1033
+ "then",
1034
+ "there",
1035
+ "therefore",
1036
+ "these",
1037
+ "they",
1038
+ "thing",
1039
+ "think",
1040
+ "third",
1041
+ "this",
1042
+ "those",
1043
+ "though",
1044
+ "thought",
1045
+ "thousand",
1046
+ "three",
1047
+ "through",
1048
+ "thus",
1049
+ "time",
1050
+ "to",
1051
+ "today",
1052
+ "together",
1053
+ "toilet",
1054
+ "tomorrow",
1055
+ "too",
1056
+ "tool",
1057
+ "tooth",
1058
+ "top",
1059
+ "total",
1060
+ "towards",
1061
+ "town",
1062
+ "track",
1063
+ "travel",
1064
+ "treat",
1065
+ "trip",
1066
+ "trouble",
1067
+ "trousers",
1068
+ "true",
1069
+ "try",
1070
+ "turn",
1071
+ "two",
1072
+ "type",
1073
+ "under",
1074
+ "understand",
1075
+ "undertake",
1076
+ "unique",
1077
+ "unit",
1078
+ "university",
1079
+ "unless",
1080
+ "until",
1081
+ "up",
1082
+ "upon",
1083
+ "us",
1084
+ "use",
1085
+ "useful",
1086
+ "user",
1087
+ "usually",
1088
+ "vegetable",
1089
+ "very",
1090
+ "video",
1091
+ "view",
1092
+ "village",
1093
+ "visit",
1094
+ "voice",
1095
+ "wait",
1096
+ "walk",
1097
+ "wall",
1098
+ "want",
1099
+ "war",
1100
+ "wash",
1101
+ "watch",
1102
+ "water",
1103
+ "way",
1104
+ "we",
1105
+ "wear",
1106
+ "web",
1107
+ "website",
1108
+ "week",
1109
+ "weekend",
1110
+ "well",
1111
+ "wet",
1112
+ "what",
1113
+ "whatever",
1114
+ "when",
1115
+ "where",
1116
+ "whether",
1117
+ "which",
1118
+ "while",
1119
+ "white",
1120
+ "who",
1121
+ "whole",
1122
+ "whom",
1123
+ "whose",
1124
+ "why",
1125
+ "wide",
1126
+ "will",
1127
+ "win",
1128
+ "window",
1129
+ "wine",
1130
+ "wish",
1131
+ "with",
1132
+ "without",
1133
+ "woman",
1134
+ "word",
1135
+ "work",
1136
+ "world",
1137
+ "worry",
1138
+ "worth",
1139
+ "would",
1140
+ "write",
1141
+ "wrong",
1142
+ "year",
1143
+ "yellow",
1144
+ "yes",
1145
+ "yesterday",
1146
+ "yet",
1147
+ "you",
1148
+ "young",
1149
+ "your",
1150
+ "yours",
1151
+ "yourself"
1152
+ ]
1153
+ }
1154
+ }