rroonga 5.0.0-x64-mingw32 → 5.0.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (733) hide show
  1. checksums.yaml +8 -8
  2. data/.yardopts +1 -0
  3. data/Rakefile +1 -16
  4. data/example/bookmark.rb +1 -6
  5. data/example/index-html.rb +0 -1
  6. data/ext/groonga/extconf.rb +4 -7
  7. data/ext/groonga/rb-grn-array.c +1 -1
  8. data/ext/groonga/rb-grn-column.c +33 -67
  9. data/ext/groonga/rb-grn-context.c +5 -5
  10. data/ext/groonga/rb-grn-database.c +2 -2
  11. data/ext/groonga/rb-grn-double-array-trie.c +4 -2
  12. data/ext/groonga/rb-grn-encoding-support.c +7 -1
  13. data/ext/groonga/rb-grn-equal-operator.c +85 -0
  14. data/ext/groonga/rb-grn-exception.c +17 -0
  15. data/ext/groonga/rb-grn-expression.c +85 -43
  16. data/ext/groonga/rb-grn-greater-equal-operator.c +88 -0
  17. data/ext/groonga/rb-grn-greater-operator.c +85 -0
  18. data/ext/groonga/rb-grn-hash.c +1 -1
  19. data/ext/groonga/rb-grn-index-column.c +150 -11
  20. data/ext/groonga/rb-grn-less-equal-operator.c +88 -0
  21. data/ext/groonga/rb-grn-less-operator.c +85 -0
  22. data/ext/groonga/rb-grn-logger.c +5 -5
  23. data/ext/groonga/rb-grn-match-operator.c +86 -0
  24. data/ext/groonga/rb-grn-normalizer.c +8 -1
  25. data/ext/groonga/rb-grn-not-equal-operator.c +85 -0
  26. data/ext/groonga/rb-grn-object.c +170 -36
  27. data/ext/groonga/rb-grn-operator.c +395 -172
  28. data/ext/groonga/rb-grn-patricia-trie.c +10 -8
  29. data/ext/groonga/rb-grn-plugin.c +51 -3
  30. data/ext/groonga/rb-grn-prefix-operator.c +86 -0
  31. data/ext/groonga/rb-grn-procedure-type.c +4 -0
  32. data/ext/groonga/rb-grn-query-logger.c +4 -4
  33. data/ext/groonga/rb-grn-regexp-operator.c +85 -0
  34. data/ext/groonga/rb-grn-snippet.c +1 -1
  35. data/ext/groonga/rb-grn-table-key-support.c +9 -5
  36. data/ext/groonga/rb-grn-table.c +52 -66
  37. data/ext/groonga/rb-grn-type.c +1 -1
  38. data/ext/groonga/rb-grn-utils.c +22 -3
  39. data/ext/groonga/rb-grn.h +31 -4
  40. data/ext/groonga/rb-groonga.c +9 -9
  41. data/lib/1.9/groonga.so +0 -0
  42. data/lib/2.0/groonga.so +0 -0
  43. data/lib/2.1/groonga.so +0 -0
  44. data/lib/2.2/groonga.so +0 -0
  45. data/lib/groonga/context.rb +31 -0
  46. data/lib/groonga/expression-builder.rb +14 -1
  47. data/lib/groonga/record.rb +10 -8
  48. data/lib/groonga/schema.rb +3 -1
  49. data/rroonga-build.rb +2 -2
  50. data/rroonga.gemspec +3 -3
  51. data/test/groonga-test-utils.rb +4 -0
  52. data/test/test-column.rb +28 -26
  53. data/test/test-exception.rb +1 -0
  54. data/test/test-expression-builder.rb +83 -1
  55. data/test/test-expression.rb +80 -48
  56. data/test/test-index-column.rb +102 -29
  57. data/test/test-normalizer.rb +35 -29
  58. data/test/test-operator.rb +214 -0
  59. data/test/test-plugin.rb +24 -6
  60. data/test/test-procedure.rb +29 -0
  61. data/test/test-schema-type.rb +14 -0
  62. data/test/test-table-select-mecab.rb +1 -4
  63. data/test/test-table.rb +7 -0
  64. data/test/test-token-regexp.rb +30 -0
  65. data/test/test-type.rb +24 -0
  66. data/vendor/local/bin/grndb.exe +0 -0
  67. data/vendor/local/bin/groonga-benchmark.exe +0 -0
  68. data/vendor/local/bin/groonga.exe +0 -0
  69. data/vendor/local/bin/libgcc_s_seh-1.dll +0 -0
  70. data/vendor/local/bin/libgroonga-0.dll +0 -0
  71. data/vendor/local/bin/libmecab-1.dll +0 -0
  72. data/vendor/local/bin/libmsgpack-3.dll +0 -0
  73. data/vendor/local/bin/libmsgpackc-2.dll +0 -0
  74. data/vendor/local/bin/libonig-5.dll +0 -0
  75. data/vendor/local/bin/libstdc++-6.dll +0 -0
  76. data/vendor/local/bin/lz4.exe +0 -0
  77. data/vendor/local/bin/lz4c.exe +0 -0
  78. data/vendor/local/bin/lz4cat +0 -0
  79. data/vendor/local/bin/mecab-config +2 -2
  80. data/vendor/local/bin/mecab.exe +0 -0
  81. data/vendor/local/bin/onig-config +1 -1
  82. data/vendor/local/bin/zlib1.dll +0 -0
  83. data/vendor/local/etc/groonga/groonga.conf +1 -1
  84. data/vendor/local/etc/groonga/httpd/groonga-httpd.conf +2 -2
  85. data/vendor/local/include/groonga/groonga/expr.h +2 -0
  86. data/vendor/local/include/groonga/groonga/groonga.h +32 -5
  87. data/vendor/local/include/groonga/groonga/ii.h +7 -0
  88. data/vendor/local/include/groonga/groonga/obj.h +37 -0
  89. data/vendor/local/include/groonga/groonga/scorer.h +95 -0
  90. data/vendor/local/include/groonga/groonga.h +1 -0
  91. data/vendor/local/lib/groonga/plugins/query_expanders/tsv.a +0 -0
  92. data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll +0 -0
  93. data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll.a +0 -0
  94. data/vendor/local/lib/groonga/plugins/query_expanders/tsv.la +2 -2
  95. data/vendor/local/lib/groonga/plugins/ruby/eval.a +0 -0
  96. data/vendor/local/lib/groonga/plugins/ruby/eval.dll +0 -0
  97. data/vendor/local/lib/groonga/plugins/ruby/eval.dll.a +0 -0
  98. data/vendor/local/lib/groonga/plugins/ruby/eval.la +2 -2
  99. data/vendor/local/lib/groonga/plugins/ruby/load.a +0 -0
  100. data/vendor/local/lib/groonga/plugins/ruby/load.dll +0 -0
  101. data/vendor/local/lib/groonga/plugins/ruby/load.dll.a +0 -0
  102. data/vendor/local/lib/groonga/plugins/ruby/load.la +2 -2
  103. data/vendor/local/lib/groonga/plugins/sharding/logical_count.rb +6 -3
  104. data/vendor/local/lib/groonga/plugins/sharding/logical_enumerator.rb +6 -5
  105. data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb +421 -17
  106. data/vendor/local/lib/groonga/plugins/suggest/suggest.a +0 -0
  107. data/vendor/local/lib/groonga/plugins/suggest/suggest.dll +0 -0
  108. data/vendor/local/lib/groonga/plugins/suggest/suggest.dll.a +0 -0
  109. data/vendor/local/lib/groonga/plugins/suggest/suggest.la +2 -2
  110. data/vendor/local/lib/groonga/plugins/table/table.a +0 -0
  111. data/vendor/local/lib/groonga/plugins/table/table.dll +0 -0
  112. data/vendor/local/lib/groonga/plugins/table/table.dll.a +0 -0
  113. data/vendor/local/lib/groonga/plugins/table/table.la +2 -2
  114. data/vendor/local/lib/groonga/plugins/token_filters/stop_word.a +0 -0
  115. data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll +0 -0
  116. data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll.a +0 -0
  117. data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la +2 -2
  118. data/vendor/local/lib/groonga/plugins/tokenizers/mecab.a +0 -0
  119. data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll +0 -0
  120. data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll.a +0 -0
  121. data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la +2 -2
  122. data/vendor/local/lib/groonga/scripts/ruby/backtrace_entry.rb +12 -4
  123. data/vendor/local/lib/groonga/scripts/ruby/database.rb +11 -3
  124. data/vendor/local/lib/groonga/scripts/ruby/expression.rb +23 -0
  125. data/vendor/local/lib/groonga/scripts/ruby/expression_size_estimator.rb +158 -0
  126. data/vendor/local/lib/groonga/scripts/ruby/index_column.rb +39 -0
  127. data/vendor/local/lib/groonga/scripts/ruby/initialize/post.rb +4 -0
  128. data/vendor/local/lib/groonga/scripts/ruby/initialize/pre.rb +2 -0
  129. data/vendor/local/lib/groonga/scripts/ruby/logger.rb +11 -7
  130. data/vendor/local/lib/groonga/scripts/ruby/object.rb +11 -0
  131. data/vendor/local/lib/groonga/scripts/ruby/operator.rb +22 -0
  132. data/vendor/local/lib/groonga/scripts/ruby/scan_info.rb +7 -2
  133. data/vendor/local/lib/groonga/scripts/ruby/scan_info_builder.rb +7 -11
  134. data/vendor/local/lib/groonga/scripts/ruby/scan_info_data.rb +137 -34
  135. data/vendor/local/lib/groonga/scripts/ruby/scan_info_search_index.rb +9 -0
  136. data/vendor/local/lib/libgroonga.a +0 -0
  137. data/vendor/local/lib/libgroonga.dll.a +0 -0
  138. data/vendor/local/lib/libgroonga.la +2 -2
  139. data/vendor/local/lib/liblz4.a +0 -0
  140. data/vendor/local/lib/liblz4.dll +0 -0
  141. data/vendor/local/lib/liblz4.dll.1 +0 -0
  142. data/vendor/local/lib/liblz4.dll.1.5.0 +0 -0
  143. data/vendor/local/lib/libmecab.a +0 -0
  144. data/vendor/local/lib/libmecab.dll.a +0 -0
  145. data/vendor/local/lib/libmecab.la +2 -2
  146. data/vendor/local/lib/libmsgpack.a +0 -0
  147. data/vendor/local/lib/libmsgpack.dll.a +0 -0
  148. data/vendor/local/lib/libmsgpack.la +2 -2
  149. data/vendor/local/lib/libmsgpackc.a +0 -0
  150. data/vendor/local/lib/libmsgpackc.dll.a +0 -0
  151. data/vendor/local/lib/libmsgpackc.la +2 -2
  152. data/vendor/local/lib/libonig.a +0 -0
  153. data/vendor/local/lib/libonig.dll.a +0 -0
  154. data/vendor/local/lib/libonig.la +2 -2
  155. data/vendor/local/lib/libz.a +0 -0
  156. data/vendor/local/lib/libz.dll.a +0 -0
  157. data/vendor/local/lib/pkgconfig/groonga.pc +3 -3
  158. data/vendor/local/lib/pkgconfig/liblz4.pc +5 -5
  159. data/vendor/local/lib/pkgconfig/msgpack.pc +1 -1
  160. data/vendor/local/lib/pkgconfig/oniguruma.pc +6 -6
  161. data/vendor/local/lib/pkgconfig/zlib.pc +3 -3
  162. data/vendor/local/libexec/mecab/mecab-cost-train.exe +0 -0
  163. data/vendor/local/libexec/mecab/mecab-dict-gen.exe +0 -0
  164. data/vendor/local/libexec/mecab/mecab-dict-index.exe +0 -0
  165. data/vendor/local/libexec/mecab/mecab-system-eval.exe +0 -0
  166. data/vendor/local/libexec/mecab/mecab-test-gen.exe +0 -0
  167. data/vendor/local/sbin/groonga-httpd-restart +1 -1
  168. data/vendor/local/sbin/groonga-httpd.exe +0 -0
  169. data/vendor/local/share/doc/groonga/en/html/.buildinfo +1 -1
  170. data/vendor/local/share/doc/groonga/en/html/_images/used-when-indexing.png +0 -0
  171. data/vendor/local/share/doc/groonga/en/html/_images/used-when-searching.png +0 -0
  172. data/vendor/local/share/doc/groonga/en/html/_sources/characteristic.txt +1 -1
  173. data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/release.txt +32 -17
  174. data/vendor/local/share/doc/groonga/en/html/_sources/install/centos.txt +3 -3
  175. data/vendor/local/share/doc/groonga/en/html/_sources/install/debian.txt +3 -3
  176. data/vendor/local/share/doc/groonga/en/html/_sources/install/fedora.txt +4 -4
  177. data/vendor/local/share/doc/groonga/en/html/_sources/install/mac_os_x.txt +3 -3
  178. data/vendor/local/share/doc/groonga/en/html/_sources/install/others.txt +3 -3
  179. data/vendor/local/share/doc/groonga/en/html/_sources/install/solaris.txt +3 -3
  180. data/vendor/local/share/doc/groonga/en/html/_sources/install/ubuntu.txt +3 -3
  181. data/vendor/local/share/doc/groonga/en/html/_sources/install/windows.txt +9 -9
  182. data/vendor/local/share/doc/groonga/en/html/_sources/news/1.0.x.txt +1 -1
  183. data/vendor/local/share/doc/groonga/en/html/_sources/news/3.x.txt +2 -2
  184. data/vendor/local/share/doc/groonga/en/html/_sources/news/4.x.txt +2 -2
  185. data/vendor/local/share/doc/groonga/en/html/_sources/news.txt +194 -0
  186. data/vendor/local/share/doc/groonga/en/html/_sources/reference/api/grn_ctx.txt +42 -0
  187. data/vendor/local/share/doc/groonga/en/html/_sources/reference/api/overview.txt +54 -0
  188. data/vendor/local/share/doc/groonga/en/html/_sources/reference/api.txt +3 -0
  189. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/cache_limit.txt +1 -1
  190. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_create.txt +2 -2
  191. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_list.txt +1 -1
  192. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_rename.txt +3 -3
  193. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/delete.txt +4 -4
  194. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/load.txt +5 -5
  195. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/lock_clear.txt +4 -4
  196. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_count.txt +173 -0
  197. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_range_filter.txt +112 -0
  198. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/normalize.txt +7 -6
  199. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/plugin_register.txt +64 -0
  200. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/plugin_unregister.txt +63 -0
  201. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/register.txt +11 -1
  202. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/request_cancel.txt +3 -2
  203. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/ruby_eval.txt +1 -1
  204. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/ruby_load.txt +1 -1
  205. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/select.txt +17 -17
  206. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/suggest.txt +12 -12
  207. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/table_create.txt +7 -7
  208. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/table_tokenize.txt +4 -4
  209. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/tokenize.txt +6 -6
  210. data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/truncate.txt +1 -1
  211. data/vendor/local/share/doc/groonga/en/html/_sources/reference/executables/groonga.txt +47 -26
  212. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/between.txt +5 -5
  213. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/geo_distance.txt +3 -3
  214. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/highlight_full.txt +6 -6
  215. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/highlight_html.txt +1 -1
  216. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/html_untag.txt +1 -1
  217. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/in_values.txt +54 -2
  218. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/query.txt +4 -4
  219. data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/sub_filter.txt +4 -4
  220. data/vendor/local/share/doc/groonga/en/html/_sources/reference/grn_expr/query_syntax.txt +44 -18
  221. data/vendor/local/share/doc/groonga/en/html/_sources/reference/grn_expr/script_syntax.txt +41 -11
  222. data/vendor/local/share/doc/groonga/en/html/_sources/reference/indexing.txt +2 -0
  223. data/vendor/local/share/doc/groonga/en/html/_sources/reference/normalizers.txt +4 -0
  224. data/vendor/local/share/doc/groonga/en/html/_sources/reference/operations.txt +2 -1
  225. data/vendor/local/share/doc/groonga/en/html/_sources/reference/regular_expression.txt +403 -0
  226. data/vendor/local/share/doc/groonga/en/html/_sources/reference/scorer.txt +217 -0
  227. data/vendor/local/share/doc/groonga/en/html/_sources/reference/scorers/scorer_tf_at_most.txt +22 -0
  228. data/vendor/local/share/doc/groonga/en/html/_sources/reference/scorers/scorer_tf_idf.txt +110 -0
  229. data/vendor/local/share/doc/groonga/en/html/_sources/reference/scoring_note.txt +13 -0
  230. data/vendor/local/share/doc/groonga/en/html/_sources/reference/tables.txt +8 -0
  231. data/vendor/local/share/doc/groonga/en/html/_sources/reference/tokenizers.txt +530 -16
  232. data/vendor/local/share/doc/groonga/en/html/_sources/reference.txt +2 -0
  233. data/vendor/local/share/doc/groonga/en/html/_sources/server/memcached.txt +15 -0
  234. data/vendor/local/share/doc/groonga/en/html/_sources/server.txt +2 -1
  235. data/vendor/local/share/doc/groonga/en/html/_sources/spec/gqtp.txt +66 -1
  236. data/vendor/local/share/doc/groonga/en/html/_sources/troubleshooting/different_results_with_the_same_keyword.txt +1 -1
  237. data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/network.txt +0 -81
  238. data/vendor/local/share/doc/groonga/en/html/_static/basic.css +6 -68
  239. data/vendor/local/share/doc/groonga/en/html/_static/doctools.js +1 -26
  240. data/vendor/local/share/doc/groonga/en/html/_static/down-pressed.png +0 -0
  241. data/vendor/local/share/doc/groonga/en/html/_static/down.png +0 -0
  242. data/vendor/local/share/doc/groonga/en/html/_static/file.png +0 -0
  243. data/vendor/local/share/doc/groonga/en/html/_static/jquery.js +9404 -4
  244. data/vendor/local/share/doc/groonga/en/html/_static/minus.png +0 -0
  245. data/vendor/local/share/doc/groonga/en/html/_static/plus.png +0 -0
  246. data/vendor/local/share/doc/groonga/en/html/_static/searchtools.js +2 -2
  247. data/vendor/local/share/doc/groonga/en/html/_static/underscore.js +1415 -31
  248. data/vendor/local/share/doc/groonga/en/html/_static/up-pressed.png +0 -0
  249. data/vendor/local/share/doc/groonga/en/html/_static/up.png +0 -0
  250. data/vendor/local/share/doc/groonga/en/html/_static/websupport.js +15 -15
  251. data/vendor/local/share/doc/groonga/en/html/characteristic.html +18 -20
  252. data/vendor/local/share/doc/groonga/en/html/client.html +22 -24
  253. data/vendor/local/share/doc/groonga/en/html/community.html +17 -19
  254. data/vendor/local/share/doc/groonga/en/html/contribution/development/com.html +17 -19
  255. data/vendor/local/share/doc/groonga/en/html/contribution/development/cooperation.html +17 -19
  256. data/vendor/local/share/doc/groonga/en/html/contribution/development/query.html +17 -19
  257. data/vendor/local/share/doc/groonga/en/html/contribution/development/release.html +51 -38
  258. data/vendor/local/share/doc/groonga/en/html/contribution/development/repository.html +17 -19
  259. data/vendor/local/share/doc/groonga/en/html/contribution/development/test.html +17 -19
  260. data/vendor/local/share/doc/groonga/en/html/contribution/development.html +17 -19
  261. data/vendor/local/share/doc/groonga/en/html/contribution/documentation/c-api.html +17 -19
  262. data/vendor/local/share/doc/groonga/en/html/contribution/documentation/i18n.html +17 -19
  263. data/vendor/local/share/doc/groonga/en/html/contribution/documentation/introduction.html +24 -26
  264. data/vendor/local/share/doc/groonga/en/html/contribution/documentation.html +17 -19
  265. data/vendor/local/share/doc/groonga/en/html/contribution/report.html +17 -19
  266. data/vendor/local/share/doc/groonga/en/html/contribution.html +18 -20
  267. data/vendor/local/share/doc/groonga/en/html/development/travis-ci.html +24 -26
  268. data/vendor/local/share/doc/groonga/en/html/development.html +17 -19
  269. data/vendor/local/share/doc/groonga/en/html/genindex.html +26 -14
  270. data/vendor/local/share/doc/groonga/en/html/index.html +150 -130
  271. data/vendor/local/share/doc/groonga/en/html/install/centos.html +28 -30
  272. data/vendor/local/share/doc/groonga/en/html/install/debian.html +24 -26
  273. data/vendor/local/share/doc/groonga/en/html/install/fedora.html +28 -30
  274. data/vendor/local/share/doc/groonga/en/html/install/mac_os_x.html +22 -24
  275. data/vendor/local/share/doc/groonga/en/html/install/others.html +87 -89
  276. data/vendor/local/share/doc/groonga/en/html/install/solaris.html +22 -24
  277. data/vendor/local/share/doc/groonga/en/html/install/ubuntu.html +25 -27
  278. data/vendor/local/share/doc/groonga/en/html/install/windows.html +30 -32
  279. data/vendor/local/share/doc/groonga/en/html/install.html +32 -34
  280. data/vendor/local/share/doc/groonga/en/html/limitations.html +17 -19
  281. data/vendor/local/share/doc/groonga/en/html/news/0.x.html +17 -19
  282. data/vendor/local/share/doc/groonga/en/html/news/1.0.x.html +19 -21
  283. data/vendor/local/share/doc/groonga/en/html/news/1.1.x.html +17 -19
  284. data/vendor/local/share/doc/groonga/en/html/news/1.2.x.html +32 -34
  285. data/vendor/local/share/doc/groonga/en/html/news/1.3.x.html +27 -29
  286. data/vendor/local/share/doc/groonga/en/html/news/2.x.html +98 -100
  287. data/vendor/local/share/doc/groonga/en/html/news/3.x.html +68 -70
  288. data/vendor/local/share/doc/groonga/en/html/news/4.x.html +102 -104
  289. data/vendor/local/share/doc/groonga/en/html/news/senna.html +17 -19
  290. data/vendor/local/share/doc/groonga/en/html/news.html +256 -27
  291. data/vendor/local/share/doc/groonga/en/html/objects.inv +0 -0
  292. data/vendor/local/share/doc/groonga/en/html/reference/api/global_configurations.html +49 -51
  293. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_cache.html +60 -62
  294. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_column.html +80 -82
  295. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_command_version.html +42 -44
  296. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_content_type.html +37 -39
  297. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ctx.html +130 -80
  298. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_db.html +48 -50
  299. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_encoding.html +44 -46
  300. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_expr.html +79 -81
  301. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_geo.html +42 -44
  302. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_hook.html +44 -46
  303. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ii.html +42 -44
  304. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_index_cursor.html +41 -43
  305. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_info.html +41 -43
  306. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_match_escalation.html +40 -42
  307. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_obj.html +89 -91
  308. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_proc.html +44 -46
  309. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_search.html +39 -41
  310. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table.html +75 -77
  311. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table_cursor.html +64 -66
  312. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_type.html +40 -42
  313. data/vendor/local/share/doc/groonga/en/html/reference/api/grn_user_data.html +39 -41
  314. data/vendor/local/share/doc/groonga/en/html/reference/api/overview.html +202 -0
  315. data/vendor/local/share/doc/groonga/en/html/reference/api/plugin.html +58 -60
  316. data/vendor/local/share/doc/groonga/en/html/reference/api.html +51 -52
  317. data/vendor/local/share/doc/groonga/en/html/reference/cast.html +17 -19
  318. data/vendor/local/share/doc/groonga/en/html/reference/column.html +17 -19
  319. data/vendor/local/share/doc/groonga/en/html/reference/columns/index.html +17 -19
  320. data/vendor/local/share/doc/groonga/en/html/reference/columns/pseudo.html +22 -24
  321. data/vendor/local/share/doc/groonga/en/html/reference/columns/scalar.html +17 -19
  322. data/vendor/local/share/doc/groonga/en/html/reference/columns/vector.html +58 -60
  323. data/vendor/local/share/doc/groonga/en/html/reference/command/command_version.html +17 -19
  324. data/vendor/local/share/doc/groonga/en/html/reference/command/output_format.html +53 -55
  325. data/vendor/local/share/doc/groonga/en/html/reference/command/request_id.html +22 -24
  326. data/vendor/local/share/doc/groonga/en/html/reference/command/return_code.html +94 -96
  327. data/vendor/local/share/doc/groonga/en/html/reference/command.html +56 -54
  328. data/vendor/local/share/doc/groonga/en/html/reference/commands/cache_limit.html +39 -41
  329. data/vendor/local/share/doc/groonga/en/html/reference/commands/check.html +43 -45
  330. data/vendor/local/share/doc/groonga/en/html/reference/commands/clearlock.html +23 -25
  331. data/vendor/local/share/doc/groonga/en/html/reference/commands/column_create.html +39 -41
  332. data/vendor/local/share/doc/groonga/en/html/reference/commands/column_list.html +71 -73
  333. data/vendor/local/share/doc/groonga/en/html/reference/commands/column_remove.html +24 -26
  334. data/vendor/local/share/doc/groonga/en/html/reference/commands/column_rename.html +44 -46
  335. data/vendor/local/share/doc/groonga/en/html/reference/commands/define_selector.html +37 -39
  336. data/vendor/local/share/doc/groonga/en/html/reference/commands/defrag.html +24 -26
  337. data/vendor/local/share/doc/groonga/en/html/reference/commands/delete.html +35 -37
  338. data/vendor/local/share/doc/groonga/en/html/reference/commands/dump.html +22 -24
  339. data/vendor/local/share/doc/groonga/en/html/reference/commands/load.html +43 -45
  340. data/vendor/local/share/doc/groonga/en/html/reference/commands/lock_clear.html +49 -47
  341. data/vendor/local/share/doc/groonga/en/html/reference/commands/log_level.html +23 -25
  342. data/vendor/local/share/doc/groonga/en/html/reference/commands/log_put.html +24 -26
  343. data/vendor/local/share/doc/groonga/en/html/reference/commands/log_reopen.html +26 -28
  344. data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_count.html +314 -0
  345. data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_range_filter.html +252 -0
  346. data/vendor/local/share/doc/groonga/en/html/reference/commands/normalize.html +87 -89
  347. data/vendor/local/share/doc/groonga/en/html/reference/commands/normalizer_list.html +46 -48
  348. data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_register.html +195 -0
  349. data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_unregister.html +193 -0
  350. data/vendor/local/share/doc/groonga/en/html/reference/commands/quit.html +38 -40
  351. data/vendor/local/share/doc/groonga/en/html/reference/commands/range_filter.html +37 -39
  352. data/vendor/local/share/doc/groonga/en/html/reference/commands/register.html +61 -51
  353. data/vendor/local/share/doc/groonga/en/html/reference/commands/request_cancel.html +72 -74
  354. data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_eval.html +54 -56
  355. data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_load.html +54 -56
  356. data/vendor/local/share/doc/groonga/en/html/reference/commands/select.html +590 -592
  357. data/vendor/local/share/doc/groonga/en/html/reference/commands/shutdown.html +37 -39
  358. data/vendor/local/share/doc/groonga/en/html/reference/commands/status.html +40 -42
  359. data/vendor/local/share/doc/groonga/en/html/reference/commands/suggest.html +92 -94
  360. data/vendor/local/share/doc/groonga/en/html/reference/commands/table_create.html +152 -154
  361. data/vendor/local/share/doc/groonga/en/html/reference/commands/table_list.html +49 -51
  362. data/vendor/local/share/doc/groonga/en/html/reference/commands/table_remove.html +39 -41
  363. data/vendor/local/share/doc/groonga/en/html/reference/commands/table_tokenize.html +68 -70
  364. data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenize.html +103 -105
  365. data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenizer_list.html +45 -47
  366. data/vendor/local/share/doc/groonga/en/html/reference/commands/truncate.html +52 -54
  367. data/vendor/local/share/doc/groonga/en/html/reference/executables/grndb.html +35 -37
  368. data/vendor/local/share/doc/groonga/en/html/reference/executables/grnslap.html +21 -23
  369. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-benchmark.html +26 -28
  370. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-httpd.html +73 -75
  371. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-server-http.html +17 -19
  372. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-create-dataset.html +17 -19
  373. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-httpd.html +42 -44
  374. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-learner.html +34 -36
  375. data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga.html +124 -90
  376. data/vendor/local/share/doc/groonga/en/html/reference/executables.html +19 -21
  377. data/vendor/local/share/doc/groonga/en/html/reference/function.html +42 -44
  378. data/vendor/local/share/doc/groonga/en/html/reference/functions/between.html +66 -68
  379. data/vendor/local/share/doc/groonga/en/html/reference/functions/edit_distance.html +44 -46
  380. data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_distance.html +113 -115
  381. data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_circle.html +55 -57
  382. data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_rectangle.html +45 -47
  383. data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_full.html +81 -83
  384. data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_html.html +65 -67
  385. data/vendor/local/share/doc/groonga/en/html/reference/functions/html_untag.html +54 -56
  386. data/vendor/local/share/doc/groonga/en/html/reference/functions/in_values.html +135 -44
  387. data/vendor/local/share/doc/groonga/en/html/reference/functions/now.html +40 -42
  388. data/vendor/local/share/doc/groonga/en/html/reference/functions/query.html +81 -83
  389. data/vendor/local/share/doc/groonga/en/html/reference/functions/rand.html +43 -45
  390. data/vendor/local/share/doc/groonga/en/html/reference/functions/snippet_html.html +67 -69
  391. data/vendor/local/share/doc/groonga/en/html/reference/functions/sub_filter.html +66 -70
  392. data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/query_syntax.html +349 -286
  393. data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/script_syntax.html +483 -417
  394. data/vendor/local/share/doc/groonga/en/html/reference/grn_expr.html +39 -41
  395. data/vendor/local/share/doc/groonga/en/html/reference/indexing.html +35 -37
  396. data/vendor/local/share/doc/groonga/en/html/reference/log.html +38 -40
  397. data/vendor/local/share/doc/groonga/en/html/reference/normalizers.html +44 -46
  398. data/vendor/local/share/doc/groonga/en/html/reference/operations/geolocation_search.html +32 -34
  399. data/vendor/local/share/doc/groonga/en/html/reference/operations.html +30 -31
  400. data/vendor/local/share/doc/groonga/en/html/reference/output.html +32 -34
  401. data/vendor/local/share/doc/groonga/en/html/reference/query_expanders/tsv.html +60 -62
  402. data/vendor/local/share/doc/groonga/en/html/reference/query_expanders.html +17 -19
  403. data/vendor/local/share/doc/groonga/en/html/reference/regular_expression.html +931 -0
  404. data/vendor/local/share/doc/groonga/en/html/reference/scorer.html +442 -0
  405. data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_at_most.html +153 -0
  406. data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_idf.html +287 -0
  407. data/vendor/local/share/doc/groonga/en/html/reference/scoring_note.html +114 -0
  408. data/vendor/local/share/doc/groonga/en/html/reference/suggest/completion.html +51 -53
  409. data/vendor/local/share/doc/groonga/en/html/reference/suggest/correction.html +40 -42
  410. data/vendor/local/share/doc/groonga/en/html/reference/suggest/introduction.html +38 -40
  411. data/vendor/local/share/doc/groonga/en/html/reference/suggest/suggestion.html +40 -42
  412. data/vendor/local/share/doc/groonga/en/html/reference/suggest.html +45 -47
  413. data/vendor/local/share/doc/groonga/en/html/reference/tables.html +52 -54
  414. data/vendor/local/share/doc/groonga/en/html/reference/token_filters.html +36 -38
  415. data/vendor/local/share/doc/groonga/en/html/reference/tokenizers.html +1394 -34
  416. data/vendor/local/share/doc/groonga/en/html/reference/tuning.html +57 -59
  417. data/vendor/local/share/doc/groonga/en/html/reference/types.html +38 -40
  418. data/vendor/local/share/doc/groonga/en/html/reference.html +139 -118
  419. data/vendor/local/share/doc/groonga/en/html/search.html +11 -11
  420. data/vendor/local/share/doc/groonga/en/html/searchindex.js +1 -1
  421. data/vendor/local/share/doc/groonga/en/html/server/gqtp.html +28 -30
  422. data/vendor/local/share/doc/groonga/en/html/server/http/comparison.html +68 -70
  423. data/vendor/local/share/doc/groonga/en/html/server/http/groonga-httpd.html +30 -32
  424. data/vendor/local/share/doc/groonga/en/html/server/http/groonga.html +29 -31
  425. data/vendor/local/share/doc/groonga/en/html/server/http.html +42 -44
  426. data/vendor/local/share/doc/groonga/en/html/server/memcached.html +137 -0
  427. data/vendor/local/share/doc/groonga/en/html/server/package.html +36 -38
  428. data/vendor/local/share/doc/groonga/en/html/server.html +23 -24
  429. data/vendor/local/share/doc/groonga/en/html/spec/gqtp.html +208 -129
  430. data/vendor/local/share/doc/groonga/en/html/spec/search.html +17 -19
  431. data/vendor/local/share/doc/groonga/en/html/spec.html +22 -24
  432. data/vendor/local/share/doc/groonga/en/html/troubleshooting/different_results_with_the_same_keyword.html +18 -20
  433. data/vendor/local/share/doc/groonga/en/html/troubleshooting/mmap_cannot_allocate_memory.html +18 -20
  434. data/vendor/local/share/doc/groonga/en/html/troubleshooting.html +17 -19
  435. data/vendor/local/share/doc/groonga/en/html/tutorial/data.html +17 -19
  436. data/vendor/local/share/doc/groonga/en/html/tutorial/drilldown.html +31 -33
  437. data/vendor/local/share/doc/groonga/en/html/tutorial/index.html +20 -22
  438. data/vendor/local/share/doc/groonga/en/html/tutorial/introduction.html +17 -19
  439. data/vendor/local/share/doc/groonga/en/html/tutorial/lexicon.html +17 -19
  440. data/vendor/local/share/doc/groonga/en/html/tutorial/match_columns.html +21 -23
  441. data/vendor/local/share/doc/groonga/en/html/tutorial/micro_blog.html +50 -52
  442. data/vendor/local/share/doc/groonga/en/html/tutorial/network.html +27 -125
  443. data/vendor/local/share/doc/groonga/en/html/tutorial/patricia_trie.html +18 -20
  444. data/vendor/local/share/doc/groonga/en/html/tutorial/query_expansion.html +20 -22
  445. data/vendor/local/share/doc/groonga/en/html/tutorial/search.html +33 -35
  446. data/vendor/local/share/doc/groonga/en/html/tutorial.html +21 -25
  447. data/vendor/local/share/doc/groonga/ja/html/.buildinfo +1 -1
  448. data/vendor/local/share/doc/groonga/ja/html/_images/used-when-indexing.png +0 -0
  449. data/vendor/local/share/doc/groonga/ja/html/_images/used-when-searching.png +0 -0
  450. data/vendor/local/share/doc/groonga/ja/html/_sources/characteristic.txt +1 -1
  451. data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/release.txt +32 -17
  452. data/vendor/local/share/doc/groonga/ja/html/_sources/install/centos.txt +3 -3
  453. data/vendor/local/share/doc/groonga/ja/html/_sources/install/debian.txt +3 -3
  454. data/vendor/local/share/doc/groonga/ja/html/_sources/install/fedora.txt +4 -4
  455. data/vendor/local/share/doc/groonga/ja/html/_sources/install/mac_os_x.txt +3 -3
  456. data/vendor/local/share/doc/groonga/ja/html/_sources/install/others.txt +3 -3
  457. data/vendor/local/share/doc/groonga/ja/html/_sources/install/solaris.txt +3 -3
  458. data/vendor/local/share/doc/groonga/ja/html/_sources/install/ubuntu.txt +3 -3
  459. data/vendor/local/share/doc/groonga/ja/html/_sources/install/windows.txt +9 -9
  460. data/vendor/local/share/doc/groonga/ja/html/_sources/news/1.0.x.txt +1 -1
  461. data/vendor/local/share/doc/groonga/ja/html/_sources/news/3.x.txt +2 -2
  462. data/vendor/local/share/doc/groonga/ja/html/_sources/news/4.x.txt +2 -2
  463. data/vendor/local/share/doc/groonga/ja/html/_sources/news.txt +194 -0
  464. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api/grn_ctx.txt +42 -0
  465. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api/overview.txt +54 -0
  466. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api.txt +3 -0
  467. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/cache_limit.txt +1 -1
  468. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_create.txt +2 -2
  469. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_list.txt +1 -1
  470. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_rename.txt +3 -3
  471. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/delete.txt +4 -4
  472. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/load.txt +5 -5
  473. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/lock_clear.txt +4 -4
  474. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_count.txt +173 -0
  475. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_range_filter.txt +112 -0
  476. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/normalize.txt +7 -6
  477. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/plugin_register.txt +64 -0
  478. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/plugin_unregister.txt +63 -0
  479. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/register.txt +11 -1
  480. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/request_cancel.txt +3 -2
  481. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/ruby_eval.txt +1 -1
  482. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/ruby_load.txt +1 -1
  483. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/select.txt +17 -17
  484. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/suggest.txt +12 -12
  485. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/table_create.txt +7 -7
  486. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/table_tokenize.txt +4 -4
  487. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/tokenize.txt +6 -6
  488. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/truncate.txt +1 -1
  489. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/executables/groonga.txt +47 -26
  490. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/between.txt +5 -5
  491. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/geo_distance.txt +3 -3
  492. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/highlight_full.txt +6 -6
  493. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/highlight_html.txt +1 -1
  494. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/html_untag.txt +1 -1
  495. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/in_values.txt +54 -2
  496. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/query.txt +4 -4
  497. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/sub_filter.txt +4 -4
  498. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/grn_expr/query_syntax.txt +44 -18
  499. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/grn_expr/script_syntax.txt +41 -11
  500. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/indexing.txt +2 -0
  501. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/normalizers.txt +4 -0
  502. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/operations.txt +2 -1
  503. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/regular_expression.txt +403 -0
  504. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/scorer.txt +217 -0
  505. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/scorers/scorer_tf_at_most.txt +22 -0
  506. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/scorers/scorer_tf_idf.txt +110 -0
  507. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/scoring_note.txt +13 -0
  508. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/tables.txt +8 -0
  509. data/vendor/local/share/doc/groonga/ja/html/_sources/reference/tokenizers.txt +530 -16
  510. data/vendor/local/share/doc/groonga/ja/html/_sources/reference.txt +2 -0
  511. data/vendor/local/share/doc/groonga/ja/html/_sources/server/memcached.txt +15 -0
  512. data/vendor/local/share/doc/groonga/ja/html/_sources/server.txt +2 -1
  513. data/vendor/local/share/doc/groonga/ja/html/_sources/spec/gqtp.txt +66 -1
  514. data/vendor/local/share/doc/groonga/ja/html/_sources/troubleshooting/different_results_with_the_same_keyword.txt +1 -1
  515. data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/network.txt +0 -81
  516. data/vendor/local/share/doc/groonga/ja/html/_static/basic.css +6 -68
  517. data/vendor/local/share/doc/groonga/ja/html/_static/doctools.js +1 -26
  518. data/vendor/local/share/doc/groonga/ja/html/_static/down-pressed.png +0 -0
  519. data/vendor/local/share/doc/groonga/ja/html/_static/down.png +0 -0
  520. data/vendor/local/share/doc/groonga/ja/html/_static/file.png +0 -0
  521. data/vendor/local/share/doc/groonga/ja/html/_static/jquery.js +9404 -4
  522. data/vendor/local/share/doc/groonga/ja/html/_static/minus.png +0 -0
  523. data/vendor/local/share/doc/groonga/ja/html/_static/plus.png +0 -0
  524. data/vendor/local/share/doc/groonga/ja/html/_static/searchtools.js +2 -2
  525. data/vendor/local/share/doc/groonga/ja/html/_static/underscore.js +1415 -31
  526. data/vendor/local/share/doc/groonga/ja/html/_static/up-pressed.png +0 -0
  527. data/vendor/local/share/doc/groonga/ja/html/_static/up.png +0 -0
  528. data/vendor/local/share/doc/groonga/ja/html/_static/websupport.js +15 -15
  529. data/vendor/local/share/doc/groonga/ja/html/characteristic.html +17 -19
  530. data/vendor/local/share/doc/groonga/ja/html/client.html +22 -24
  531. data/vendor/local/share/doc/groonga/ja/html/community.html +17 -19
  532. data/vendor/local/share/doc/groonga/ja/html/contribution/development/com.html +17 -19
  533. data/vendor/local/share/doc/groonga/ja/html/contribution/development/cooperation.html +17 -19
  534. data/vendor/local/share/doc/groonga/ja/html/contribution/development/query.html +17 -19
  535. data/vendor/local/share/doc/groonga/ja/html/contribution/development/release.html +51 -38
  536. data/vendor/local/share/doc/groonga/ja/html/contribution/development/repository.html +17 -19
  537. data/vendor/local/share/doc/groonga/ja/html/contribution/development/test.html +17 -19
  538. data/vendor/local/share/doc/groonga/ja/html/contribution/development.html +17 -19
  539. data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/c-api.html +17 -19
  540. data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/i18n.html +17 -19
  541. data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/introduction.html +24 -26
  542. data/vendor/local/share/doc/groonga/ja/html/contribution/documentation.html +17 -19
  543. data/vendor/local/share/doc/groonga/ja/html/contribution/report.html +17 -19
  544. data/vendor/local/share/doc/groonga/ja/html/contribution.html +18 -20
  545. data/vendor/local/share/doc/groonga/ja/html/development/travis-ci.html +20 -22
  546. data/vendor/local/share/doc/groonga/ja/html/development.html +17 -19
  547. data/vendor/local/share/doc/groonga/ja/html/genindex.html +26 -14
  548. data/vendor/local/share/doc/groonga/ja/html/index.html +150 -130
  549. data/vendor/local/share/doc/groonga/ja/html/install/centos.html +31 -33
  550. data/vendor/local/share/doc/groonga/ja/html/install/debian.html +25 -27
  551. data/vendor/local/share/doc/groonga/ja/html/install/fedora.html +29 -31
  552. data/vendor/local/share/doc/groonga/ja/html/install/mac_os_x.html +22 -24
  553. data/vendor/local/share/doc/groonga/ja/html/install/others.html +78 -80
  554. data/vendor/local/share/doc/groonga/ja/html/install/solaris.html +21 -23
  555. data/vendor/local/share/doc/groonga/ja/html/install/ubuntu.html +26 -28
  556. data/vendor/local/share/doc/groonga/ja/html/install/windows.html +29 -31
  557. data/vendor/local/share/doc/groonga/ja/html/install.html +32 -34
  558. data/vendor/local/share/doc/groonga/ja/html/limitations.html +17 -19
  559. data/vendor/local/share/doc/groonga/ja/html/news/0.x.html +17 -19
  560. data/vendor/local/share/doc/groonga/ja/html/news/1.0.x.html +19 -21
  561. data/vendor/local/share/doc/groonga/ja/html/news/1.1.x.html +17 -19
  562. data/vendor/local/share/doc/groonga/ja/html/news/1.2.x.html +32 -34
  563. data/vendor/local/share/doc/groonga/ja/html/news/1.3.x.html +27 -29
  564. data/vendor/local/share/doc/groonga/ja/html/news/2.x.html +91 -93
  565. data/vendor/local/share/doc/groonga/ja/html/news/3.x.html +59 -61
  566. data/vendor/local/share/doc/groonga/ja/html/news/4.x.html +89 -91
  567. data/vendor/local/share/doc/groonga/ja/html/news/senna.html +17 -19
  568. data/vendor/local/share/doc/groonga/ja/html/news.html +210 -27
  569. data/vendor/local/share/doc/groonga/ja/html/objects.inv +0 -0
  570. data/vendor/local/share/doc/groonga/ja/html/reference/api/global_configurations.html +49 -51
  571. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_cache.html +55 -57
  572. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_column.html +80 -82
  573. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_command_version.html +42 -44
  574. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_content_type.html +37 -39
  575. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ctx.html +126 -76
  576. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_db.html +48 -50
  577. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_encoding.html +44 -46
  578. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_expr.html +74 -76
  579. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_geo.html +42 -44
  580. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_hook.html +44 -46
  581. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ii.html +42 -44
  582. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_index_cursor.html +41 -43
  583. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_info.html +41 -43
  584. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_match_escalation.html +40 -42
  585. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_obj.html +89 -91
  586. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_proc.html +44 -46
  587. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_search.html +39 -41
  588. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table.html +75 -77
  589. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table_cursor.html +64 -66
  590. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_type.html +40 -42
  591. data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_user_data.html +39 -41
  592. data/vendor/local/share/doc/groonga/ja/html/reference/api/overview.html +197 -0
  593. data/vendor/local/share/doc/groonga/ja/html/reference/api/plugin.html +58 -60
  594. data/vendor/local/share/doc/groonga/ja/html/reference/api.html +51 -52
  595. data/vendor/local/share/doc/groonga/ja/html/reference/cast.html +17 -19
  596. data/vendor/local/share/doc/groonga/ja/html/reference/column.html +17 -19
  597. data/vendor/local/share/doc/groonga/ja/html/reference/columns/index.html +17 -19
  598. data/vendor/local/share/doc/groonga/ja/html/reference/columns/pseudo.html +22 -24
  599. data/vendor/local/share/doc/groonga/ja/html/reference/columns/scalar.html +17 -19
  600. data/vendor/local/share/doc/groonga/ja/html/reference/columns/vector.html +48 -50
  601. data/vendor/local/share/doc/groonga/ja/html/reference/command/command_version.html +17 -19
  602. data/vendor/local/share/doc/groonga/ja/html/reference/command/output_format.html +43 -45
  603. data/vendor/local/share/doc/groonga/ja/html/reference/command/request_id.html +22 -24
  604. data/vendor/local/share/doc/groonga/ja/html/reference/command/return_code.html +93 -95
  605. data/vendor/local/share/doc/groonga/ja/html/reference/command.html +56 -54
  606. data/vendor/local/share/doc/groonga/ja/html/reference/commands/cache_limit.html +35 -37
  607. data/vendor/local/share/doc/groonga/ja/html/reference/commands/check.html +43 -45
  608. data/vendor/local/share/doc/groonga/ja/html/reference/commands/clearlock.html +23 -25
  609. data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_create.html +39 -41
  610. data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_list.html +62 -64
  611. data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_remove.html +24 -26
  612. data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_rename.html +38 -40
  613. data/vendor/local/share/doc/groonga/ja/html/reference/commands/define_selector.html +37 -39
  614. data/vendor/local/share/doc/groonga/ja/html/reference/commands/defrag.html +24 -26
  615. data/vendor/local/share/doc/groonga/ja/html/reference/commands/delete.html +32 -34
  616. data/vendor/local/share/doc/groonga/ja/html/reference/commands/dump.html +22 -24
  617. data/vendor/local/share/doc/groonga/ja/html/reference/commands/load.html +33 -35
  618. data/vendor/local/share/doc/groonga/ja/html/reference/commands/lock_clear.html +48 -46
  619. data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_level.html +23 -25
  620. data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_put.html +24 -26
  621. data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_reopen.html +26 -28
  622. data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_count.html +314 -0
  623. data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_range_filter.html +250 -0
  624. data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalize.html +80 -81
  625. data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalizer_list.html +46 -48
  626. data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_register.html +188 -0
  627. data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_unregister.html +190 -0
  628. data/vendor/local/share/doc/groonga/ja/html/reference/commands/quit.html +38 -40
  629. data/vendor/local/share/doc/groonga/ja/html/reference/commands/range_filter.html +37 -39
  630. data/vendor/local/share/doc/groonga/ja/html/reference/commands/register.html +57 -47
  631. data/vendor/local/share/doc/groonga/ja/html/reference/commands/request_cancel.html +71 -73
  632. data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_eval.html +53 -55
  633. data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_load.html +53 -55
  634. data/vendor/local/share/doc/groonga/ja/html/reference/commands/select.html +394 -396
  635. data/vendor/local/share/doc/groonga/ja/html/reference/commands/shutdown.html +37 -39
  636. data/vendor/local/share/doc/groonga/ja/html/reference/commands/status.html +38 -40
  637. data/vendor/local/share/doc/groonga/ja/html/reference/commands/suggest.html +78 -80
  638. data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_create.html +123 -125
  639. data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_list.html +49 -51
  640. data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_remove.html +39 -41
  641. data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_tokenize.html +61 -63
  642. data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenize.html +89 -91
  643. data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenizer_list.html +46 -48
  644. data/vendor/local/share/doc/groonga/ja/html/reference/commands/truncate.html +51 -53
  645. data/vendor/local/share/doc/groonga/ja/html/reference/executables/grndb.html +35 -37
  646. data/vendor/local/share/doc/groonga/ja/html/reference/executables/grnslap.html +21 -23
  647. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-benchmark.html +26 -28
  648. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-httpd.html +61 -63
  649. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-server-http.html +17 -19
  650. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-create-dataset.html +17 -19
  651. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-httpd.html +42 -44
  652. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-learner.html +34 -36
  653. data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga.html +126 -90
  654. data/vendor/local/share/doc/groonga/ja/html/reference/executables.html +19 -21
  655. data/vendor/local/share/doc/groonga/ja/html/reference/function.html +42 -44
  656. data/vendor/local/share/doc/groonga/ja/html/reference/functions/between.html +63 -65
  657. data/vendor/local/share/doc/groonga/ja/html/reference/functions/edit_distance.html +44 -46
  658. data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_distance.html +94 -96
  659. data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_circle.html +55 -57
  660. data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_rectangle.html +45 -47
  661. data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_full.html +66 -68
  662. data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_html.html +55 -57
  663. data/vendor/local/share/doc/groonga/ja/html/reference/functions/html_untag.html +53 -55
  664. data/vendor/local/share/doc/groonga/ja/html/reference/functions/in_values.html +135 -44
  665. data/vendor/local/share/doc/groonga/ja/html/reference/functions/now.html +40 -42
  666. data/vendor/local/share/doc/groonga/ja/html/reference/functions/query.html +70 -72
  667. data/vendor/local/share/doc/groonga/ja/html/reference/functions/rand.html +43 -45
  668. data/vendor/local/share/doc/groonga/ja/html/reference/functions/snippet_html.html +53 -55
  669. data/vendor/local/share/doc/groonga/ja/html/reference/functions/sub_filter.html +56 -62
  670. data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/query_syntax.html +229 -171
  671. data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/script_syntax.html +381 -322
  672. data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr.html +36 -38
  673. data/vendor/local/share/doc/groonga/ja/html/reference/indexing.html +34 -36
  674. data/vendor/local/share/doc/groonga/ja/html/reference/log.html +38 -40
  675. data/vendor/local/share/doc/groonga/ja/html/reference/normalizers.html +38 -40
  676. data/vendor/local/share/doc/groonga/ja/html/reference/operations/geolocation_search.html +32 -34
  677. data/vendor/local/share/doc/groonga/ja/html/reference/operations.html +28 -30
  678. data/vendor/local/share/doc/groonga/ja/html/reference/output.html +28 -30
  679. data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders/tsv.html +39 -41
  680. data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders.html +17 -19
  681. data/vendor/local/share/doc/groonga/ja/html/reference/regular_expression.html +878 -0
  682. data/vendor/local/share/doc/groonga/ja/html/reference/scorer.html +442 -0
  683. data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_at_most.html +154 -0
  684. data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_idf.html +287 -0
  685. data/vendor/local/share/doc/groonga/ja/html/reference/scoring_note.html +115 -0
  686. data/vendor/local/share/doc/groonga/ja/html/reference/suggest/completion.html +48 -50
  687. data/vendor/local/share/doc/groonga/ja/html/reference/suggest/correction.html +40 -42
  688. data/vendor/local/share/doc/groonga/ja/html/reference/suggest/introduction.html +38 -40
  689. data/vendor/local/share/doc/groonga/ja/html/reference/suggest/suggestion.html +40 -42
  690. data/vendor/local/share/doc/groonga/ja/html/reference/suggest.html +45 -47
  691. data/vendor/local/share/doc/groonga/ja/html/reference/tables.html +42 -44
  692. data/vendor/local/share/doc/groonga/ja/html/reference/token_filters.html +37 -39
  693. data/vendor/local/share/doc/groonga/ja/html/reference/tokenizers.html +1300 -34
  694. data/vendor/local/share/doc/groonga/ja/html/reference/tuning.html +57 -59
  695. data/vendor/local/share/doc/groonga/ja/html/reference/types.html +38 -40
  696. data/vendor/local/share/doc/groonga/ja/html/reference.html +139 -118
  697. data/vendor/local/share/doc/groonga/ja/html/search.html +11 -11
  698. data/vendor/local/share/doc/groonga/ja/html/searchindex.js +1 -1
  699. data/vendor/local/share/doc/groonga/ja/html/server/gqtp.html +28 -30
  700. data/vendor/local/share/doc/groonga/ja/html/server/http/comparison.html +62 -64
  701. data/vendor/local/share/doc/groonga/ja/html/server/http/groonga-httpd.html +30 -32
  702. data/vendor/local/share/doc/groonga/ja/html/server/http/groonga.html +29 -31
  703. data/vendor/local/share/doc/groonga/ja/html/server/http.html +42 -44
  704. data/vendor/local/share/doc/groonga/ja/html/server/memcached.html +138 -0
  705. data/vendor/local/share/doc/groonga/ja/html/server/package.html +35 -37
  706. data/vendor/local/share/doc/groonga/ja/html/server.html +23 -24
  707. data/vendor/local/share/doc/groonga/ja/html/spec/gqtp.html +207 -128
  708. data/vendor/local/share/doc/groonga/ja/html/spec/search.html +17 -19
  709. data/vendor/local/share/doc/groonga/ja/html/spec.html +22 -24
  710. data/vendor/local/share/doc/groonga/ja/html/troubleshooting/different_results_with_the_same_keyword.html +18 -20
  711. data/vendor/local/share/doc/groonga/ja/html/troubleshooting/mmap_cannot_allocate_memory.html +18 -20
  712. data/vendor/local/share/doc/groonga/ja/html/troubleshooting.html +17 -19
  713. data/vendor/local/share/doc/groonga/ja/html/tutorial/data.html +17 -19
  714. data/vendor/local/share/doc/groonga/ja/html/tutorial/drilldown.html +30 -32
  715. data/vendor/local/share/doc/groonga/ja/html/tutorial/index.html +17 -19
  716. data/vendor/local/share/doc/groonga/ja/html/tutorial/introduction.html +17 -19
  717. data/vendor/local/share/doc/groonga/ja/html/tutorial/lexicon.html +17 -19
  718. data/vendor/local/share/doc/groonga/ja/html/tutorial/match_columns.html +23 -25
  719. data/vendor/local/share/doc/groonga/ja/html/tutorial/micro_blog.html +47 -49
  720. data/vendor/local/share/doc/groonga/ja/html/tutorial/network.html +27 -125
  721. data/vendor/local/share/doc/groonga/ja/html/tutorial/patricia_trie.html +18 -20
  722. data/vendor/local/share/doc/groonga/ja/html/tutorial/query_expansion.html +20 -22
  723. data/vendor/local/share/doc/groonga/ja/html/tutorial/search.html +31 -33
  724. data/vendor/local/share/doc/groonga/ja/html/tutorial.html +21 -25
  725. data/vendor/local/share/license/mruby/README.md +2 -2
  726. data/vendor/local/share/man/ja/man1/groonga.1 +6205 -2251
  727. data/vendor/local/share/man/man1/groonga.1 +7210 -3029
  728. metadata +75 -11
  729. data/doc/text/news.textile +0 -1217
  730. data/vendor/local/share/doc/groonga/en/html/_static/jquery-1.11.1.js +0 -10308
  731. data/vendor/local/share/doc/groonga/en/html/_static/underscore-1.3.1.js +0 -999
  732. data/vendor/local/share/doc/groonga/ja/html/_static/jquery-1.11.1.js +0 -10308
  733. data/vendor/local/share/doc/groonga/ja/html/_static/underscore-1.3.1.js +0 -999
data/vendor/local/share/doc/groonga/en/html/reference/tokenizers.html CHANGED
@@ -7,7 +7,7 @@
7
7
  <head>
8
8
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
9
9
 
10
- <title>7.8. Tokenizers &mdash; Groonga v5.0.0 documentation</title>
10
+ <title>7.8. Tokenizers &mdash; Groonga v5.0.1-42-g4d10df1 documentation</title>
11
11
 
12
12
  <link rel="stylesheet" href="../_static/groonga.css" type="text/css" />
13
13
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
@@ -15,7 +15,7 @@
15
15
  <script type="text/javascript">
16
16
  var DOCUMENTATION_OPTIONS = {
17
17
  URL_ROOT: '../',
18
- VERSION: '5.0.0',
18
+ VERSION: '5.0.1-42-g4d10df1',
19
19
  COLLAPSE_INDEX: false,
20
20
  FILE_SUFFIX: '.html',
21
21
  HAS_SOURCE: true
@@ -25,12 +25,12 @@
25
25
  <script type="text/javascript" src="../_static/underscore.js"></script>
26
26
  <script type="text/javascript" src="../_static/doctools.js"></script>
27
27
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
28
- <link rel="top" title="Groonga v5.0.0 documentation" href="../index.html" />
28
+ <link rel="top" title="Groonga v5.0.1-42-g4d10df1 documentation" href="../index.html" />
29
29
  <link rel="up" title="7. Reference manual" href="../reference.html" />
30
30
  <link rel="next" title="7.9. Token filters" href="token_filters.html" />
31
31
  <link rel="prev" title="7.7. Normalizers" href="normalizers.html" />
32
32
  </head>
33
- <body role="document">
33
+ <body>
34
34
  <div class="header">
35
35
  <h1 class="title">
36
36
  <a id="top-link" href="../index.html">
@@ -48,7 +48,7 @@
48
48
  </div>
49
49
 
50
50
 
51
- <div class="related" role="navigation" aria-label="related navigation">
51
+ <div class="related">
52
52
  <h3>Navigation</h3>
53
53
  <ul>
54
54
  <li class="right" style="margin-right: 10px">
@@ -60,7 +60,7 @@
60
60
  <li class="right" >
61
61
  <a href="normalizers.html" title="7.7. Normalizers"
62
62
  accesskey="P">previous</a> |</li>
63
- <li><a href="../index.html">Groonga v5.0.0 documentation</a> &raquo;</li>
63
+ <li><a href="../index.html">Groonga v5.0.1-42-g4d10df1 documentation</a> &raquo;</li>
64
64
  <li><a href="../reference.html" accesskey="U">7. Reference manual</a> &raquo;</li>
65
65
  </ul>
66
66
  </div>
@@ -68,48 +68,1408 @@
68
68
  <div class="document">
69
69
  <div class="documentwrapper">
70
70
  <div class="bodywrapper">
71
- <div class="body" role="main">
71
+ <div class="body">
72
72
 
73
73
  <div class="section" id="tokenizers">
74
74
  <h1>7.8. Tokenizers<a class="headerlink" href="#tokenizers" title="Permalink to this headline">¶</a></h1>
75
- <p>TODO: Write me.</p>
76
- <p>Here are the list of built-in tokenizers:</p>
77
- <ul class="simple">
78
- <li>TokenBigram</li>
79
- <li>TokenBigramSplitSymbol</li>
80
- <li>TokenBigramSplitSymbolAlpha</li>
81
- <li>TokenBigramSplitSymbolAlphaDigit</li>
82
- <li>TokenBigramIgnoreBlank</li>
83
- <li>TokenBigramIgnoreBlankSplitSymbol</li>
84
- <li>TokenBigramIgnoreBlankSplitAlpha</li>
85
- <li>TokenBigramIgnoreBlankSplitAlphaDigit</li>
86
- <li>TokenDelimit</li>
87
- <li>TokenDelimitNull</li>
88
- <li>TokenTrigram</li>
89
- <li>TokenUnigram</li>
75
+ <div class="section" id="summary">
76
+ <h2>7.8.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
77
+ <p>Groonga has tokenizer module that tokenizes text. It is used when
78
+ the following cases:</p>
79
+ <blockquote>
80
+ <div><ul>
81
+ <li><p class="first">Indexing text</p>
82
+ <div class="figure align-center">
83
+ <a class="reference internal image-reference" href="../_images/used-when-indexing.png"><img alt="../_images/used-when-indexing.png" src="../_images/used-when-indexing.png" style="width: 80%;" /></a>
84
+ <p class="caption">Tokenizer is used when indexing text.</p>
85
+ </div>
86
+ </li>
87
+ <li><p class="first">Searching by query</p>
88
+ <div class="figure align-center">
89
+ <a class="reference internal image-reference" href="../_images/used-when-searching.png"><img alt="../_images/used-when-searching.png" src="../_images/used-when-searching.png" style="width: 80%;" /></a>
90
+ <p class="caption">Tokenizer is used when searching by query.</p>
91
+ </div>
92
+ </li>
93
+ </ul>
94
+ </div></blockquote>
95
+ <p>Tokenizer is an important module for full-text search. You can change
96
+ trade-off between <a class="reference external" href="http://en.wikipedia.org/wiki/Precision_and_recall">precision and recall</a> by changing
97
+ tokenizer.</p>
98
+ <p>Normally, <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> is a suitable tokenizer. If you don't
99
+ know much about tokenizer, it's recommended that you choose
100
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>.</p>
101
+ <p>You can try a tokenizer by <a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a> and
102
+ <a class="reference internal" href="commands/table_tokenize.html"><em>table_tokenize</em></a>. Here is an example to
103
+ try <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> tokenizer by
104
+ <a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a>:</p>
105
+ <p>Execution example:</p>
106
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Hello World&quot;
107
+ # [
108
+ # [
109
+ # 0,
110
+ # 1337566253.89858,
111
+ # 0.000355720520019531
112
+ # ],
113
+ # [
114
+ # {
115
+ # &quot;position&quot;: 0,
116
+ # &quot;value&quot;: &quot;He&quot;
117
+ # },
118
+ # {
119
+ # &quot;position&quot;: 1,
120
+ # &quot;value&quot;: &quot;el&quot;
121
+ # },
122
+ # {
123
+ # &quot;position&quot;: 2,
124
+ # &quot;value&quot;: &quot;ll&quot;
125
+ # },
126
+ # {
127
+ # &quot;position&quot;: 3,
128
+ # &quot;value&quot;: &quot;lo&quot;
129
+ # },
130
+ # {
131
+ # &quot;position&quot;: 4,
132
+ # &quot;value&quot;: &quot;o &quot;
133
+ # },
134
+ # {
135
+ # &quot;position&quot;: 5,
136
+ # &quot;value&quot;: &quot; W&quot;
137
+ # },
138
+ # {
139
+ # &quot;position&quot;: 6,
140
+ # &quot;value&quot;: &quot;Wo&quot;
141
+ # },
142
+ # {
143
+ # &quot;position&quot;: 7,
144
+ # &quot;value&quot;: &quot;or&quot;
145
+ # },
146
+ # {
147
+ # &quot;position&quot;: 8,
148
+ # &quot;value&quot;: &quot;rl&quot;
149
+ # },
150
+ # {
151
+ # &quot;position&quot;: 9,
152
+ # &quot;value&quot;: &quot;ld&quot;
153
+ # },
154
+ # {
155
+ # &quot;position&quot;: 10,
156
+ # &quot;value&quot;: &quot;d&quot;
157
+ # }
158
+ # ]
159
+ # ]
160
+ </pre></div>
161
+ </div>
162
+ </div>
163
+ <div class="section" id="what-is-tokenize">
164
+ <h2>7.8.2. What is &quot;tokenize&quot;?<a class="headerlink" href="#what-is-tokenize" title="Permalink to this headline">¶</a></h2>
165
+ <p>&quot;tokenize&quot; is the process that extracts zero or more tokens from a
166
+ text. There are some &quot;tokenize&quot; methods.</p>
167
+ <p>For example, <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> is tokenized to the following tokens by
168
+ bigram tokenize method:</p>
169
+ <blockquote>
170
+ <div><ul class="simple">
171
+ <li><tt class="docutils literal"><span class="pre">He</span></tt></li>
172
+ <li><tt class="docutils literal"><span class="pre">el</span></tt></li>
173
+ <li><tt class="docutils literal"><span class="pre">ll</span></tt></li>
174
+ <li><tt class="docutils literal"><span class="pre">lo</span></tt></li>
175
+ <li><tt class="docutils literal"><span class="pre">o_</span></tt> (<tt class="docutils literal"><span class="pre">_</span></tt> means a white-space)</li>
176
+ <li><tt class="docutils literal"><span class="pre">_W</span></tt> (<tt class="docutils literal"><span class="pre">_</span></tt> means a white-space)</li>
177
+ <li><tt class="docutils literal"><span class="pre">Wo</span></tt></li>
178
+ <li><tt class="docutils literal"><span class="pre">or</span></tt></li>
179
+ <li><tt class="docutils literal"><span class="pre">rl</span></tt></li>
180
+ <li><tt class="docutils literal"><span class="pre">ld</span></tt></li>
181
+ </ul>
182
+ </div></blockquote>
183
+ <p>In the above example, 10 tokens are extracted from one text <tt class="docutils literal"><span class="pre">Hello</span>
184
+ <span class="pre">World</span></tt>.</p>
185
+ <p>For example, <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> is tokenized to the following tokens by
186
+ white-space-separate tokenize method:</p>
187
+ <blockquote>
188
+ <div><ul class="simple">
189
+ <li><tt class="docutils literal"><span class="pre">Hello</span></tt></li>
190
+ <li><tt class="docutils literal"><span class="pre">World</span></tt></li>
191
+ </ul>
192
+ </div></blockquote>
193
+ <p>In the above example, 2 tokens are extracted from one text <tt class="docutils literal"><span class="pre">Hello</span>
194
+ <span class="pre">World</span></tt>.</p>
195
+ <p>Token is used as search key. You can find indexed documents only by
196
+ tokens that are extracted by used tokenize method. For example, you
197
+ can find <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> by <tt class="docutils literal"><span class="pre">ll</span></tt> with bigram tokenize method but you
198
+ can't find <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> by <tt class="docutils literal"><span class="pre">ll</span></tt> with white-space-separate tokenize
199
+ method. Because white-space-separate tokenize method doesn't extract
200
+ <tt class="docutils literal"><span class="pre">ll</span></tt> token. It just extracts <tt class="docutils literal"><span class="pre">Hello</span></tt> and <tt class="docutils literal"><span class="pre">World</span></tt> tokens.</p>
201
+ <p>In general, tokenize method that generates small tokens increases
202
+ recall but decreases precision. Tokenize method that generates large
203
+ tokens increases precision but decreases recall.</p>
204
+ <p>For example, we can find <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> and <tt class="docutils literal"><span class="pre">A</span> <span class="pre">or</span> <span class="pre">B</span></tt> by <tt class="docutils literal"><span class="pre">or</span></tt> with
205
+ bigram tokenize method. <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> is a noise for people who
206
+ wants to search &quot;logical and&quot;. It means that precision is
207
+ decreased. But recall is increased.</p>
208
+ <p>We can find only <tt class="docutils literal"><span class="pre">A</span> <span class="pre">or</span> <span class="pre">B</span></tt> by <tt class="docutils literal"><span class="pre">or</span></tt> with white-space-separate
209
+ tokenize method. Because <tt class="docutils literal"><span class="pre">World</span></tt> is tokenized to one token <tt class="docutils literal"><span class="pre">World</span></tt>
210
+ with white-space-separate tokenize method. It means that precision is
211
+ increased for people who wants to search &quot;logical and&quot;. But recall is
212
+ decreased because <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> that contains <tt class="docutils literal"><span class="pre">or</span></tt> isn't found.</p>
213
+ </div>
214
+ <div class="section" id="built-in-tokenizsers">
215
+ <h2>7.8.3. Built-in tokenizsers<a class="headerlink" href="#built-in-tokenizsers" title="Permalink to this headline">¶</a></h2>
216
+ <p>Here is a list of built-in tokenizers:</p>
217
+ <blockquote>
218
+ <div><ul class="simple">
219
+ <li><tt class="docutils literal"><span class="pre">TokenBigram</span></tt></li>
220
+ <li><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></tt></li>
221
+ <li><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt></li>
222
+ <li><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></tt></li>
223
+ <li><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt></li>
224
+ <li><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt></li>
225
+ <li><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitAlpha</span></tt></li>
226
+ <li><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitAlphaDigit</span></tt></li>
227
+ <li><tt class="docutils literal"><span class="pre">TokenUnigram</span></tt></li>
228
+ <li><tt class="docutils literal"><span class="pre">TokenTrigram</span></tt></li>
229
+ <li><tt class="docutils literal"><span class="pre">TokenDelimit</span></tt></li>
230
+ <li><tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt></li>
231
+ <li><tt class="docutils literal"><span class="pre">TokenMecab</span></tt></li>
232
+ <li><tt class="docutils literal"><span class="pre">TokenRegexp</span></tt></li>
90
233
  </ul>
234
+ </div></blockquote>
235
+ <div class="section" id="tokenbigram">
236
+ <span id="token-bigram"></span><h3>7.8.3.1. <tt class="docutils literal"><span class="pre">TokenBigram</span></tt><a class="headerlink" href="#tokenbigram" title="Permalink to this headline">¶</a></h3>
237
+ <p><tt class="docutils literal"><span class="pre">TokenBigram</span></tt> is a bigram based tokenizer. It's recommended to use
238
+ this tokenizer for most cases.</p>
239
+ <p>Bigram tokenize method tokenizes a text to two adjacent characters
240
+ tokens. For example, <tt class="docutils literal"><span class="pre">Hello</span></tt> is tokenized to the following tokens:</p>
241
+ <blockquote>
242
+ <div><ul class="simple">
243
+ <li><tt class="docutils literal"><span class="pre">He</span></tt></li>
244
+ <li><tt class="docutils literal"><span class="pre">el</span></tt></li>
245
+ <li><tt class="docutils literal"><span class="pre">ll</span></tt></li>
246
+ <li><tt class="docutils literal"><span class="pre">lo</span></tt></li>
247
+ </ul>
248
+ </div></blockquote>
249
+ <p>Bigram tokenize method is good for recall because you can find all
250
+ texts by query consists of two or more characters.</p>
251
+ <p>In general, you can't find all texts by query consists of one
252
+ character because one character token doesn't exist. But you can find
253
+ all texts by query consists of one character in Groonga. Because
254
+ Groonga find tokens that start with query by predictive search. For
255
+ example, Groonga can find <tt class="docutils literal"><span class="pre">ll</span></tt> and <tt class="docutils literal"><span class="pre">lo</span></tt> tokens by <tt class="docutils literal"><span class="pre">l</span></tt> query.</p>
256
+ <p>Bigram tokenize method isn't good for precision because you can find
257
+ texts that includes query in word. For example, you can find <tt class="docutils literal"><span class="pre">world</span></tt>
258
+ by <tt class="docutils literal"><span class="pre">or</span></tt>. This is more sensitive for ASCII only languages rather than
259
+ non-ASCII languages. <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> has solution for this problem
260
+ described in the bellow.</p>
261
+ <p><tt class="docutils literal"><span class="pre">TokenBigram</span></tt> behavior is different when it's worked with any
262
+ <a class="reference internal" href="normalizers.html"><em>Normalizers</em></a>.</p>
263
+ <p>If no normalizer is used, <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses pure bigram (all tokens
264
+ except the last token have two characters) tokenize method:</p>
265
+ <p>Execution example:</p>
266
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Hello World&quot;
267
+ # [
268
+ # [
269
+ # 0,
270
+ # 1337566253.89858,
271
+ # 0.000355720520019531
272
+ # ],
273
+ # [
274
+ # {
275
+ # &quot;position&quot;: 0,
276
+ # &quot;value&quot;: &quot;He&quot;
277
+ # },
278
+ # {
279
+ # &quot;position&quot;: 1,
280
+ # &quot;value&quot;: &quot;el&quot;
281
+ # },
282
+ # {
283
+ # &quot;position&quot;: 2,
284
+ # &quot;value&quot;: &quot;ll&quot;
285
+ # },
286
+ # {
287
+ # &quot;position&quot;: 3,
288
+ # &quot;value&quot;: &quot;lo&quot;
289
+ # },
290
+ # {
291
+ # &quot;position&quot;: 4,
292
+ # &quot;value&quot;: &quot;o &quot;
293
+ # },
294
+ # {
295
+ # &quot;position&quot;: 5,
296
+ # &quot;value&quot;: &quot; W&quot;
297
+ # },
298
+ # {
299
+ # &quot;position&quot;: 6,
300
+ # &quot;value&quot;: &quot;Wo&quot;
301
+ # },
302
+ # {
303
+ # &quot;position&quot;: 7,
304
+ # &quot;value&quot;: &quot;or&quot;
305
+ # },
306
+ # {
307
+ # &quot;position&quot;: 8,
308
+ # &quot;value&quot;: &quot;rl&quot;
309
+ # },
310
+ # {
311
+ # &quot;position&quot;: 9,
312
+ # &quot;value&quot;: &quot;ld&quot;
313
+ # },
314
+ # {
315
+ # &quot;position&quot;: 10,
316
+ # &quot;value&quot;: &quot;d&quot;
317
+ # }
318
+ # ]
319
+ # ]
320
+ </pre></div>
321
+ </div>
322
+ <p>If normalizer is used, <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses white-space-separate like
323
+ tokenize method for ASCII characters. <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses bigram
324
+ tokenize method for non-ASCII characters.</p>
325
+ <p>You may be confused with this combined behavior. But it's reasonable
326
+ for most use cases such as English text (only ASCII characters) and
327
+ Japanese text (ASCII and non-ASCII characters are mixed).</p>
328
+ <p>Most languages consists of only ASCII characters use white-space for
329
+ word separator. White-space-separate tokenize method is suitable for
330
+ the case.</p>
331
+ <p>Languages consists of non-ASCII characters don't use white-space for
332
+ word separator. Bigram tokenize method is suitable for the case.</p>
333
+ <p>Mixed tokenize method is suitable for mixed language case.</p>
334
+ <p>If you want to use bigram tokenize method for ASCII character, see
335
+ <tt class="docutils literal"><span class="pre">TokenBigramSplitXXX</span></tt> type tokenizers such as
336
+ <a class="reference internal" href="#token-bigram-split-symbol-alpha"><em>TokenBigramSplitSymbolAlpha</em></a>.</p>
337
+ <p>Let's confirm <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> behavior by example.</p>
338
+ <p><tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses one or more white-spaces as token delimiter for
339
+ ASCII characters:</p>
340
+ <p>Execution example:</p>
341
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Hello World&quot; NormalizerAuto
342
+ # [
343
+ # [
344
+ # 0,
345
+ # 1337566253.89858,
346
+ # 0.000355720520019531
347
+ # ],
348
+ # [
349
+ # {
350
+ # &quot;position&quot;: 0,
351
+ # &quot;value&quot;: &quot;hello&quot;
352
+ # },
353
+ # {
354
+ # &quot;position&quot;: 1,
355
+ # &quot;value&quot;: &quot;world&quot;
356
+ # }
357
+ # ]
358
+ # ]
359
+ </pre></div>
360
+ </div>
361
+ <p><tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses character type change as token delimiter for
362
+ ASCII characters. Character type is one of them:</p>
363
+ <blockquote>
364
+ <div><ul class="simple">
365
+ <li>Alphabet</li>
366
+ <li>Digit</li>
367
+ <li>Symbol (such as <tt class="docutils literal"><span class="pre">(</span></tt>, <tt class="docutils literal"><span class="pre">)</span></tt> and <tt class="docutils literal"><span class="pre">!</span></tt>)</li>
368
+ <li>Hiragana</li>
369
+ <li>Katakana</li>
370
+ <li>Kanji</li>
371
+ <li>Others</li>
372
+ </ul>
373
+ </div></blockquote>
374
+ <p>The following example shows two token delimiters:</p>
375
+ <blockquote>
376
+ <div><ul class="simple">
377
+ <li>at between <tt class="docutils literal"><span class="pre">100</span></tt> (digits) and <tt class="docutils literal"><span class="pre">cents</span></tt> (alphabets)</li>
378
+ <li>at between <tt class="docutils literal"><span class="pre">cents</span></tt> (alphabets) and <tt class="docutils literal"><span class="pre">!!!</span></tt> (symbols)</li>
379
+ </ul>
380
+ </div></blockquote>
381
+ <p>Execution example:</p>
382
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;100cents!!!&quot; NormalizerAuto
383
+ # [
384
+ # [
385
+ # 0,
386
+ # 1337566253.89858,
387
+ # 0.000355720520019531
388
+ # ],
389
+ # [
390
+ # {
391
+ # &quot;position&quot;: 0,
392
+ # &quot;value&quot;: &quot;100&quot;
393
+ # },
394
+ # {
395
+ # &quot;position&quot;: 1,
396
+ # &quot;value&quot;: &quot;cents&quot;
397
+ # },
398
+ # {
399
+ # &quot;position&quot;: 2,
400
+ # &quot;value&quot;: &quot;!!!&quot;
401
+ # }
402
+ # ]
403
+ # ]
404
+ </pre></div>
405
+ </div>
406
+ <p>Here is an example that <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> uses bigram tokenize method
407
+ for non-ASCII characters.</p>
408
+ <p>Execution example:</p>
409
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;日本語の勉強&quot; NormalizerAuto
410
+ # [
411
+ # [
412
+ # 0,
413
+ # 1337566253.89858,
414
+ # 0.000355720520019531
415
+ # ],
416
+ # [
417
+ # {
418
+ # &quot;position&quot;: 0,
419
+ # &quot;value&quot;: &quot;日本&quot;
420
+ # },
421
+ # {
422
+ # &quot;position&quot;: 1,
423
+ # &quot;value&quot;: &quot;本語&quot;
424
+ # },
425
+ # {
426
+ # &quot;position&quot;: 2,
427
+ # &quot;value&quot;: &quot;語の&quot;
428
+ # },
429
+ # {
430
+ # &quot;position&quot;: 3,
431
+ # &quot;value&quot;: &quot;の勉&quot;
432
+ # },
433
+ # {
434
+ # &quot;position&quot;: 4,
435
+ # &quot;value&quot;: &quot;勉強&quot;
436
+ # },
437
+ # {
438
+ # &quot;position&quot;: 5,
439
+ # &quot;value&quot;: &quot;強&quot;
440
+ # }
441
+ # ]
442
+ # ]
443
+ </pre></div>
444
+ </div>
445
+ </div>
446
+ <div class="section" id="tokenbigramsplitsymbol">
447
+ <span id="token-bigram-split-symbol"></span><h3>7.8.3.2. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></tt><a class="headerlink" href="#tokenbigramsplitsymbol" title="Permalink to this headline">¶</a></h3>
448
+ <p><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></tt> is similar to <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The
449
+ difference between them is symbol handling. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></tt>
450
+ tokenizes symbols by bigram tokenize method:</p>
451
+ <p>Execution example:</p>
452
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbol &quot;100cents!!!&quot; NormalizerAuto
453
+ # [
454
+ # [
455
+ # 0,
456
+ # 1337566253.89858,
457
+ # 0.000355720520019531
458
+ # ],
459
+ # [
460
+ # {
461
+ # &quot;position&quot;: 0,
462
+ # &quot;value&quot;: &quot;100&quot;
463
+ # },
464
+ # {
465
+ # &quot;position&quot;: 1,
466
+ # &quot;value&quot;: &quot;cents&quot;
467
+ # },
468
+ # {
469
+ # &quot;position&quot;: 2,
470
+ # &quot;value&quot;: &quot;!!&quot;
471
+ # },
472
+ # {
473
+ # &quot;position&quot;: 3,
474
+ # &quot;value&quot;: &quot;!!&quot;
475
+ # },
476
+ # {
477
+ # &quot;position&quot;: 4,
478
+ # &quot;value&quot;: &quot;!&quot;
479
+ # }
480
+ # ]
481
+ # ]
482
+ </pre></div>
483
+ </div>
484
+ </div>
485
+ <div class="section" id="tokenbigramsplitsymbolalpha">
486
+ <span id="token-bigram-split-symbol-alpha"></span><h3>7.8.3.3. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt><a class="headerlink" href="#tokenbigramsplitsymbolalpha" title="Permalink to this headline">¶</a></h3>
487
+ <p><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt> is similar to <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The
488
+ difference between them is symbol and alphabet
489
+ handling. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt> tokenizes symbols and
490
+ alphabets by bigram tokenize method:</p>
491
+ <p>Execution example:</p>
492
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbolAlpha &quot;100cents!!!&quot; NormalizerAuto
493
+ # [
494
+ # [
495
+ # 0,
496
+ # 1337566253.89858,
497
+ # 0.000355720520019531
498
+ # ],
499
+ # [
500
+ # {
501
+ # &quot;position&quot;: 0,
502
+ # &quot;value&quot;: &quot;100&quot;
503
+ # },
504
+ # {
505
+ # &quot;position&quot;: 1,
506
+ # &quot;value&quot;: &quot;ce&quot;
507
+ # },
508
+ # {
509
+ # &quot;position&quot;: 2,
510
+ # &quot;value&quot;: &quot;en&quot;
511
+ # },
512
+ # {
513
+ # &quot;position&quot;: 3,
514
+ # &quot;value&quot;: &quot;nt&quot;
515
+ # },
516
+ # {
517
+ # &quot;position&quot;: 4,
518
+ # &quot;value&quot;: &quot;ts&quot;
519
+ # },
520
+ # {
521
+ # &quot;position&quot;: 5,
522
+ # &quot;value&quot;: &quot;s!&quot;
523
+ # },
524
+ # {
525
+ # &quot;position&quot;: 6,
526
+ # &quot;value&quot;: &quot;!!&quot;
527
+ # },
528
+ # {
529
+ # &quot;position&quot;: 7,
530
+ # &quot;value&quot;: &quot;!!&quot;
531
+ # },
532
+ # {
533
+ # &quot;position&quot;: 8,
534
+ # &quot;value&quot;: &quot;!&quot;
535
+ # }
536
+ # ]
537
+ # ]
538
+ </pre></div>
539
+ </div>
540
+ </div>
541
+ <div class="section" id="tokenbigramsplitsymbolalphadigit">
542
+ <span id="token-bigram-split-symbol-alpha-digit"></span><h3>7.8.3.4. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></tt><a class="headerlink" href="#tokenbigramsplitsymbolalphadigit" title="Permalink to this headline">¶</a></h3>
543
+ <p><tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></tt> is similar to
544
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The difference between them is symbol, alphabet
545
+ and digit handling. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></tt> tokenizes
546
+ symbols, alphabets and digits by bigram tokenize method. It means that
547
+ all characters are tokenized by bigram tokenize method:</p>
548
+ <p>Execution example:</p>
549
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbolAlphaDigit &quot;100cents!!!&quot; NormalizerAuto
550
+ # [
551
+ # [
552
+ # 0,
553
+ # 1337566253.89858,
554
+ # 0.000355720520019531
555
+ # ],
556
+ # [
557
+ # {
558
+ # &quot;position&quot;: 0,
559
+ # &quot;value&quot;: &quot;10&quot;
560
+ # },
561
+ # {
562
+ # &quot;position&quot;: 1,
563
+ # &quot;value&quot;: &quot;00&quot;
564
+ # },
565
+ # {
566
+ # &quot;position&quot;: 2,
567
+ # &quot;value&quot;: &quot;0c&quot;
568
+ # },
569
+ # {
570
+ # &quot;position&quot;: 3,
571
+ # &quot;value&quot;: &quot;ce&quot;
572
+ # },
573
+ # {
574
+ # &quot;position&quot;: 4,
575
+ # &quot;value&quot;: &quot;en&quot;
576
+ # },
577
+ # {
578
+ # &quot;position&quot;: 5,
579
+ # &quot;value&quot;: &quot;nt&quot;
580
+ # },
581
+ # {
582
+ # &quot;position&quot;: 6,
583
+ # &quot;value&quot;: &quot;ts&quot;
584
+ # },
585
+ # {
586
+ # &quot;position&quot;: 7,
587
+ # &quot;value&quot;: &quot;s!&quot;
588
+ # },
589
+ # {
590
+ # &quot;position&quot;: 8,
591
+ # &quot;value&quot;: &quot;!!&quot;
592
+ # },
593
+ # {
594
+ # &quot;position&quot;: 9,
595
+ # &quot;value&quot;: &quot;!!&quot;
596
+ # },
597
+ # {
598
+ # &quot;position&quot;: 10,
599
+ # &quot;value&quot;: &quot;!&quot;
600
+ # }
601
+ # ]
602
+ # ]
603
+ </pre></div>
604
+ </div>
605
+ </div>
606
+ <div class="section" id="tokenbigramignoreblank">
607
+ <span id="token-bigram-ignore-blank"></span><h3>7.8.3.5. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt><a class="headerlink" href="#tokenbigramignoreblank" title="Permalink to this headline">¶</a></h3>
608
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt> is similar to <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The
609
+ difference between them is blank handling. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt>
610
+ ignores white-spaces in continuous symbols and non-ASCII characters.</p>
611
+ <p>You can find difference of them by <tt class="docutils literal"><span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></tt> text because it
612
+ has symbols and non-ASCII characters.</p>
613
+ <p>Here is a result by <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> :</p>
614
+ <p>Execution example:</p>
615
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;日 本 語 ! ! !&quot; NormalizerAuto
616
+ # [
617
+ # [
618
+ # 0,
619
+ # 1337566253.89858,
620
+ # 0.000355720520019531
621
+ # ],
622
+ # [
623
+ # {
624
+ # &quot;position&quot;: 0,
625
+ # &quot;value&quot;: &quot;日&quot;
626
+ # },
627
+ # {
628
+ # &quot;position&quot;: 1,
629
+ # &quot;value&quot;: &quot;本&quot;
630
+ # },
631
+ # {
632
+ # &quot;position&quot;: 2,
633
+ # &quot;value&quot;: &quot;語&quot;
634
+ # },
635
+ # {
636
+ # &quot;position&quot;: 3,
637
+ # &quot;value&quot;: &quot;!&quot;
638
+ # },
639
+ # {
640
+ # &quot;position&quot;: 4,
641
+ # &quot;value&quot;: &quot;!&quot;
642
+ # },
643
+ # {
644
+ # &quot;position&quot;: 5,
645
+ # &quot;value&quot;: &quot;!&quot;
646
+ # }
647
+ # ]
648
+ # ]
649
+ </pre></div>
650
+ </div>
651
+ <p>Here is a result by <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt>:</p>
652
+ <p>Execution example:</p>
653
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlank &quot;日 本 語 ! ! !&quot; NormalizerAuto
654
+ # [
655
+ # [
656
+ # 0,
657
+ # 1337566253.89858,
658
+ # 0.000355720520019531
659
+ # ],
660
+ # [
661
+ # {
662
+ # &quot;position&quot;: 0,
663
+ # &quot;value&quot;: &quot;日本&quot;
664
+ # },
665
+ # {
666
+ # &quot;position&quot;: 1,
667
+ # &quot;value&quot;: &quot;本語&quot;
668
+ # },
669
+ # {
670
+ # &quot;position&quot;: 2,
671
+ # &quot;value&quot;: &quot;語&quot;
672
+ # },
673
+ # {
674
+ # &quot;position&quot;: 3,
675
+ # &quot;value&quot;: &quot;!!!&quot;
676
+ # }
677
+ # ]
678
+ # ]
679
+ </pre></div>
680
+ </div>
681
+ </div>
682
+ <div class="section" id="tokenbigramignoreblanksplitsymbol">
683
+ <span id="token-bigram-ignore-blank-split-symbol"></span><h3>7.8.3.6. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt><a class="headerlink" href="#tokenbigramignoreblanksplitsymbol" title="Permalink to this headline">¶</a></h3>
684
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt> is similar to
685
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The differences between them are the followings:</p>
686
+ <blockquote>
687
+ <div><ul class="simple">
688
+ <li>Blank handling</li>
689
+ <li>Symbol handling</li>
690
+ </ul>
691
+ </div></blockquote>
692
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt> ignores white-spaces in
693
+ continuous symbols and non-ASCII characters.</p>
694
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt> tokenizes symbols by bigram
695
+ tokenize method.</p>
696
+ <p>You can find difference of them by <tt class="docutils literal"><span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></tt> text because it
697
+ has symbols and non-ASCII characters.</p>
698
+ <p>Here is a result by <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> :</p>
699
+ <p>Execution example:</p>
700
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;日 本 語 ! ! !&quot; NormalizerAuto
701
+ # [
702
+ # [
703
+ # 0,
704
+ # 1337566253.89858,
705
+ # 0.000355720520019531
706
+ # ],
707
+ # [
708
+ # {
709
+ # &quot;position&quot;: 0,
710
+ # &quot;value&quot;: &quot;日&quot;
711
+ # },
712
+ # {
713
+ # &quot;position&quot;: 1,
714
+ # &quot;value&quot;: &quot;本&quot;
715
+ # },
716
+ # {
717
+ # &quot;position&quot;: 2,
718
+ # &quot;value&quot;: &quot;語&quot;
719
+ # },
720
+ # {
721
+ # &quot;position&quot;: 3,
722
+ # &quot;value&quot;: &quot;!&quot;
723
+ # },
724
+ # {
725
+ # &quot;position&quot;: 4,
726
+ # &quot;value&quot;: &quot;!&quot;
727
+ # },
728
+ # {
729
+ # &quot;position&quot;: 5,
730
+ # &quot;value&quot;: &quot;!&quot;
731
+ # }
732
+ # ]
733
+ # ]
734
+ </pre></div>
735
+ </div>
736
+ <p>Here is a result by <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt>:</p>
737
+ <p>Execution example:</p>
738
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbol &quot;日 本 語 ! ! !&quot; NormalizerAuto
739
+ # [
740
+ # [
741
+ # 0,
742
+ # 1337566253.89858,
743
+ # 0.000355720520019531
744
+ # ],
745
+ # [
746
+ # {
747
+ # &quot;position&quot;: 0,
748
+ # &quot;value&quot;: &quot;日本&quot;
749
+ # },
750
+ # {
751
+ # &quot;position&quot;: 1,
752
+ # &quot;value&quot;: &quot;本語&quot;
753
+ # },
754
+ # {
755
+ # &quot;position&quot;: 2,
756
+ # &quot;value&quot;: &quot;語!&quot;
757
+ # },
758
+ # {
759
+ # &quot;position&quot;: 3,
760
+ # &quot;value&quot;: &quot;!!&quot;
761
+ # },
762
+ # {
763
+ # &quot;position&quot;: 4,
764
+ # &quot;value&quot;: &quot;!!&quot;
765
+ # },
766
+ # {
767
+ # &quot;position&quot;: 5,
768
+ # &quot;value&quot;: &quot;!&quot;
769
+ # }
770
+ # ]
771
+ # ]
772
+ </pre></div>
773
+ </div>
774
+ </div>
775
+ <div class="section" id="tokenbigramignoreblanksplitsymbolalpha">
776
+ <span id="token-bigram-ignore-blank-split-symbol-alpha"></span><h3>7.8.3.7. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt><a class="headerlink" href="#tokenbigramignoreblanksplitsymbolalpha" title="Permalink to this headline">¶</a></h3>
777
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt> is similar to
778
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The differences between them are the followings:</p>
779
+ <blockquote>
780
+ <div><ul class="simple">
781
+ <li>Blank handling</li>
782
+ <li>Symbol and alphabet handling</li>
783
+ </ul>
784
+ </div></blockquote>
785
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt> ignores white-spaces in
786
+ continuous symbols and non-ASCII characters.</p>
787
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt> tokenizes symbols and
788
+ alphabets by bigram tokenize method.</p>
789
+ <p>You can find difference of them by <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></tt> text because it
790
+ has symbols and non-ASCII characters with white spaces and alphabets.</p>
791
+ <p>Here is a result by <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> :</p>
792
+ <p>Execution example:</p>
793
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Hello 日 本 語 ! ! !&quot; NormalizerAuto
794
+ # [
795
+ # [
796
+ # 0,
797
+ # 1337566253.89858,
798
+ # 0.000355720520019531
799
+ # ],
800
+ # [
801
+ # {
802
+ # &quot;position&quot;: 0,
803
+ # &quot;value&quot;: &quot;hello&quot;
804
+ # },
805
+ # {
806
+ # &quot;position&quot;: 1,
807
+ # &quot;value&quot;: &quot;日&quot;
808
+ # },
809
+ # {
810
+ # &quot;position&quot;: 2,
811
+ # &quot;value&quot;: &quot;本&quot;
812
+ # },
813
+ # {
814
+ # &quot;position&quot;: 3,
815
+ # &quot;value&quot;: &quot;語&quot;
816
+ # },
817
+ # {
818
+ # &quot;position&quot;: 4,
819
+ # &quot;value&quot;: &quot;!&quot;
820
+ # },
821
+ # {
822
+ # &quot;position&quot;: 5,
823
+ # &quot;value&quot;: &quot;!&quot;
824
+ # },
825
+ # {
826
+ # &quot;position&quot;: 6,
827
+ # &quot;value&quot;: &quot;!&quot;
828
+ # }
829
+ # ]
830
+ # ]
831
+ </pre></div>
832
+ </div>
833
+ <p>Here is a result by <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt>:</p>
834
+ <p>Execution example:</p>
835
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbolAlpha &quot;Hello 日 本 語 ! ! !&quot; NormalizerAuto
836
+ # [
837
+ # [
838
+ # 0,
839
+ # 1337566253.89858,
840
+ # 0.000355720520019531
841
+ # ],
842
+ # [
843
+ # {
844
+ # &quot;position&quot;: 0,
845
+ # &quot;value&quot;: &quot;he&quot;
846
+ # },
847
+ # {
848
+ # &quot;position&quot;: 1,
849
+ # &quot;value&quot;: &quot;el&quot;
850
+ # },
851
+ # {
852
+ # &quot;position&quot;: 2,
853
+ # &quot;value&quot;: &quot;ll&quot;
854
+ # },
855
+ # {
856
+ # &quot;position&quot;: 3,
857
+ # &quot;value&quot;: &quot;lo&quot;
858
+ # },
859
+ # {
860
+ # &quot;position&quot;: 4,
861
+ # &quot;value&quot;: &quot;o日&quot;
862
+ # },
863
+ # {
864
+ # &quot;position&quot;: 5,
865
+ # &quot;value&quot;: &quot;日本&quot;
866
+ # },
867
+ # {
868
+ # &quot;position&quot;: 6,
869
+ # &quot;value&quot;: &quot;本語&quot;
870
+ # },
871
+ # {
872
+ # &quot;position&quot;: 7,
873
+ # &quot;value&quot;: &quot;語!&quot;
874
+ # },
875
+ # {
876
+ # &quot;position&quot;: 8,
877
+ # &quot;value&quot;: &quot;!!&quot;
878
+ # },
879
+ # {
880
+ # &quot;position&quot;: 9,
881
+ # &quot;value&quot;: &quot;!!&quot;
882
+ # },
883
+ # {
884
+ # &quot;position&quot;: 10,
885
+ # &quot;value&quot;: &quot;!&quot;
886
+ # }
887
+ # ]
888
+ # ]
889
+ </pre></div>
890
+ </div>
891
+ </div>
892
+ <div class="section" id="tokenbigramignoreblanksplitsymbolalphadigit">
893
+ <span id="token-bigram-ignore-blank-split-symbol-alpha-digit"></span><h3>7.8.3.8. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt><a class="headerlink" href="#tokenbigramignoreblanksplitsymbolalphadigit" title="Permalink to this headline">¶</a></h3>
894
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt> is similar to
895
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The differences between them are the followings:</p>
896
+ <blockquote>
897
+ <div><ul class="simple">
898
+ <li>Blank handling</li>
899
+ <li>Symbol, alphabet and digit handling</li>
900
+ </ul>
901
+ </div></blockquote>
902
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt> ignores white-spaces
903
+ in continuous symbols and non-ASCII characters.</p>
904
+ <p><tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt> tokenizes symbols,
905
+ alphabets and digits by bigram tokenize method. It means that all
906
+ characters are tokenized by bigram tokenize method.</p>
907
+ <p>You can find difference of them by <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">777</span></tt> text
908
+ because it has symbols and non-ASCII characters with white spaces,
909
+ alphabets and digits.</p>
910
+ <p>Here is a result by <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> :</p>
911
+ <p>Execution example:</p>
912
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Hello 日 本 語 ! ! ! 777&quot; NormalizerAuto
913
+ # [
914
+ # [
915
+ # 0,
916
+ # 1337566253.89858,
917
+ # 0.000355720520019531
918
+ # ],
919
+ # [
920
+ # {
921
+ # &quot;position&quot;: 0,
922
+ # &quot;value&quot;: &quot;hello&quot;
923
+ # },
924
+ # {
925
+ # &quot;position&quot;: 1,
926
+ # &quot;value&quot;: &quot;日&quot;
927
+ # },
928
+ # {
929
+ # &quot;position&quot;: 2,
930
+ # &quot;value&quot;: &quot;本&quot;
931
+ # },
932
+ # {
933
+ # &quot;position&quot;: 3,
934
+ # &quot;value&quot;: &quot;語&quot;
935
+ # },
936
+ # {
937
+ # &quot;position&quot;: 4,
938
+ # &quot;value&quot;: &quot;!&quot;
939
+ # },
940
+ # {
941
+ # &quot;position&quot;: 5,
942
+ # &quot;value&quot;: &quot;!&quot;
943
+ # },
944
+ # {
945
+ # &quot;position&quot;: 6,
946
+ # &quot;value&quot;: &quot;!&quot;
947
+ # },
948
+ # {
949
+ # &quot;position&quot;: 7,
950
+ # &quot;value&quot;: &quot;777&quot;
951
+ # }
952
+ # ]
953
+ # ]
954
+ </pre></div>
955
+ </div>
956
+ <p>Here is a result by <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt>:</p>
957
+ <p>Execution example:</p>
958
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbolAlphaDigit &quot;Hello 日 本 語 ! ! ! 777&quot; NormalizerAuto
959
+ # [
960
+ # [
961
+ # 0,
962
+ # 1337566253.89858,
963
+ # 0.000355720520019531
964
+ # ],
965
+ # [
966
+ # {
967
+ # &quot;position&quot;: 0,
968
+ # &quot;value&quot;: &quot;he&quot;
969
+ # },
970
+ # {
971
+ # &quot;position&quot;: 1,
972
+ # &quot;value&quot;: &quot;el&quot;
973
+ # },
974
+ # {
975
+ # &quot;position&quot;: 2,
976
+ # &quot;value&quot;: &quot;ll&quot;
977
+ # },
978
+ # {
979
+ # &quot;position&quot;: 3,
980
+ # &quot;value&quot;: &quot;lo&quot;
981
+ # },
982
+ # {
983
+ # &quot;position&quot;: 4,
984
+ # &quot;value&quot;: &quot;o日&quot;
985
+ # },
986
+ # {
987
+ # &quot;position&quot;: 5,
988
+ # &quot;value&quot;: &quot;日本&quot;
989
+ # },
990
+ # {
991
+ # &quot;position&quot;: 6,
992
+ # &quot;value&quot;: &quot;本語&quot;
993
+ # },
994
+ # {
995
+ # &quot;position&quot;: 7,
996
+ # &quot;value&quot;: &quot;語!&quot;
997
+ # },
998
+ # {
999
+ # &quot;position&quot;: 8,
1000
+ # &quot;value&quot;: &quot;!!&quot;
1001
+ # },
1002
+ # {
1003
+ # &quot;position&quot;: 9,
1004
+ # &quot;value&quot;: &quot;!!&quot;
1005
+ # },
1006
+ # {
1007
+ # &quot;position&quot;: 10,
1008
+ # &quot;value&quot;: &quot;!7&quot;
1009
+ # },
1010
+ # {
1011
+ # &quot;position&quot;: 11,
1012
+ # &quot;value&quot;: &quot;77&quot;
1013
+ # },
1014
+ # {
1015
+ # &quot;position&quot;: 12,
1016
+ # &quot;value&quot;: &quot;77&quot;
1017
+ # },
1018
+ # {
1019
+ # &quot;position&quot;: 13,
1020
+ # &quot;value&quot;: &quot;7&quot;
1021
+ # }
1022
+ # ]
1023
+ # ]
1024
+ </pre></div>
1025
+ </div>
1026
+ </div>
1027
+ <div class="section" id="tokenunigram">
1028
+ <span id="token-unigram"></span><h3>7.8.3.9. <tt class="docutils literal"><span class="pre">TokenUnigram</span></tt><a class="headerlink" href="#tokenunigram" title="Permalink to this headline">¶</a></h3>
1029
+ <p><tt class="docutils literal"><span class="pre">TokenUnigram</span></tt> is similar to <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The differences
1030
+ between them is token unit. <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> uses 2 characters per
1031
+ token. <tt class="docutils literal"><span class="pre">TokenUnigram</span></tt> uses 1 character per token.</p>
1032
+ <p>Execution example:</p>
1033
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenUnigram &quot;100cents!!!&quot; NormalizerAuto
1034
+ # [
1035
+ # [
1036
+ # 0,
1037
+ # 1337566253.89858,
1038
+ # 0.000355720520019531
1039
+ # ],
1040
+ # [
1041
+ # {
1042
+ # &quot;position&quot;: 0,
1043
+ # &quot;value&quot;: &quot;100&quot;
1044
+ # },
1045
+ # {
1046
+ # &quot;position&quot;: 1,
1047
+ # &quot;value&quot;: &quot;cents&quot;
1048
+ # },
1049
+ # {
1050
+ # &quot;position&quot;: 2,
1051
+ # &quot;value&quot;: &quot;!!!&quot;
1052
+ # }
1053
+ # ]
1054
+ # ]
1055
+ </pre></div>
1056
+ </div>
1057
+ </div>
1058
+ <div class="section" id="tokentrigram">
1059
+ <span id="token-trigram"></span><h3>7.8.3.10. <tt class="docutils literal"><span class="pre">TokenTrigram</span></tt><a class="headerlink" href="#tokentrigram" title="Permalink to this headline">¶</a></h3>
1060
+ <p><tt class="docutils literal"><span class="pre">TokenTrigram</span></tt> is similar to <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a>. The differences
1061
+ between them is token unit. <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> uses 2 characters per
1062
+ token. <tt class="docutils literal"><span class="pre">TokenTrigram</span></tt> uses 3 characters per token.</p>
1063
+ <p>Execution example:</p>
1064
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenTrigram &quot;10000cents!!!!!&quot; NormalizerAuto
1065
+ # [
1066
+ # [
1067
+ # 0,
1068
+ # 1337566253.89858,
1069
+ # 0.000355720520019531
1070
+ # ],
1071
+ # [
1072
+ # {
1073
+ # &quot;position&quot;: 0,
1074
+ # &quot;value&quot;: &quot;10000&quot;
1075
+ # },
1076
+ # {
1077
+ # &quot;position&quot;: 1,
1078
+ # &quot;value&quot;: &quot;cents&quot;
1079
+ # },
1080
+ # {
1081
+ # &quot;position&quot;: 2,
1082
+ # &quot;value&quot;: &quot;!!!!!&quot;
1083
+ # }
1084
+ # ]
1085
+ # ]
1086
+ </pre></div>
1087
+ </div>
1088
+ </div>
1089
+ <div class="section" id="tokendelimit">
1090
+ <span id="token-delimit"></span><h3>7.8.3.11. <tt class="docutils literal"><span class="pre">TokenDelimit</span></tt><a class="headerlink" href="#tokendelimit" title="Permalink to this headline">¶</a></h3>
1091
+ <p><tt class="docutils literal"><span class="pre">TokenDelimit</span></tt> extracts token by splitting one or more space
1092
+ characters (<tt class="docutils literal"><span class="pre">U+0020</span></tt>). For example, <tt class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></tt> is tokenized to
1093
+ <tt class="docutils literal"><span class="pre">Hello</span></tt> and <tt class="docutils literal"><span class="pre">World</span></tt>.</p>
1094
+ <p><tt class="docutils literal"><span class="pre">TokenDelimit</span></tt> is suitable for tag text. You can extract <tt class="docutils literal"><span class="pre">groonga</span></tt>
1095
+ and <tt class="docutils literal"><span class="pre">full-text-search</span></tt> and <tt class="docutils literal"><span class="pre">http</span></tt> as tags from <tt class="docutils literal"><span class="pre">groonga</span>
1096
+ <span class="pre">full-text-search</span> <span class="pre">http</span></tt>.</p>
1097
+ <p>Here is an example of <tt class="docutils literal"><span class="pre">TokenDelimit</span></tt>:</p>
1098
+ <p>Execution example:</p>
1099
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenDelimit &quot;Groonga full-text-search HTTP&quot; NormalizerAuto
1100
+ # [
1101
+ # [
1102
+ # 0,
1103
+ # 1337566253.89858,
1104
+ # 0.000355720520019531
1105
+ # ],
1106
+ # [
1107
+ # {
1108
+ # &quot;position&quot;: 0,
1109
+ # &quot;value&quot;: &quot;groonga&quot;
1110
+ # },
1111
+ # {
1112
+ # &quot;position&quot;: 1,
1113
+ # &quot;value&quot;: &quot;full-text-search&quot;
1114
+ # },
1115
+ # {
1116
+ # &quot;position&quot;: 2,
1117
+ # &quot;value&quot;: &quot;http&quot;
1118
+ # }
1119
+ # ]
1120
+ # ]
1121
+ </pre></div>
1122
+ </div>
1123
+ </div>
1124
+ <div class="section" id="tokendelimitnull">
1125
+ <span id="token-delimit-null"></span><h3>7.8.3.12. <tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt><a class="headerlink" href="#tokendelimitnull" title="Permalink to this headline">¶</a></h3>
1126
+ <p><tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt> is similar to <a class="reference internal" href="#token-delimit"><em>TokenDelimit</em></a>. The
1127
+ difference between them is separator character. <a class="reference internal" href="#token-delimit"><em>TokenDelimit</em></a>
1128
+ uses space character (<tt class="docutils literal"><span class="pre">U+0020</span></tt>) but <tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt> uses NUL
1129
+ character (<tt class="docutils literal"><span class="pre">U+0000</span></tt>).</p>
1130
+ <p><tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt> is also suitable for tag text.</p>
1131
+ <p>Here is an example of <tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt>:</p>
1132
+ <p>Execution example:</p>
1133
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenDelimitNull &quot;Groonga\u0000full-text-search\u0000HTTP&quot; NormalizerAuto
1134
+ # [
1135
+ # [
1136
+ # 0,
1137
+ # 1337566253.89858,
1138
+ # 0.000355720520019531
1139
+ # ],
1140
+ # [
1141
+ # {
1142
+ # &quot;position&quot;: 0,
1143
+ # &quot;value&quot;: &quot;groongau0000full-text-searchu0000http&quot;
1144
+ # }
1145
+ # ]
1146
+ # ]
1147
+ </pre></div>
1148
+ </div>
1149
+ </div>
1150
+ <div class="section" id="tokenmecab">
1151
+ <span id="token-mecab"></span><h3>7.8.3.13. <tt class="docutils literal"><span class="pre">TokenMecab</span></tt><a class="headerlink" href="#tokenmecab" title="Permalink to this headline">¶</a></h3>
1152
+ <p><tt class="docutils literal"><span class="pre">TokenMecab</span></tt> is a tokenizer based on <a class="reference external" href="http://mecab.sourceforge.net/">MeCab</a> part-of-speech and
1153
+ morphological analyzer.</p>
1154
+ <p>MeCab doesn't depend on Japanese. You can use MeCab for other
1155
+ languages by creating dictionary for the languages. You can use <a class="reference external" href="http://sourceforge.jp/projects/naist-jdic/">NAIST
1156
+ Japanese Dictionary</a>
1157
+ for Japanese.</p>
1158
+ <p><tt class="docutils literal"><span class="pre">TokenMecab</span></tt> is good for precision rather than recall. You can find
1159
+ <tt class="docutils literal"><span class="pre">東京都</span></tt> and <tt class="docutils literal"><span class="pre">京都</span></tt> texts by <tt class="docutils literal"><span class="pre">京都</span></tt> query with
1160
+ <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> but <tt class="docutils literal"><span class="pre">東京都</span></tt> isn't expected. You can find only
1161
+ <tt class="docutils literal"><span class="pre">京都</span></tt> text by <tt class="docutils literal"><span class="pre">京都</span></tt> query with <tt class="docutils literal"><span class="pre">TokenMecab</span></tt>.</p>
1162
+ <p>If you want to support neologisms, you need to keep updating your
1163
+ MeCab dictionary. It needs maintain cost. (<a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> doesn't
1164
+ require dictionary maintenance because <a class="reference internal" href="#token-bigram"><em>TokenBigram</em></a> doesn't use
1165
+ dictionary.) <a class="reference external" href="https://github.com/neologd/mecab-ipadic-neologd">mecab-ipadic-NEologd : Neologism dictionary for MeCab</a> may help you.</p>
1166
+ <p>Here is an example of <tt class="docutils literal"><span class="pre">TokenMeCab</span></tt>. <tt class="docutils literal"><span class="pre">東京都</span></tt> is tokenized to <tt class="docutils literal"><span class="pre">東京</span></tt>
1167
+ and <tt class="docutils literal"><span class="pre">都</span></tt>. They don't include <tt class="docutils literal"><span class="pre">京都</span></tt>:</p>
1168
+ <p>Execution example:</p>
1169
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenMecab &quot;東京都&quot;
1170
+ # [
1171
+ # [
1172
+ # 0,
1173
+ # 1337566253.89858,
1174
+ # 0.000355720520019531
1175
+ # ],
1176
+ # [
1177
+ # {
1178
+ # &quot;position&quot;: 0,
1179
+ # &quot;value&quot;: &quot;東京&quot;
1180
+ # },
1181
+ # {
1182
+ # &quot;position&quot;: 1,
1183
+ # &quot;value&quot;: &quot;都&quot;
1184
+ # }
1185
+ # ]
1186
+ # ]
1187
+ </pre></div>
1188
+ </div>
1189
+ </div>
1190
+ <div class="section" id="tokenregexp">
1191
+ <span id="token-regexp"></span><h3>7.8.3.14. <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt><a class="headerlink" href="#tokenregexp" title="Permalink to this headline">¶</a></h3>
1192
+ <div class="versionadded">
1193
+ <p><span class="versionmodified">New in version 5.0.1.</span></p>
1194
+ </div>
1195
+ <div class="admonition caution">
1196
+ <p class="first admonition-title">Caution</p>
1197
+ <p class="last">This tokenizer is experimental. Specification may be changed.</p>
1198
+ </div>
1199
+ <div class="admonition caution">
1200
+ <p class="first admonition-title">Caution</p>
1201
+ <p class="last">This tokenizer can be used only with UTF-8. You can't use this
1202
+ tokenizer with EUC-JP, Shift_JIS and so on.</p>
1203
+ </div>
1204
+ <p><tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> is a tokenizer for supporting regular expression
1205
+ search by index.</p>
1206
+ <p>In general, regular expression search is evaluated as sequential
1207
+ search. But the following cases can be evaluated as index search:</p>
1208
+ <blockquote>
1209
+ <div><ul class="simple">
1210
+ <li>Literal only case such as <tt class="docutils literal"><span class="pre">hello</span></tt></li>
1211
+ <li>The beginning of text and literal case such as <tt class="docutils literal"><span class="pre">\A/home/alice</span></tt></li>
1212
+ <li>The end of text and literal case such as <tt class="docutils literal"><span class="pre">\.txt\z</span></tt></li>
1213
+ </ul>
1214
+ </div></blockquote>
1215
+ <p>In most cases, index search is faster than sequential search.</p>
1216
+ <p><tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> is based on bigram tokenize method. <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt>
1217
+ adds the beginning of text mark (<tt class="docutils literal"><span class="pre">U+FFEF</span></tt>) at the begging of text
1218
+ and the end of text mark (<tt class="docutils literal"><span class="pre">U+FFF0</span></tt>) to the end of text when you
1219
+ index text:</p>
1220
+ <p>Execution example:</p>
1221
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp &quot;/home/alice/test.txt&quot; NormalizerAuto --mode ADD
1222
+ # [
1223
+ # [
1224
+ # 0,
1225
+ # 1337566253.89858,
1226
+ # 0.000355720520019531
1227
+ # ],
1228
+ # [
1229
+ # {
1230
+ # &quot;position&quot;: 0,
1231
+ # &quot;value&quot;: &quot;￯&quot;
1232
+ # },
1233
+ # {
1234
+ # &quot;position&quot;: 1,
1235
+ # &quot;value&quot;: &quot;/h&quot;
1236
+ # },
1237
+ # {
1238
+ # &quot;position&quot;: 2,
1239
+ # &quot;value&quot;: &quot;ho&quot;
1240
+ # },
1241
+ # {
1242
+ # &quot;position&quot;: 3,
1243
+ # &quot;value&quot;: &quot;om&quot;
1244
+ # },
1245
+ # {
1246
+ # &quot;position&quot;: 4,
1247
+ # &quot;value&quot;: &quot;me&quot;
1248
+ # },
1249
+ # {
1250
+ # &quot;position&quot;: 5,
1251
+ # &quot;value&quot;: &quot;e/&quot;
1252
+ # },
1253
+ # {
1254
+ # &quot;position&quot;: 6,
1255
+ # &quot;value&quot;: &quot;/a&quot;
1256
+ # },
1257
+ # {
1258
+ # &quot;position&quot;: 7,
1259
+ # &quot;value&quot;: &quot;al&quot;
1260
+ # },
1261
+ # {
1262
+ # &quot;position&quot;: 8,
1263
+ # &quot;value&quot;: &quot;li&quot;
1264
+ # },
1265
+ # {
1266
+ # &quot;position&quot;: 9,
1267
+ # &quot;value&quot;: &quot;ic&quot;
1268
+ # },
1269
+ # {
1270
+ # &quot;position&quot;: 10,
1271
+ # &quot;value&quot;: &quot;ce&quot;
1272
+ # },
1273
+ # {
1274
+ # &quot;position&quot;: 11,
1275
+ # &quot;value&quot;: &quot;e/&quot;
1276
+ # },
1277
+ # {
1278
+ # &quot;position&quot;: 12,
1279
+ # &quot;value&quot;: &quot;/t&quot;
1280
+ # },
1281
+ # {
1282
+ # &quot;position&quot;: 13,
1283
+ # &quot;value&quot;: &quot;te&quot;
1284
+ # },
1285
+ # {
1286
+ # &quot;position&quot;: 14,
1287
+ # &quot;value&quot;: &quot;es&quot;
1288
+ # },
1289
+ # {
1290
+ # &quot;position&quot;: 15,
1291
+ # &quot;value&quot;: &quot;st&quot;
1292
+ # },
1293
+ # {
1294
+ # &quot;position&quot;: 16,
1295
+ # &quot;value&quot;: &quot;t.&quot;
1296
+ # },
1297
+ # {
1298
+ # &quot;position&quot;: 17,
1299
+ # &quot;value&quot;: &quot;.t&quot;
1300
+ # },
1301
+ # {
1302
+ # &quot;position&quot;: 18,
1303
+ # &quot;value&quot;: &quot;tx&quot;
1304
+ # },
1305
+ # {
1306
+ # &quot;position&quot;: 19,
1307
+ # &quot;value&quot;: &quot;xt&quot;
1308
+ # },
1309
+ # {
1310
+ # &quot;position&quot;: 20,
1311
+ # &quot;value&quot;: &quot;t&quot;
1312
+ # },
1313
+ # {
1314
+ # &quot;position&quot;: 21,
1315
+ # &quot;value&quot;: &quot;￰&quot;
1316
+ # }
1317
+ # ]
1318
+ # ]
1319
+ </pre></div>
1320
+ </div>
1321
+ <p>The beginning of text mark is used for the beginning of text search by
1322
+ <tt class="docutils literal"><span class="pre">\A</span></tt>. If you use <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> for tokenizing query,
1323
+ <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> adds the beginning of text mark (<tt class="docutils literal"><span class="pre">U+FFEF</span></tt>) as the
1324
+ first token. The beginning of text mark must be appeared at the first,
1325
+ you can get results of the beginning of text search.</p>
1326
+ <p>Execution example:</p>
1327
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp &quot;\\A/home/alice/&quot; NormalizerAuto --mode GET
1328
+ # [
1329
+ # [
1330
+ # 0,
1331
+ # 1337566253.89858,
1332
+ # 0.000355720520019531
1333
+ # ],
1334
+ # [
1335
+ # {
1336
+ # &quot;position&quot;: 0,
1337
+ # &quot;value&quot;: &quot;￯&quot;
1338
+ # },
1339
+ # {
1340
+ # &quot;position&quot;: 1,
1341
+ # &quot;value&quot;: &quot;/h&quot;
1342
+ # },
1343
+ # {
1344
+ # &quot;position&quot;: 2,
1345
+ # &quot;value&quot;: &quot;ho&quot;
1346
+ # },
1347
+ # {
1348
+ # &quot;position&quot;: 3,
1349
+ # &quot;value&quot;: &quot;om&quot;
1350
+ # },
1351
+ # {
1352
+ # &quot;position&quot;: 4,
1353
+ # &quot;value&quot;: &quot;me&quot;
1354
+ # },
1355
+ # {
1356
+ # &quot;position&quot;: 5,
1357
+ # &quot;value&quot;: &quot;e/&quot;
1358
+ # },
1359
+ # {
1360
+ # &quot;position&quot;: 6,
1361
+ # &quot;value&quot;: &quot;/a&quot;
1362
+ # },
1363
+ # {
1364
+ # &quot;position&quot;: 7,
1365
+ # &quot;value&quot;: &quot;al&quot;
1366
+ # },
1367
+ # {
1368
+ # &quot;position&quot;: 8,
1369
+ # &quot;value&quot;: &quot;li&quot;
1370
+ # },
1371
+ # {
1372
+ # &quot;position&quot;: 9,
1373
+ # &quot;value&quot;: &quot;ic&quot;
1374
+ # },
1375
+ # {
1376
+ # &quot;position&quot;: 10,
1377
+ # &quot;value&quot;: &quot;ce&quot;
1378
+ # },
1379
+ # {
1380
+ # &quot;position&quot;: 11,
1381
+ # &quot;value&quot;: &quot;e/&quot;
1382
+ # }
1383
+ # ]
1384
+ # ]
1385
+ </pre></div>
1386
+ </div>
1387
+ <p>The end of text mark is used for the end of text search by <tt class="docutils literal"><span class="pre">\z</span></tt>.
1388
+ If you use <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> for tokenizing query, <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> adds
1389
+ the end of text mark (<tt class="docutils literal"><span class="pre">U+FFF0</span></tt>) as the last token. The end of text
1390
+ mark must be appeared at the end, you can get results of the end of
1391
+ text search.</p>
1392
+ <p>Execution example:</p>
1393
+ <div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp &quot;\\.txt\\z&quot; NormalizerAuto --mode GET
1394
+ # [
1395
+ # [
1396
+ # 0,
1397
+ # 1337566253.89858,
1398
+ # 0.000355720520019531
1399
+ # ],
1400
+ # [
1401
+ # {
1402
+ # &quot;position&quot;: 0,
1403
+ # &quot;value&quot;: &quot;\\.&quot;
1404
+ # },
1405
+ # {
1406
+ # &quot;position&quot;: 1,
1407
+ # &quot;value&quot;: &quot;.t&quot;
1408
+ # },
1409
+ # {
1410
+ # &quot;position&quot;: 2,
1411
+ # &quot;value&quot;: &quot;tx&quot;
1412
+ # },
1413
+ # {
1414
+ # &quot;position&quot;: 3,
1415
+ # &quot;value&quot;: &quot;xt&quot;
1416
+ # },
1417
+ # {
1418
+ # &quot;position&quot;: 5,
1419
+ # &quot;value&quot;: &quot;￰&quot;
1420
+ # }
1421
+ # ]
1422
+ # ]
1423
+ </pre></div>
1424
+ </div>
1425
+ </div>
1426
+ </div>
91
1427
  </div>
92
1428
 
93
1429
 
94
1430
  </div>
95
1431
  </div>
96
1432
  </div>
97
- <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
1433
+ <div class="sphinxsidebar">
98
1434
  <div class="sphinxsidebarwrapper">
1435
+ <h3><a href="../index.html">Table Of Contents</a></h3>
1436
+ <ul>
1437
+ <li><a class="reference internal" href="#">7.8. Tokenizers</a><ul>
1438
+ <li><a class="reference internal" href="#summary">7.8.1. Summary</a></li>
1439
+ <li><a class="reference internal" href="#what-is-tokenize">7.8.2. What is &quot;tokenize&quot;?</a></li>
1440
+ <li><a class="reference internal" href="#built-in-tokenizsers">7.8.3. Built-in tokenizsers</a><ul>
1441
+ <li><a class="reference internal" href="#tokenbigram">7.8.3.1. <tt class="docutils literal"><span class="pre">TokenBigram</span></tt></a></li>
1442
+ <li><a class="reference internal" href="#tokenbigramsplitsymbol">7.8.3.2. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></tt></a></li>
1443
+ <li><a class="reference internal" href="#tokenbigramsplitsymbolalpha">7.8.3.3. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt></a></li>
1444
+ <li><a class="reference internal" href="#tokenbigramsplitsymbolalphadigit">7.8.3.4. <tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></tt></a></li>
1445
+ <li><a class="reference internal" href="#tokenbigramignoreblank">7.8.3.5. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></tt></a></li>
1446
+ <li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbol">7.8.3.6. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></tt></a></li>
1447
+ <li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalpha">7.8.3.7. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></tt></a></li>
1448
+ <li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalphadigit">7.8.3.8. <tt class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></tt></a></li>
1449
+ <li><a class="reference internal" href="#tokenunigram">7.8.3.9. <tt class="docutils literal"><span class="pre">TokenUnigram</span></tt></a></li>
1450
+ <li><a class="reference internal" href="#tokentrigram">7.8.3.10. <tt class="docutils literal"><span class="pre">TokenTrigram</span></tt></a></li>
1451
+ <li><a class="reference internal" href="#tokendelimit">7.8.3.11. <tt class="docutils literal"><span class="pre">TokenDelimit</span></tt></a></li>
1452
+ <li><a class="reference internal" href="#tokendelimitnull">7.8.3.12. <tt class="docutils literal"><span class="pre">TokenDelimitNull</span></tt></a></li>
1453
+ <li><a class="reference internal" href="#tokenmecab">7.8.3.13. <tt class="docutils literal"><span class="pre">TokenMecab</span></tt></a></li>
1454
+ <li><a class="reference internal" href="#tokenregexp">7.8.3.14. <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt></a></li>
1455
+ </ul>
1456
+ </li>
1457
+ </ul>
1458
+ </li>
1459
+ </ul>
1460
+
99
1461
  <h4>Previous topic</h4>
100
1462
  <p class="topless"><a href="normalizers.html"
101
1463
  title="previous chapter">7.7. Normalizers</a></p>
102
1464
  <h4>Next topic</h4>
103
1465
  <p class="topless"><a href="token_filters.html"
104
1466
  title="next chapter">7.9. Token filters</a></p>
105
- <div role="note" aria-label="source link">
106
- <h3>This Page</h3>
107
- <ul class="this-page-menu">
108
- <li><a href="../_sources/reference/tokenizers.txt"
109
- rel="nofollow">Show Source</a></li>
110
- </ul>
111
- </div>
112
- <div id="searchbox" style="display: none" role="search">
1467
+ <h3>This Page</h3>
1468
+ <ul class="this-page-menu">
1469
+ <li><a href="../_sources/reference/tokenizers.txt"
1470
+ rel="nofollow">Show Source</a></li>
1471
+ </ul>
1472
+ <div id="searchbox" style="display: none">
113
1473
  <h3>Quick search</h3>
114
1474
  <form class="search" action="../search.html" method="get">
115
1475
  <input type="text" name="q" />
@@ -126,7 +1486,7 @@
126
1486
  </div>
127
1487
  <div class="clearer"></div>
128
1488
  </div>
129
- <div class="related" role="navigation" aria-label="related navigation">
1489
+ <div class="related">
130
1490
  <h3>Navigation</h3>
131
1491
  <ul>
132
1492
  <li class="right" style="margin-right: 10px">
@@ -138,11 +1498,11 @@
138
1498
  <li class="right" >
139
1499
  <a href="normalizers.html" title="7.7. Normalizers"
140
1500
  >previous</a> |</li>
141
- <li><a href="../index.html">Groonga v5.0.0 documentation</a> &raquo;</li>
1501
+ <li><a href="../index.html">Groonga v5.0.1-42-g4d10df1 documentation</a> &raquo;</li>
142
1502
  <li><a href="../reference.html" >7. Reference manual</a> &raquo;</li>
143
1503
  </ul>
144
1504
  </div>
145
- <div class="footer" role="contentinfo">
1505
+ <div class="footer">
146
1506
  &copy; Copyright 2009-2015, Brazil, Inc.
147
1507
  </div>
148
1508
  </body>