rroonga 5.0.3-x86-mingw32 → 5.0.4-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
 - data/README.md +1 -1
 - data/ext/groonga/rb-grn-column.c +2 -0
 - data/ext/groonga/rb-grn-database.c +1 -0
 - data/ext/groonga/rb-grn-flushable.c +81 -0
 - data/ext/groonga/rb-grn-table.c +1 -0
 - data/ext/groonga/rb-grn.h +3 -1
 - data/ext/groonga/rb-groonga.c +1 -0
 - data/lib/1.9/groonga.so +0 -0
 - data/lib/2.0/groonga.so +0 -0
 - data/lib/2.1/groonga.so +0 -0
 - data/lib/2.2/groonga.so +0 -0
 - data/rroonga-build.rb +3 -3
 - data/test/test-flushable.rb +49 -0
 - data/vendor/local/bin/grndb.exe +0 -0
 - data/vendor/local/bin/groonga-benchmark.exe +0 -0
 - data/vendor/local/bin/groonga.exe +0 -0
 - data/vendor/local/bin/libgroonga-0.dll +0 -0
 - data/vendor/local/bin/libmecab-1.dll +0 -0
 - data/vendor/local/bin/libmsgpack-4.dll +0 -0
 - data/vendor/local/bin/libmsgpackc-2.dll +0 -0
 - data/vendor/local/bin/libonig-5.dll +0 -0
 - data/vendor/local/bin/lz4.exe +0 -0
 - data/vendor/local/bin/lz4c.exe +0 -0
 - data/vendor/local/bin/lz4cat +0 -0
 - data/vendor/local/bin/mecab-config +2 -2
 - data/vendor/local/bin/mecab.exe +0 -0
 - data/vendor/local/bin/onig-config +1 -1
 - data/vendor/local/bin/zlib1.dll +0 -0
 - data/vendor/local/etc/groonga/httpd/groonga-httpd.conf +2 -2
 - data/vendor/local/include/groonga/groonga/groonga.h +2 -0
 - data/vendor/local/include/groonga/groonga/plugin.h +5 -0
 - data/vendor/local/lib/groonga/plugins/functions/vector.a +0 -0
 - data/vendor/local/lib/groonga/plugins/functions/vector.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/functions/vector.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/functions/vector.la +2 -2
 - data/vendor/local/lib/groonga/plugins/query_expanders/tsv.a +0 -0
 - data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/query_expanders/tsv.la +2 -2
 - data/vendor/local/lib/groonga/plugins/ruby/eval.a +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/eval.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/eval.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/eval.la +2 -2
 - data/vendor/local/lib/groonga/plugins/ruby/load.a +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/load.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/load.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/ruby/load.la +2 -2
 - data/vendor/local/lib/groonga/plugins/sharding.rb +2 -0
 - data/vendor/local/lib/groonga/plugins/sharding/logical_enumerator.rb +135 -21
 - data/vendor/local/lib/groonga/plugins/sharding/logical_select.rb +174 -0
 - data/vendor/local/lib/groonga/plugins/sharding/logical_table_remove.rb +70 -0
 - data/vendor/local/lib/groonga/plugins/suggest/suggest.a +0 -0
 - data/vendor/local/lib/groonga/plugins/suggest/suggest.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/suggest/suggest.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/suggest/suggest.la +2 -2
 - data/vendor/local/lib/groonga/plugins/table/table.a +0 -0
 - data/vendor/local/lib/groonga/plugins/table/table.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/table/table.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/table/table.la +2 -2
 - data/vendor/local/lib/groonga/plugins/token_filters/stop_word.a +0 -0
 - data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la +2 -2
 - data/vendor/local/lib/groonga/plugins/tokenizers/mecab.a +0 -0
 - data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll +0 -0
 - data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll.a +0 -0
 - data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la +2 -2
 - data/vendor/local/lib/groonga/scripts/ruby/initialize/post.rb +1 -0
 - data/vendor/local/lib/groonga/scripts/ruby/table.rb +77 -0
 - data/vendor/local/lib/libgroonga.a +0 -0
 - data/vendor/local/lib/libgroonga.dll.a +0 -0
 - data/vendor/local/lib/libgroonga.la +2 -2
 - data/vendor/local/lib/liblz4.dll +0 -0
 - data/vendor/local/lib/liblz4.dll.1 +0 -0
 - data/vendor/local/lib/liblz4.dll.1.5.0 +0 -0
 - data/vendor/local/lib/libmecab.dll.a +0 -0
 - data/vendor/local/lib/libmecab.la +2 -2
 - data/vendor/local/lib/libmsgpack.a +0 -0
 - data/vendor/local/lib/libmsgpack.dll.a +0 -0
 - data/vendor/local/lib/libmsgpack.la +2 -2
 - data/vendor/local/lib/libmsgpackc.a +0 -0
 - data/vendor/local/lib/libmsgpackc.dll.a +0 -0
 - data/vendor/local/lib/libmsgpackc.la +2 -2
 - data/vendor/local/lib/libonig.a +0 -0
 - data/vendor/local/lib/libonig.dll.a +0 -0
 - data/vendor/local/lib/libonig.la +2 -2
 - data/vendor/local/lib/libz.dll.a +0 -0
 - data/vendor/local/lib/pkgconfig/groonga.pc +3 -3
 - data/vendor/local/lib/pkgconfig/liblz4.pc +5 -5
 - data/vendor/local/lib/pkgconfig/msgpack.pc +1 -1
 - data/vendor/local/lib/pkgconfig/oniguruma.pc +6 -6
 - data/vendor/local/lib/pkgconfig/zlib.pc +3 -3
 - data/vendor/local/libexec/mecab/mecab-cost-train.exe +0 -0
 - data/vendor/local/libexec/mecab/mecab-dict-gen.exe +0 -0
 - data/vendor/local/libexec/mecab/mecab-dict-index.exe +0 -0
 - data/vendor/local/libexec/mecab/mecab-system-eval.exe +0 -0
 - data/vendor/local/libexec/mecab/mecab-test-gen.exe +0 -0
 - data/vendor/local/sbin/groonga-httpd-restart +1 -2
 - data/vendor/local/sbin/groonga-httpd.exe +0 -0
 - data/vendor/local/share/doc/groonga/en/html/.buildinfo +1 -1
 - data/vendor/local/share/doc/groonga/en/html/_sources/development/travis-ci.txt +6 -1
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/centos.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/debian.txt +4 -4
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/fedora.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/mac_os_x.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/others.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/solaris.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/ubuntu.txt +3 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/install/windows.txt +9 -9
 - data/vendor/local/share/doc/groonga/en/html/_sources/news.txt +67 -6
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/dump.txt +11 -3
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/io_flush.txt +60 -0
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_select.txt +100 -0
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_table_remove.txt +94 -0
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/executables/groonga.txt +2 -2
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/normalizers.txt +1 -1
 - data/vendor/local/share/doc/groonga/en/html/_sources/reference/tuning.txt +32 -30
 - data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/match_columns.txt +6 -6
 - data/vendor/local/share/doc/groonga/en/html/characteristic.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/client.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/community.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/com.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/cooperation.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/query.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/release.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/repository.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/development/test.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/documentation.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/documentation/c-api.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/documentation/i18n.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/documentation/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/contribution/report.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/development.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/development/travis-ci.html +11 -7
 - data/vendor/local/share/doc/groonga/en/html/genindex.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/index.html +36 -33
 - data/vendor/local/share/doc/groonga/en/html/install.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/install/centos.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/debian.html +9 -9
 - data/vendor/local/share/doc/groonga/en/html/install/fedora.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/mac_os_x.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/others.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/solaris.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/ubuntu.html +8 -8
 - data/vendor/local/share/doc/groonga/en/html/install/windows.html +14 -14
 - data/vendor/local/share/doc/groonga/en/html/limitations.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news.html +129 -46
 - data/vendor/local/share/doc/groonga/en/html/news/0.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/1.0.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/1.1.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/1.2.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/1.3.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/2.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/3.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/4.x.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/news/senna.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/objects.inv +0 -0
 - data/vendor/local/share/doc/groonga/en/html/reference.html +36 -33
 - data/vendor/local/share/doc/groonga/en/html/reference/api.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/global_configurations.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_cache.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_column.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_command_version.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_content_type.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ctx.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_db.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_encoding.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_expr.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_geo.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_hook.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ii.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_index_cursor.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_info.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_match_escalation.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_obj.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_proc.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_search.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table_cursor.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_type.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/grn_user_data.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/overview.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/api/plugin.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/cast.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/column.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/columns/index.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/columns/pseudo.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/columns/scalar.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/columns/vector.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/command.html +36 -33
 - data/vendor/local/share/doc/groonga/en/html/reference/command/command_version.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/command/output_format.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/command/request_id.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/command/return_code.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/cache_limit.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/check.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/clearlock.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/column_create.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/column_list.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/column_remove.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/column_rename.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/define_selector.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/defrag.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/delete.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/dump.html +23 -12
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/io_flush.html +197 -0
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/load.html +30 -30
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/lock_clear.html +27 -27
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/log_level.html +27 -27
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/log_put.html +27 -27
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/log_reopen.html +29 -29
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_count.html +43 -43
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_range_filter.html +50 -50
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_select.html +238 -0
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_table_remove.html +232 -0
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/normalize.html +38 -38
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/normalizer_list.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_register.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_unregister.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/quit.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/range_filter.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/register.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/request_cancel.html +31 -31
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_eval.html +29 -29
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_load.html +29 -29
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/select.html +107 -107
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/shutdown.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/status.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/suggest.html +27 -27
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/table_create.html +49 -49
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/table_list.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/table_remove.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/table_tokenize.html +39 -39
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenize.html +238 -105
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenizer_list.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/commands/truncate.html +25 -25
 - data/vendor/local/share/doc/groonga/en/html/reference/executables.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/grndb.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/grnslap.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-benchmark.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-server-http.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-create-dataset.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-learner.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga.html +6 -6
 - data/vendor/local/share/doc/groonga/en/html/reference/function.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/between.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/edit_distance.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_distance.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_circle.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_rectangle.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_full.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_html.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/html_untag.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/in_values.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/now.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/query.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/rand.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/snippet_html.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/sub_filter.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/functions/vector_size.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/grn_expr.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/query_syntax.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/script_syntax.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/indexing.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/log.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/normalizers.html +6 -6
 - data/vendor/local/share/doc/groonga/en/html/reference/operations.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/operations/geolocation_search.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/output.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/query_expanders.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/query_expanders/tsv.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/regular_expression.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/scorer.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_at_most.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_idf.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/scoring_note.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/suggest.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/suggest/completion.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/suggest/correction.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/suggest/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/suggest/suggestion.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/tables.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/token_filters.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/tokenizers.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/reference/tuning.html +37 -35
 - data/vendor/local/share/doc/groonga/en/html/reference/types.html +9 -9
 - data/vendor/local/share/doc/groonga/en/html/search.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/searchindex.js +1 -1
 - data/vendor/local/share/doc/groonga/en/html/server.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/gqtp.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/http.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/http/comparison.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/http/groonga-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/http/groonga.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/memcached.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/server/package.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/spec.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/spec/gqtp.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/spec/search.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/troubleshooting.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/troubleshooting/different_results_with_the_same_keyword.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/troubleshooting/mmap_cannot_allocate_memory.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/data.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/drilldown.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/index.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/lexicon.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/match_columns.html +7 -7
 - data/vendor/local/share/doc/groonga/en/html/tutorial/micro_blog.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/network.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/patricia_trie.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/query_expansion.html +5 -5
 - data/vendor/local/share/doc/groonga/en/html/tutorial/search.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/.buildinfo +1 -1
 - data/vendor/local/share/doc/groonga/ja/html/_sources/development/travis-ci.txt +6 -1
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/centos.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/debian.txt +4 -4
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/fedora.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/mac_os_x.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/others.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/solaris.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/ubuntu.txt +3 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/install/windows.txt +9 -9
 - data/vendor/local/share/doc/groonga/ja/html/_sources/news.txt +67 -6
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/dump.txt +11 -3
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/io_flush.txt +60 -0
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_select.txt +100 -0
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_table_remove.txt +94 -0
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/executables/groonga.txt +2 -2
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/normalizers.txt +1 -1
 - data/vendor/local/share/doc/groonga/ja/html/_sources/reference/tuning.txt +32 -30
 - data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/match_columns.txt +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/characteristic.html +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/client.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/community.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/com.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/cooperation.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/query.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/release.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/repository.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/development/test.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/documentation.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/c-api.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/i18n.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/contribution/report.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/development.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/development/travis-ci.html +9 -7
 - data/vendor/local/share/doc/groonga/ja/html/genindex.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/index.html +37 -34
 - data/vendor/local/share/doc/groonga/ja/html/install.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/install/centos.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/debian.html +9 -9
 - data/vendor/local/share/doc/groonga/ja/html/install/fedora.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/mac_os_x.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/others.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/solaris.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/ubuntu.html +8 -8
 - data/vendor/local/share/doc/groonga/ja/html/install/windows.html +14 -14
 - data/vendor/local/share/doc/groonga/ja/html/limitations.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news.html +126 -47
 - data/vendor/local/share/doc/groonga/ja/html/news/0.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/1.0.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/1.1.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/1.2.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/1.3.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/2.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/3.x.html +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/news/4.x.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/news/senna.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/objects.inv +0 -0
 - data/vendor/local/share/doc/groonga/ja/html/reference.html +37 -34
 - data/vendor/local/share/doc/groonga/ja/html/reference/api.html +9 -9
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/global_configurations.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_cache.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_column.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_command_version.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_content_type.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ctx.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_db.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_encoding.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_expr.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_geo.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_hook.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ii.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_index_cursor.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_info.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_match_escalation.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_obj.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_proc.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_search.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table_cursor.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_type.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_user_data.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/overview.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/api/plugin.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/cast.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/column.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/columns/index.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/columns/pseudo.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/columns/scalar.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/columns/vector.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/command.html +36 -33
 - data/vendor/local/share/doc/groonga/ja/html/reference/command/command_version.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/command/output_format.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/command/request_id.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/command/return_code.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/cache_limit.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/check.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/clearlock.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_create.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_list.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_remove.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_rename.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/define_selector.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/defrag.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/delete.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/dump.html +23 -12
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/io_flush.html +198 -0
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/load.html +30 -30
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/lock_clear.html +27 -27
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_level.html +27 -27
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_put.html +27 -27
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_reopen.html +29 -29
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_count.html +43 -43
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_range_filter.html +50 -50
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_select.html +236 -0
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_table_remove.html +230 -0
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalize.html +38 -38
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalizer_list.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_register.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_unregister.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/quit.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/range_filter.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/register.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/request_cancel.html +31 -31
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_eval.html +29 -29
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_load.html +29 -29
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/select.html +107 -107
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/shutdown.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/status.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/suggest.html +27 -27
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_create.html +49 -49
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_list.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_remove.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_tokenize.html +39 -39
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenize.html +238 -105
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenizer_list.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/commands/truncate.html +25 -25
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/grndb.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/grnslap.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-benchmark.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-server-http.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-create-dataset.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-learner.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga.html +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/reference/function.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/between.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/edit_distance.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_distance.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_circle.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_rectangle.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_full.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_html.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/html_untag.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/in_values.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/now.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/query.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/rand.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/snippet_html.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/sub_filter.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/functions/vector_size.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/query_syntax.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/script_syntax.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/indexing.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/log.html +9 -9
 - data/vendor/local/share/doc/groonga/ja/html/reference/normalizers.html +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/reference/operations.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/operations/geolocation_search.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/output.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders/tsv.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/regular_expression.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/scorer.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_at_most.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_idf.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/scoring_note.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/suggest.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/suggest/completion.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/suggest/correction.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/suggest/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/suggest/suggestion.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/tables.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/token_filters.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/tokenizers.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/reference/tuning.html +41 -71
 - data/vendor/local/share/doc/groonga/ja/html/reference/types.html +9 -9
 - data/vendor/local/share/doc/groonga/ja/html/search.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/searchindex.js +1 -1
 - data/vendor/local/share/doc/groonga/ja/html/server.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/gqtp.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/http.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/http/comparison.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/http/groonga-httpd.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/http/groonga.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/memcached.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/server/package.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/spec.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/spec/gqtp.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/spec/search.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/troubleshooting.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/troubleshooting/different_results_with_the_same_keyword.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/troubleshooting/mmap_cannot_allocate_memory.html +6 -6
 - data/vendor/local/share/doc/groonga/ja/html/tutorial.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/data.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/drilldown.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/index.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/introduction.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/lexicon.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/match_columns.html +7 -7
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/micro_blog.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/network.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/patricia_trie.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/query_expansion.html +5 -5
 - data/vendor/local/share/doc/groonga/ja/html/tutorial/search.html +5 -5
 - data/vendor/local/share/man/ja/man1/groonga.1 +487 -164
 - data/vendor/local/share/man/man1/groonga.1 +492 -129
 - metadata +20 -2
 
| 
         @@ -7,7 +7,7 @@ 
     | 
|
| 
       7 
7 
     | 
    
         
             
              <head>
         
     | 
| 
       8 
8 
     | 
    
         
             
                <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
         
     | 
| 
       9 
9 
     | 
    
         | 
| 
       10 
     | 
    
         
            -
                <title>7.3. 
     | 
| 
      
 10 
     | 
    
         
            +
                <title>7.3.44. tokenize — Groonga v5.0.4-139-g6629adbドキュメント</title>
         
     | 
| 
       11 
11 
     | 
    
         | 
| 
       12 
12 
     | 
    
         
             
                <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
         
     | 
| 
       13 
13 
     | 
    
         
             
                <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
         
     | 
| 
         @@ -15,7 +15,7 @@ 
     | 
|
| 
       15 
15 
     | 
    
         
             
                <script type="text/javascript">
         
     | 
| 
       16 
16 
     | 
    
         
             
                  var DOCUMENTATION_OPTIONS = {
         
     | 
| 
       17 
17 
     | 
    
         
             
                    URL_ROOT:    '../../',
         
     | 
| 
       18 
     | 
    
         
            -
                    VERSION:     '5.0.4',
         
     | 
| 
      
 18 
     | 
    
         
            +
                    VERSION:     '5.0.4-139-g6629adb',
         
     | 
| 
       19 
19 
     | 
    
         
             
                    COLLAPSE_INDEX: false,
         
     | 
| 
       20 
20 
     | 
    
         
             
                    FILE_SUFFIX: '.html',
         
     | 
| 
       21 
21 
     | 
    
         
             
                    HAS_SOURCE:  true
         
     | 
| 
         @@ -26,10 +26,10 @@ 
     | 
|
| 
       26 
26 
     | 
    
         
             
                <script type="text/javascript" src="../../_static/doctools.js"></script>
         
     | 
| 
       27 
27 
     | 
    
         
             
                <script type="text/javascript" src="../../_static/translations.js"></script>
         
     | 
| 
       28 
28 
     | 
    
         
             
                <link rel="shortcut icon" href="../../_static/favicon.ico"/>
         
     | 
| 
       29 
     | 
    
         
            -
                <link rel="top" title="Groonga v5.0.4ドキュメント" href="../../index.html" />
         
     | 
| 
      
 29 
     | 
    
         
            +
                <link rel="top" title="Groonga v5.0.4-139-g6629adbドキュメント" href="../../index.html" />
         
     | 
| 
       30 
30 
     | 
    
         
             
                <link rel="up" title="7.3. コマンド" href="../command.html" />
         
     | 
| 
       31 
     | 
    
         
            -
                <link rel="next" title="7.3. 
     | 
| 
       32 
     | 
    
         
            -
                <link rel="prev" title="7.3. 
     | 
| 
      
 31 
     | 
    
         
            +
                <link rel="next" title="7.3.45. tokenizer_list" href="tokenizer_list.html" />
         
     | 
| 
      
 32 
     | 
    
         
            +
                <link rel="prev" title="7.3.43. table_tokenize" href="table_tokenize.html" /> 
         
     | 
| 
       33 
33 
     | 
    
         
             
              </head>
         
     | 
| 
       34 
34 
     | 
    
         
             
              <body>
         
     | 
| 
       35 
35 
     | 
    
         
             
            <div class="header">
         
     | 
| 
         @@ -56,12 +56,12 @@ 
     | 
|
| 
       56 
56 
     | 
    
         
             
                      <a href="../../genindex.html" title="総合索引"
         
     | 
| 
       57 
57 
     | 
    
         
             
                         accesskey="I">索引</a></li>
         
     | 
| 
       58 
58 
     | 
    
         
             
                    <li class="right" >
         
     | 
| 
       59 
     | 
    
         
            -
                      <a href="tokenizer_list.html" title="7.3. 
     | 
| 
      
 59 
     | 
    
         
            +
                      <a href="tokenizer_list.html" title="7.3.45. tokenizer_list"
         
     | 
| 
       60 
60 
     | 
    
         
             
                         accesskey="N">次へ</a> |</li>
         
     | 
| 
       61 
61 
     | 
    
         
             
                    <li class="right" >
         
     | 
| 
       62 
     | 
    
         
            -
                      <a href="table_tokenize.html" title="7.3. 
     | 
| 
      
 62 
     | 
    
         
            +
                      <a href="table_tokenize.html" title="7.3.43. table_tokenize"
         
     | 
| 
       63 
63 
     | 
    
         
             
                         accesskey="P">前へ</a> |</li>
         
     | 
| 
       64 
     | 
    
         
            -
                    <li><a href="../../index.html">Groonga v5.0.4ドキュメント</a> »</li>
         
     | 
| 
      
 64 
     | 
    
         
            +
                    <li><a href="../../index.html">Groonga v5.0.4-139-g6629adbドキュメント</a> »</li>
         
     | 
| 
       65 
65 
     | 
    
         
             
                      <li><a href="../../reference.html" >7. リファレンスマニュアル</a> »</li>
         
     | 
| 
       66 
66 
     | 
    
         
             
                      <li><a href="../command.html" accesskey="U">7.3. コマンド</a> »</li> 
         
     | 
| 
       67 
67 
     | 
    
         
             
                  </ul>
         
     | 
| 
         @@ -73,13 +73,13 @@ 
     | 
|
| 
       73 
73 
     | 
    
         
             
                      <div class="body">
         
     | 
| 
       74 
74 
     | 
    
         | 
| 
       75 
75 
     | 
    
         
             
              <div class="section" id="tokenize">
         
     | 
| 
       76 
     | 
    
         
            -
            <h1>7.3. 
     | 
| 
      
 76 
     | 
    
         
            +
            <h1>7.3.44. <tt class="docutils literal"><span class="pre">tokenize</span></tt><a class="headerlink" href="#tokenize" title="このヘッドラインへのパーマリンク">¶</a></h1>
         
     | 
| 
       77 
77 
     | 
    
         
             
            <div class="section" id="summary">
         
     | 
| 
       78 
     | 
    
         
            -
            <h2>7.3. 
     | 
| 
      
 78 
     | 
    
         
            +
            <h2>7.3.44.1. 概要<a class="headerlink" href="#summary" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       79 
79 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドは指定したトークナイザーでテキストをトークナイズします。これはトークナイズ処理のデバッグに便利です。</p>
         
     | 
| 
       80 
80 
     | 
    
         
             
            </div>
         
     | 
| 
       81 
81 
     | 
    
         
             
            <div class="section" id="syntax">
         
     | 
| 
       82 
     | 
    
         
            -
            <h2>7.3. 
     | 
| 
      
 82 
     | 
    
         
            +
            <h2>7.3.44.2. 構文<a class="headerlink" href="#syntax" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       83 
83 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドには必須の引数と省略可能な引数があります。 <tt class="docutils literal"><span class="pre">tokenizer</span></tt> と <tt class="docutils literal"><span class="pre">string</span></tt> が必須の引数で、他の引数はすべて省略可能です。</p>
         
     | 
| 
       84 
84 
     | 
    
         
             
            <div class="highlight-none"><div class="highlight"><pre>tokenize tokenizer
         
     | 
| 
       85 
85 
     | 
    
         
             
                     string
         
     | 
| 
         @@ -91,7 +91,7 @@ 
     | 
|
| 
       91 
91 
     | 
    
         
             
            </div>
         
     | 
| 
       92 
92 
     | 
    
         
             
            </div>
         
     | 
| 
       93 
93 
     | 
    
         
             
            <div class="section" id="usage">
         
     | 
| 
       94 
     | 
    
         
            -
            <h2>7.3. 
     | 
| 
      
 94 
     | 
    
         
            +
            <h2>7.3.44.3. 使い方<a class="headerlink" href="#usage" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       95 
95 
     | 
    
         
             
            <p>以下は簡単な使用例です。</p>
         
     | 
| 
       96 
96 
     | 
    
         
             
            <p>実行例:</p>
         
     | 
| 
       97 
97 
     | 
    
         
             
            <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Fulltext Search"
         
     | 
| 
         @@ -104,62 +104,77 @@ 
     | 
|
| 
       104 
104 
     | 
    
         
             
            #   [
         
     | 
| 
       105 
105 
     | 
    
         
             
            #     {
         
     | 
| 
       106 
106 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 107 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       107 
108 
     | 
    
         
             
            #       "value": "Fu"
         
     | 
| 
       108 
109 
     | 
    
         
             
            #     },
         
     | 
| 
       109 
110 
     | 
    
         
             
            #     {
         
     | 
| 
       110 
111 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 112 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       111 
113 
     | 
    
         
             
            #       "value": "ul"
         
     | 
| 
       112 
114 
     | 
    
         
             
            #     },
         
     | 
| 
       113 
115 
     | 
    
         
             
            #     {
         
     | 
| 
       114 
116 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 117 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       115 
118 
     | 
    
         
             
            #       "value": "ll"
         
     | 
| 
       116 
119 
     | 
    
         
             
            #     },
         
     | 
| 
       117 
120 
     | 
    
         
             
            #     {
         
     | 
| 
       118 
121 
     | 
    
         
             
            #       "position": 3,
         
     | 
| 
      
 122 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       119 
123 
     | 
    
         
             
            #       "value": "lt"
         
     | 
| 
       120 
124 
     | 
    
         
             
            #     },
         
     | 
| 
       121 
125 
     | 
    
         
             
            #     {
         
     | 
| 
       122 
126 
     | 
    
         
             
            #       "position": 4,
         
     | 
| 
      
 127 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       123 
128 
     | 
    
         
             
            #       "value": "te"
         
     | 
| 
       124 
129 
     | 
    
         
             
            #     },
         
     | 
| 
       125 
130 
     | 
    
         
             
            #     {
         
     | 
| 
       126 
131 
     | 
    
         
             
            #       "position": 5,
         
     | 
| 
      
 132 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       127 
133 
     | 
    
         
             
            #       "value": "ex"
         
     | 
| 
       128 
134 
     | 
    
         
             
            #     },
         
     | 
| 
       129 
135 
     | 
    
         
             
            #     {
         
     | 
| 
       130 
136 
     | 
    
         
             
            #       "position": 6,
         
     | 
| 
      
 137 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       131 
138 
     | 
    
         
             
            #       "value": "xt"
         
     | 
| 
       132 
139 
     | 
    
         
             
            #     },
         
     | 
| 
       133 
140 
     | 
    
         
             
            #     {
         
     | 
| 
       134 
141 
     | 
    
         
             
            #       "position": 7,
         
     | 
| 
      
 142 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       135 
143 
     | 
    
         
             
            #       "value": "t "
         
     | 
| 
       136 
144 
     | 
    
         
             
            #     },
         
     | 
| 
       137 
145 
     | 
    
         
             
            #     {
         
     | 
| 
       138 
146 
     | 
    
         
             
            #       "position": 8,
         
     | 
| 
      
 147 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       139 
148 
     | 
    
         
             
            #       "value": " S"
         
     | 
| 
       140 
149 
     | 
    
         
             
            #     },
         
     | 
| 
       141 
150 
     | 
    
         
             
            #     {
         
     | 
| 
       142 
151 
     | 
    
         
             
            #       "position": 9,
         
     | 
| 
      
 152 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       143 
153 
     | 
    
         
             
            #       "value": "Se"
         
     | 
| 
       144 
154 
     | 
    
         
             
            #     },
         
     | 
| 
       145 
155 
     | 
    
         
             
            #     {
         
     | 
| 
       146 
156 
     | 
    
         
             
            #       "position": 10,
         
     | 
| 
      
 157 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       147 
158 
     | 
    
         
             
            #       "value": "ea"
         
     | 
| 
       148 
159 
     | 
    
         
             
            #     },
         
     | 
| 
       149 
160 
     | 
    
         
             
            #     {
         
     | 
| 
       150 
161 
     | 
    
         
             
            #       "position": 11,
         
     | 
| 
      
 162 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       151 
163 
     | 
    
         
             
            #       "value": "ar"
         
     | 
| 
       152 
164 
     | 
    
         
             
            #     },
         
     | 
| 
       153 
165 
     | 
    
         
             
            #     {
         
     | 
| 
       154 
166 
     | 
    
         
             
            #       "position": 12,
         
     | 
| 
      
 167 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       155 
168 
     | 
    
         
             
            #       "value": "rc"
         
     | 
| 
       156 
169 
     | 
    
         
             
            #     },
         
     | 
| 
       157 
170 
     | 
    
         
             
            #     {
         
     | 
| 
       158 
171 
     | 
    
         
             
            #       "position": 13,
         
     | 
| 
      
 172 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       159 
173 
     | 
    
         
             
            #       "value": "ch"
         
     | 
| 
       160 
174 
     | 
    
         
             
            #     },
         
     | 
| 
       161 
175 
     | 
    
         
             
            #     {
         
     | 
| 
       162 
176 
     | 
    
         
             
            #       "position": 14,
         
     | 
| 
      
 177 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       163 
178 
     | 
    
         
             
            #       "value": "h"
         
     | 
| 
       164 
179 
     | 
    
         
             
            #     }
         
     | 
| 
       165 
180 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -169,13 +184,13 @@ 
     | 
|
| 
       169 
184 
     | 
    
         
             
            <p>この例では必須のパラメーターだけ使っています。 <tt class="docutils literal"><span class="pre">tokenizer</span></tt> には <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> を、 <tt class="docutils literal"><span class="pre">string</span></tt> には <tt class="docutils literal"><span class="pre">"Fulltext</span> <span class="pre">Search"</span></tt> を指定しています。この例は <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> トークナイザーで <tt class="docutils literal"><span class="pre">"Fulltext</span> <span class="pre">Search"</span></tt> をトークナイズしたトークンを返します。この例では <tt class="docutils literal"><span class="pre">"Fulltext</span> <span class="pre">Search"</span></tt> を正規化していません。</p>
         
     | 
| 
       170 
185 
     | 
    
         
             
            </div>
         
     | 
| 
       171 
186 
     | 
    
         
             
            <div class="section" id="parameters">
         
     | 
| 
       172 
     | 
    
         
            -
            <h2>7.3. 
     | 
| 
      
 187 
     | 
    
         
            +
            <h2>7.3.44.4. 引数<a class="headerlink" href="#parameters" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       173 
188 
     | 
    
         
             
            <p>このセクションではすべての引数について説明します。引数はカテゴリわけしています。</p>
         
     | 
| 
       174 
189 
     | 
    
         
             
            <div class="section" id="required-parameters">
         
     | 
| 
       175 
     | 
    
         
            -
            <h3>7.3. 
     | 
| 
      
 190 
     | 
    
         
            +
            <h3>7.3.44.4.1. 必須引数<a class="headerlink" href="#required-parameters" title="このヘッドラインへのパーマリンク">¶</a></h3>
         
     | 
| 
       176 
191 
     | 
    
         
             
            <p>必須引数は二つあります。 <tt class="docutils literal"><span class="pre">tokenizer</span></tt> と <tt class="docutils literal"><span class="pre">string</span></tt> です。</p>
         
     | 
| 
       177 
192 
     | 
    
         
             
            <div class="section" id="tokenizer">
         
     | 
| 
       178 
     | 
    
         
            -
            <span id="tokenize-tokenizer"></span><h4>7.3. 
     | 
| 
      
 193 
     | 
    
         
            +
            <span id="tokenize-tokenizer"></span><h4>7.3.44.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt><a class="headerlink" href="#tokenizer" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       179 
194 
     | 
    
         
             
            <p>トークナイザー名を指定します。 <tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドは <tt class="docutils literal"><span class="pre">tokenizer</span></tt> で指定された名前のトークナイザーを使います。</p>
         
     | 
| 
       180 
195 
     | 
    
         
             
            <p>組み込みのトークナイザーについては <a class="reference internal" href="../tokenizers.html"><em>トークナイザー</em></a> を参照してください。</p>
         
     | 
| 
       181 
196 
     | 
    
         
             
            <p>以下は組み込みの <tt class="docutils literal"><span class="pre">TokenTrigram</span></tt> トークナイザーを使う例です。</p>
         
     | 
| 
         @@ -190,62 +205,77 @@ 
     | 
|
| 
       190 
205 
     | 
    
         
             
            #   [
         
     | 
| 
       191 
206 
     | 
    
         
             
            #     {
         
     | 
| 
       192 
207 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 208 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       193 
209 
     | 
    
         
             
            #       "value": "Ful"
         
     | 
| 
       194 
210 
     | 
    
         
             
            #     },
         
     | 
| 
       195 
211 
     | 
    
         
             
            #     {
         
     | 
| 
       196 
212 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 213 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       197 
214 
     | 
    
         
             
            #       "value": "ull"
         
     | 
| 
       198 
215 
     | 
    
         
             
            #     },
         
     | 
| 
       199 
216 
     | 
    
         
             
            #     {
         
     | 
| 
       200 
217 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 218 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       201 
219 
     | 
    
         
             
            #       "value": "llt"
         
     | 
| 
       202 
220 
     | 
    
         
             
            #     },
         
     | 
| 
       203 
221 
     | 
    
         
             
            #     {
         
     | 
| 
       204 
222 
     | 
    
         
             
            #       "position": 3,
         
     | 
| 
      
 223 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       205 
224 
     | 
    
         
             
            #       "value": "lte"
         
     | 
| 
       206 
225 
     | 
    
         
             
            #     },
         
     | 
| 
       207 
226 
     | 
    
         
             
            #     {
         
     | 
| 
       208 
227 
     | 
    
         
             
            #       "position": 4,
         
     | 
| 
      
 228 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       209 
229 
     | 
    
         
             
            #       "value": "tex"
         
     | 
| 
       210 
230 
     | 
    
         
             
            #     },
         
     | 
| 
       211 
231 
     | 
    
         
             
            #     {
         
     | 
| 
       212 
232 
     | 
    
         
             
            #       "position": 5,
         
     | 
| 
      
 233 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       213 
234 
     | 
    
         
             
            #       "value": "ext"
         
     | 
| 
       214 
235 
     | 
    
         
             
            #     },
         
     | 
| 
       215 
236 
     | 
    
         
             
            #     {
         
     | 
| 
       216 
237 
     | 
    
         
             
            #       "position": 6,
         
     | 
| 
      
 238 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       217 
239 
     | 
    
         
             
            #       "value": "xt "
         
     | 
| 
       218 
240 
     | 
    
         
             
            #     },
         
     | 
| 
       219 
241 
     | 
    
         
             
            #     {
         
     | 
| 
       220 
242 
     | 
    
         
             
            #       "position": 7,
         
     | 
| 
      
 243 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       221 
244 
     | 
    
         
             
            #       "value": "t S"
         
     | 
| 
       222 
245 
     | 
    
         
             
            #     },
         
     | 
| 
       223 
246 
     | 
    
         
             
            #     {
         
     | 
| 
       224 
247 
     | 
    
         
             
            #       "position": 8,
         
     | 
| 
      
 248 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       225 
249 
     | 
    
         
             
            #       "value": " Se"
         
     | 
| 
       226 
250 
     | 
    
         
             
            #     },
         
     | 
| 
       227 
251 
     | 
    
         
             
            #     {
         
     | 
| 
       228 
252 
     | 
    
         
             
            #       "position": 9,
         
     | 
| 
      
 253 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       229 
254 
     | 
    
         
             
            #       "value": "Sea"
         
     | 
| 
       230 
255 
     | 
    
         
             
            #     },
         
     | 
| 
       231 
256 
     | 
    
         
             
            #     {
         
     | 
| 
       232 
257 
     | 
    
         
             
            #       "position": 10,
         
     | 
| 
      
 258 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       233 
259 
     | 
    
         
             
            #       "value": "ear"
         
     | 
| 
       234 
260 
     | 
    
         
             
            #     },
         
     | 
| 
       235 
261 
     | 
    
         
             
            #     {
         
     | 
| 
       236 
262 
     | 
    
         
             
            #       "position": 11,
         
     | 
| 
      
 263 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       237 
264 
     | 
    
         
             
            #       "value": "arc"
         
     | 
| 
       238 
265 
     | 
    
         
             
            #     },
         
     | 
| 
       239 
266 
     | 
    
         
             
            #     {
         
     | 
| 
       240 
267 
     | 
    
         
             
            #       "position": 12,
         
     | 
| 
      
 268 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       241 
269 
     | 
    
         
             
            #       "value": "rch"
         
     | 
| 
       242 
270 
     | 
    
         
             
            #     },
         
     | 
| 
       243 
271 
     | 
    
         
             
            #     {
         
     | 
| 
       244 
272 
     | 
    
         
             
            #       "position": 13,
         
     | 
| 
      
 273 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       245 
274 
     | 
    
         
             
            #       "value": "ch"
         
     | 
| 
       246 
275 
     | 
    
         
             
            #     },
         
     | 
| 
       247 
276 
     | 
    
         
             
            #     {
         
     | 
| 
       248 
277 
     | 
    
         
             
            #       "position": 14,
         
     | 
| 
      
 278 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       249 
279 
     | 
    
         
             
            #       "value": "h"
         
     | 
| 
       250 
280 
     | 
    
         
             
            #     }
         
     | 
| 
       251 
281 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -255,7 +285,7 @@ 
     | 
|
| 
       255 
285 
     | 
    
         
             
            <p>他のトークナイザーを使いたい場合は、 <a class="reference internal" href="register.html"><em>register</em></a> コマンドでトークナイザープラグインを登録する必要があります。例えば、 <a class="reference external" href="http://www.phontron.com/kytea/">KyTea</a> ベースのトークナイザーを <tt class="docutils literal"><span class="pre">tokenizers/kytea</span></tt> を登録することで使えます。</p>
         
     | 
| 
       256 
286 
     | 
    
         
             
            </div>
         
     | 
| 
       257 
287 
     | 
    
         
             
            <div class="section" id="string">
         
     | 
| 
       258 
     | 
    
         
            -
            <span id="tokenize-string"></span><h4>7.3. 
     | 
| 
      
 288 
     | 
    
         
            +
            <span id="tokenize-string"></span><h4>7.3.44.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt><a class="headerlink" href="#string" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       259 
289 
     | 
    
         
             
            <p>トークナイズしたい文字列を指定します。</p>
         
     | 
| 
       260 
290 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">string</span></tt> の中に文字列を含める場合は、シングルクォート( <tt class="docutils literal"><span class="pre">'</span></tt> )またはダブルクォート( <tt class="docutils literal"><span class="pre">"</span></tt> )で <tt class="docutils literal"><span class="pre">string</span></tt> をクォートする必要があります。</p>
         
     | 
| 
       261 
291 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">string</span></tt> の中で空白を使う例です。</p>
         
     | 
| 
         @@ -270,162 +300,202 @@ 
     | 
|
| 
       270 
300 
     | 
    
         
             
            #   [
         
     | 
| 
       271 
301 
     | 
    
         
             
            #     {
         
     | 
| 
       272 
302 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 303 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       273 
304 
     | 
    
         
             
            #       "value": "Gr"
         
     | 
| 
       274 
305 
     | 
    
         
             
            #     },
         
     | 
| 
       275 
306 
     | 
    
         
             
            #     {
         
     | 
| 
       276 
307 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 308 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       277 
309 
     | 
    
         
             
            #       "value": "ro"
         
     | 
| 
       278 
310 
     | 
    
         
             
            #     },
         
     | 
| 
       279 
311 
     | 
    
         
             
            #     {
         
     | 
| 
       280 
312 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 313 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       281 
314 
     | 
    
         
             
            #       "value": "oo"
         
     | 
| 
       282 
315 
     | 
    
         
             
            #     },
         
     | 
| 
       283 
316 
     | 
    
         
             
            #     {
         
     | 
| 
       284 
317 
     | 
    
         
             
            #       "position": 3,
         
     | 
| 
      
 318 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       285 
319 
     | 
    
         
             
            #       "value": "on"
         
     | 
| 
       286 
320 
     | 
    
         
             
            #     },
         
     | 
| 
       287 
321 
     | 
    
         
             
            #     {
         
     | 
| 
       288 
322 
     | 
    
         
             
            #       "position": 4,
         
     | 
| 
      
 323 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       289 
324 
     | 
    
         
             
            #       "value": "ng"
         
     | 
| 
       290 
325 
     | 
    
         
             
            #     },
         
     | 
| 
       291 
326 
     | 
    
         
             
            #     {
         
     | 
| 
       292 
327 
     | 
    
         
             
            #       "position": 5,
         
     | 
| 
      
 328 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       293 
329 
     | 
    
         
             
            #       "value": "ga"
         
     | 
| 
       294 
330 
     | 
    
         
             
            #     },
         
     | 
| 
       295 
331 
     | 
    
         
             
            #     {
         
     | 
| 
       296 
332 
     | 
    
         
             
            #       "position": 6,
         
     | 
| 
      
 333 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       297 
334 
     | 
    
         
             
            #       "value": "a "
         
     | 
| 
       298 
335 
     | 
    
         
             
            #     },
         
     | 
| 
       299 
336 
     | 
    
         
             
            #     {
         
     | 
| 
       300 
337 
     | 
    
         
             
            #       "position": 7,
         
     | 
| 
      
 338 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       301 
339 
     | 
    
         
             
            #       "value": " i"
         
     | 
| 
       302 
340 
     | 
    
         
             
            #     },
         
     | 
| 
       303 
341 
     | 
    
         
             
            #     {
         
     | 
| 
       304 
342 
     | 
    
         
             
            #       "position": 8,
         
     | 
| 
      
 343 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       305 
344 
     | 
    
         
             
            #       "value": "is"
         
     | 
| 
       306 
345 
     | 
    
         
             
            #     },
         
     | 
| 
       307 
346 
     | 
    
         
             
            #     {
         
     | 
| 
       308 
347 
     | 
    
         
             
            #       "position": 9,
         
     | 
| 
      
 348 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       309 
349 
     | 
    
         
             
            #       "value": "s "
         
     | 
| 
       310 
350 
     | 
    
         
             
            #     },
         
     | 
| 
       311 
351 
     | 
    
         
             
            #     {
         
     | 
| 
       312 
352 
     | 
    
         
             
            #       "position": 10,
         
     | 
| 
      
 353 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       313 
354 
     | 
    
         
             
            #       "value": " a"
         
     | 
| 
       314 
355 
     | 
    
         
             
            #     },
         
     | 
| 
       315 
356 
     | 
    
         
             
            #     {
         
     | 
| 
       316 
357 
     | 
    
         
             
            #       "position": 11,
         
     | 
| 
      
 358 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       317 
359 
     | 
    
         
             
            #       "value": "a "
         
     | 
| 
       318 
360 
     | 
    
         
             
            #     },
         
     | 
| 
       319 
361 
     | 
    
         
             
            #     {
         
     | 
| 
       320 
362 
     | 
    
         
             
            #       "position": 12,
         
     | 
| 
      
 363 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       321 
364 
     | 
    
         
             
            #       "value": " f"
         
     | 
| 
       322 
365 
     | 
    
         
             
            #     },
         
     | 
| 
       323 
366 
     | 
    
         
             
            #     {
         
     | 
| 
       324 
367 
     | 
    
         
             
            #       "position": 13,
         
     | 
| 
      
 368 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       325 
369 
     | 
    
         
             
            #       "value": "fa"
         
     | 
| 
       326 
370 
     | 
    
         
             
            #     },
         
     | 
| 
       327 
371 
     | 
    
         
             
            #     {
         
     | 
| 
       328 
372 
     | 
    
         
             
            #       "position": 14,
         
     | 
| 
      
 373 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       329 
374 
     | 
    
         
             
            #       "value": "as"
         
     | 
| 
       330 
375 
     | 
    
         
             
            #     },
         
     | 
| 
       331 
376 
     | 
    
         
             
            #     {
         
     | 
| 
       332 
377 
     | 
    
         
             
            #       "position": 15,
         
     | 
| 
      
 378 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       333 
379 
     | 
    
         
             
            #       "value": "st"
         
     | 
| 
       334 
380 
     | 
    
         
             
            #     },
         
     | 
| 
       335 
381 
     | 
    
         
             
            #     {
         
     | 
| 
       336 
382 
     | 
    
         
             
            #       "position": 16,
         
     | 
| 
      
 383 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       337 
384 
     | 
    
         
             
            #       "value": "t "
         
     | 
| 
       338 
385 
     | 
    
         
             
            #     },
         
     | 
| 
       339 
386 
     | 
    
         
             
            #     {
         
     | 
| 
       340 
387 
     | 
    
         
             
            #       "position": 17,
         
     | 
| 
      
 388 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       341 
389 
     | 
    
         
             
            #       "value": " f"
         
     | 
| 
       342 
390 
     | 
    
         
             
            #     },
         
     | 
| 
       343 
391 
     | 
    
         
             
            #     {
         
     | 
| 
       344 
392 
     | 
    
         
             
            #       "position": 18,
         
     | 
| 
      
 393 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       345 
394 
     | 
    
         
             
            #       "value": "fu"
         
     | 
| 
       346 
395 
     | 
    
         
             
            #     },
         
     | 
| 
       347 
396 
     | 
    
         
             
            #     {
         
     | 
| 
       348 
397 
     | 
    
         
             
            #       "position": 19,
         
     | 
| 
      
 398 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       349 
399 
     | 
    
         
             
            #       "value": "ul"
         
     | 
| 
       350 
400 
     | 
    
         
             
            #     },
         
     | 
| 
       351 
401 
     | 
    
         
             
            #     {
         
     | 
| 
       352 
402 
     | 
    
         
             
            #       "position": 20,
         
     | 
| 
      
 403 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       353 
404 
     | 
    
         
             
            #       "value": "ll"
         
     | 
| 
       354 
405 
     | 
    
         
             
            #     },
         
     | 
| 
       355 
406 
     | 
    
         
             
            #     {
         
     | 
| 
       356 
407 
     | 
    
         
             
            #       "position": 21,
         
     | 
| 
      
 408 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       357 
409 
     | 
    
         
             
            #       "value": "lt"
         
     | 
| 
       358 
410 
     | 
    
         
             
            #     },
         
     | 
| 
       359 
411 
     | 
    
         
             
            #     {
         
     | 
| 
       360 
412 
     | 
    
         
             
            #       "position": 22,
         
     | 
| 
      
 413 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       361 
414 
     | 
    
         
             
            #       "value": "te"
         
     | 
| 
       362 
415 
     | 
    
         
             
            #     },
         
     | 
| 
       363 
416 
     | 
    
         
             
            #     {
         
     | 
| 
       364 
417 
     | 
    
         
             
            #       "position": 23,
         
     | 
| 
      
 418 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       365 
419 
     | 
    
         
             
            #       "value": "ex"
         
     | 
| 
       366 
420 
     | 
    
         
             
            #     },
         
     | 
| 
       367 
421 
     | 
    
         
             
            #     {
         
     | 
| 
       368 
422 
     | 
    
         
             
            #       "position": 24,
         
     | 
| 
      
 423 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       369 
424 
     | 
    
         
             
            #       "value": "xt"
         
     | 
| 
       370 
425 
     | 
    
         
             
            #     },
         
     | 
| 
       371 
426 
     | 
    
         
             
            #     {
         
     | 
| 
       372 
427 
     | 
    
         
             
            #       "position": 25,
         
     | 
| 
      
 428 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       373 
429 
     | 
    
         
             
            #       "value": "t "
         
     | 
| 
       374 
430 
     | 
    
         
             
            #     },
         
     | 
| 
       375 
431 
     | 
    
         
             
            #     {
         
     | 
| 
       376 
432 
     | 
    
         
             
            #       "position": 26,
         
     | 
| 
      
 433 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       377 
434 
     | 
    
         
             
            #       "value": " e"
         
     | 
| 
       378 
435 
     | 
    
         
             
            #     },
         
     | 
| 
       379 
436 
     | 
    
         
             
            #     {
         
     | 
| 
       380 
437 
     | 
    
         
             
            #       "position": 27,
         
     | 
| 
      
 438 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       381 
439 
     | 
    
         
             
            #       "value": "ea"
         
     | 
| 
       382 
440 
     | 
    
         
             
            #     },
         
     | 
| 
       383 
441 
     | 
    
         
             
            #     {
         
     | 
| 
       384 
442 
     | 
    
         
             
            #       "position": 28,
         
     | 
| 
      
 443 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       385 
444 
     | 
    
         
             
            #       "value": "ar"
         
     | 
| 
       386 
445 
     | 
    
         
             
            #     },
         
     | 
| 
       387 
446 
     | 
    
         
             
            #     {
         
     | 
| 
       388 
447 
     | 
    
         
             
            #       "position": 29,
         
     | 
| 
      
 448 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       389 
449 
     | 
    
         
             
            #       "value": "rc"
         
     | 
| 
       390 
450 
     | 
    
         
             
            #     },
         
     | 
| 
       391 
451 
     | 
    
         
             
            #     {
         
     | 
| 
       392 
452 
     | 
    
         
             
            #       "position": 30,
         
     | 
| 
      
 453 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       393 
454 
     | 
    
         
             
            #       "value": "ch"
         
     | 
| 
       394 
455 
     | 
    
         
             
            #     },
         
     | 
| 
       395 
456 
     | 
    
         
             
            #     {
         
     | 
| 
       396 
457 
     | 
    
         
             
            #       "position": 31,
         
     | 
| 
      
 458 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       397 
459 
     | 
    
         
             
            #       "value": "h "
         
     | 
| 
       398 
460 
     | 
    
         
             
            #     },
         
     | 
| 
       399 
461 
     | 
    
         
             
            #     {
         
     | 
| 
       400 
462 
     | 
    
         
             
            #       "position": 32,
         
     | 
| 
      
 463 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       401 
464 
     | 
    
         
             
            #       "value": " e"
         
     | 
| 
       402 
465 
     | 
    
         
             
            #     },
         
     | 
| 
       403 
466 
     | 
    
         
             
            #     {
         
     | 
| 
       404 
467 
     | 
    
         
             
            #       "position": 33,
         
     | 
| 
      
 468 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       405 
469 
     | 
    
         
             
            #       "value": "en"
         
     | 
| 
       406 
470 
     | 
    
         
             
            #     },
         
     | 
| 
       407 
471 
     | 
    
         
             
            #     {
         
     | 
| 
       408 
472 
     | 
    
         
             
            #       "position": 34,
         
     | 
| 
      
 473 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       409 
474 
     | 
    
         
             
            #       "value": "ng"
         
     | 
| 
       410 
475 
     | 
    
         
             
            #     },
         
     | 
| 
       411 
476 
     | 
    
         
             
            #     {
         
     | 
| 
       412 
477 
     | 
    
         
             
            #       "position": 35,
         
     | 
| 
      
 478 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       413 
479 
     | 
    
         
             
            #       "value": "gi"
         
     | 
| 
       414 
480 
     | 
    
         
             
            #     },
         
     | 
| 
       415 
481 
     | 
    
         
             
            #     {
         
     | 
| 
       416 
482 
     | 
    
         
             
            #       "position": 36,
         
     | 
| 
      
 483 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       417 
484 
     | 
    
         
             
            #       "value": "in"
         
     | 
| 
       418 
485 
     | 
    
         
             
            #     },
         
     | 
| 
       419 
486 
     | 
    
         
             
            #     {
         
     | 
| 
       420 
487 
     | 
    
         
             
            #       "position": 37,
         
     | 
| 
      
 488 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       421 
489 
     | 
    
         
             
            #       "value": "ne"
         
     | 
| 
       422 
490 
     | 
    
         
             
            #     },
         
     | 
| 
       423 
491 
     | 
    
         
             
            #     {
         
     | 
| 
       424 
492 
     | 
    
         
             
            #       "position": 38,
         
     | 
| 
      
 493 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       425 
494 
     | 
    
         
             
            #       "value": "e!"
         
     | 
| 
       426 
495 
     | 
    
         
             
            #     },
         
     | 
| 
       427 
496 
     | 
    
         
             
            #     {
         
     | 
| 
       428 
497 
     | 
    
         
             
            #       "position": 39,
         
     | 
| 
      
 498 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       429 
499 
     | 
    
         
             
            #       "value": "!"
         
     | 
| 
       430 
500 
     | 
    
         
             
            #     }
         
     | 
| 
       431 
501 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -435,10 +505,10 @@ 
     | 
|
| 
       435 
505 
     | 
    
         
             
            </div>
         
     | 
| 
       436 
506 
     | 
    
         
             
            </div>
         
     | 
| 
       437 
507 
     | 
    
         
             
            <div class="section" id="optional-parameters">
         
     | 
| 
       438 
     | 
    
         
            -
            <h3>7.3. 
     | 
| 
      
 508 
     | 
    
         
            +
            <h3>7.3.44.4.2. 省略可能引数<a class="headerlink" href="#optional-parameters" title="このヘッドラインへのパーマリンク">¶</a></h3>
         
     | 
| 
       439 
509 
     | 
    
         
             
            <p>いくつか省略可能な引数があります。</p>
         
     | 
| 
       440 
510 
     | 
    
         
             
            <div class="section" id="normalizer">
         
     | 
| 
       441 
     | 
    
         
            -
            <span id="tokenize-normalizer"></span><h4>7.3. 
     | 
| 
      
 511 
     | 
    
         
            +
            <span id="tokenize-normalizer"></span><h4>7.3.44.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt><a class="headerlink" href="#normalizer" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       442 
512 
     | 
    
         
             
            <p>ノーマライザー名を指定します。 <tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドは <tt class="docutils literal"><span class="pre">normalizer</span></tt> という名前のノーマライザーを使います。ノーマライザーは <tt class="docutils literal"><span class="pre">TokenBigrma</span></tt> など、N-gram関連のトークナイザーにとってとても重要です。</p>
         
     | 
| 
       443 
513 
     | 
    
         
             
            <p>ノーマライザーはノーマライズ中にそれぞれの文字の種類を検出します。N-gram系のトークナイザーはトークナイズ中に文字の種類を利用します。</p>
         
     | 
| 
       444 
514 
     | 
    
         
             
            <p>以下はノーマライザーを使わない例です。</p>
         
     | 
| 
         @@ -453,62 +523,77 @@ 
     | 
|
| 
       453 
523 
     | 
    
         
             
            #   [
         
     | 
| 
       454 
524 
     | 
    
         
             
            #     {
         
     | 
| 
       455 
525 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 526 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       456 
527 
     | 
    
         
             
            #       "value": "Fu"
         
     | 
| 
       457 
528 
     | 
    
         
             
            #     },
         
     | 
| 
       458 
529 
     | 
    
         
             
            #     {
         
     | 
| 
       459 
530 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 531 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       460 
532 
     | 
    
         
             
            #       "value": "ul"
         
     | 
| 
       461 
533 
     | 
    
         
             
            #     },
         
     | 
| 
       462 
534 
     | 
    
         
             
            #     {
         
     | 
| 
       463 
535 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 536 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       464 
537 
     | 
    
         
             
            #       "value": "ll"
         
     | 
| 
       465 
538 
     | 
    
         
             
            #     },
         
     | 
| 
       466 
539 
     | 
    
         
             
            #     {
         
     | 
| 
       467 
540 
     | 
    
         
             
            #       "position": 3,
         
     | 
| 
      
 541 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       468 
542 
     | 
    
         
             
            #       "value": "lt"
         
     | 
| 
       469 
543 
     | 
    
         
             
            #     },
         
     | 
| 
       470 
544 
     | 
    
         
             
            #     {
         
     | 
| 
       471 
545 
     | 
    
         
             
            #       "position": 4,
         
     | 
| 
      
 546 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       472 
547 
     | 
    
         
             
            #       "value": "te"
         
     | 
| 
       473 
548 
     | 
    
         
             
            #     },
         
     | 
| 
       474 
549 
     | 
    
         
             
            #     {
         
     | 
| 
       475 
550 
     | 
    
         
             
            #       "position": 5,
         
     | 
| 
      
 551 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       476 
552 
     | 
    
         
             
            #       "value": "ex"
         
     | 
| 
       477 
553 
     | 
    
         
             
            #     },
         
     | 
| 
       478 
554 
     | 
    
         
             
            #     {
         
     | 
| 
       479 
555 
     | 
    
         
             
            #       "position": 6,
         
     | 
| 
      
 556 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       480 
557 
     | 
    
         
             
            #       "value": "xt"
         
     | 
| 
       481 
558 
     | 
    
         
             
            #     },
         
     | 
| 
       482 
559 
     | 
    
         
             
            #     {
         
     | 
| 
       483 
560 
     | 
    
         
             
            #       "position": 7,
         
     | 
| 
      
 561 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       484 
562 
     | 
    
         
             
            #       "value": "t "
         
     | 
| 
       485 
563 
     | 
    
         
             
            #     },
         
     | 
| 
       486 
564 
     | 
    
         
             
            #     {
         
     | 
| 
       487 
565 
     | 
    
         
             
            #       "position": 8,
         
     | 
| 
      
 566 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       488 
567 
     | 
    
         
             
            #       "value": " S"
         
     | 
| 
       489 
568 
     | 
    
         
             
            #     },
         
     | 
| 
       490 
569 
     | 
    
         
             
            #     {
         
     | 
| 
       491 
570 
     | 
    
         
             
            #       "position": 9,
         
     | 
| 
      
 571 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       492 
572 
     | 
    
         
             
            #       "value": "Se"
         
     | 
| 
       493 
573 
     | 
    
         
             
            #     },
         
     | 
| 
       494 
574 
     | 
    
         
             
            #     {
         
     | 
| 
       495 
575 
     | 
    
         
             
            #       "position": 10,
         
     | 
| 
      
 576 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       496 
577 
     | 
    
         
             
            #       "value": "ea"
         
     | 
| 
       497 
578 
     | 
    
         
             
            #     },
         
     | 
| 
       498 
579 
     | 
    
         
             
            #     {
         
     | 
| 
       499 
580 
     | 
    
         
             
            #       "position": 11,
         
     | 
| 
      
 581 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       500 
582 
     | 
    
         
             
            #       "value": "ar"
         
     | 
| 
       501 
583 
     | 
    
         
             
            #     },
         
     | 
| 
       502 
584 
     | 
    
         
             
            #     {
         
     | 
| 
       503 
585 
     | 
    
         
             
            #       "position": 12,
         
     | 
| 
      
 586 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       504 
587 
     | 
    
         
             
            #       "value": "rc"
         
     | 
| 
       505 
588 
     | 
    
         
             
            #     },
         
     | 
| 
       506 
589 
     | 
    
         
             
            #     {
         
     | 
| 
       507 
590 
     | 
    
         
             
            #       "position": 13,
         
     | 
| 
      
 591 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       508 
592 
     | 
    
         
             
            #       "value": "ch"
         
     | 
| 
       509 
593 
     | 
    
         
             
            #     },
         
     | 
| 
       510 
594 
     | 
    
         
             
            #     {
         
     | 
| 
       511 
595 
     | 
    
         
             
            #       "position": 14,
         
     | 
| 
      
 596 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       512 
597 
     | 
    
         
             
            #       "value": "h"
         
     | 
| 
       513 
598 
     | 
    
         
             
            #     }
         
     | 
| 
       514 
599 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -528,10 +613,12 @@ 
     | 
|
| 
       528 
613 
     | 
    
         
             
            #   [
         
     | 
| 
       529 
614 
     | 
    
         
             
            #     {
         
     | 
| 
       530 
615 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 616 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       531 
617 
     | 
    
         
             
            #       "value": "fulltext"
         
     | 
| 
       532 
618 
     | 
    
         
             
            #     },
         
     | 
| 
       533 
619 
     | 
    
         
             
            #     {
         
     | 
| 
       534 
620 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 621 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       535 
622 
     | 
    
         
             
            #       "value": "search"
         
     | 
| 
       536 
623 
     | 
    
         
             
            #     }
         
     | 
| 
       537 
624 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -551,58 +638,72 @@ 
     | 
|
| 
       551 
638 
     | 
    
         
             
            #   [
         
     | 
| 
       552 
639 
     | 
    
         
             
            #     {
         
     | 
| 
       553 
640 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 641 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       554 
642 
     | 
    
         
             
            #       "value": "fu"
         
     | 
| 
       555 
643 
     | 
    
         
             
            #     },
         
     | 
| 
       556 
644 
     | 
    
         
             
            #     {
         
     | 
| 
       557 
645 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 646 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       558 
647 
     | 
    
         
             
            #       "value": "ul"
         
     | 
| 
       559 
648 
     | 
    
         
             
            #     },
         
     | 
| 
       560 
649 
     | 
    
         
             
            #     {
         
     | 
| 
       561 
650 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 651 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       562 
652 
     | 
    
         
             
            #       "value": "ll"
         
     | 
| 
       563 
653 
     | 
    
         
             
            #     },
         
     | 
| 
       564 
654 
     | 
    
         
             
            #     {
         
     | 
| 
       565 
655 
     | 
    
         
             
            #       "position": 3,
         
     | 
| 
      
 656 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       566 
657 
     | 
    
         
             
            #       "value": "lt"
         
     | 
| 
       567 
658 
     | 
    
         
             
            #     },
         
     | 
| 
       568 
659 
     | 
    
         
             
            #     {
         
     | 
| 
       569 
660 
     | 
    
         
             
            #       "position": 4,
         
     | 
| 
      
 661 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       570 
662 
     | 
    
         
             
            #       "value": "te"
         
     | 
| 
       571 
663 
     | 
    
         
             
            #     },
         
     | 
| 
       572 
664 
     | 
    
         
             
            #     {
         
     | 
| 
       573 
665 
     | 
    
         
             
            #       "position": 5,
         
     | 
| 
      
 666 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       574 
667 
     | 
    
         
             
            #       "value": "ex"
         
     | 
| 
       575 
668 
     | 
    
         
             
            #     },
         
     | 
| 
       576 
669 
     | 
    
         
             
            #     {
         
     | 
| 
       577 
670 
     | 
    
         
             
            #       "position": 6,
         
     | 
| 
      
 671 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       578 
672 
     | 
    
         
             
            #       "value": "xt"
         
     | 
| 
       579 
673 
     | 
    
         
             
            #     },
         
     | 
| 
       580 
674 
     | 
    
         
             
            #     {
         
     | 
| 
       581 
675 
     | 
    
         
             
            #       "position": 7,
         
     | 
| 
      
 676 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       582 
677 
     | 
    
         
             
            #       "value": "t"
         
     | 
| 
       583 
678 
     | 
    
         
             
            #     },
         
     | 
| 
       584 
679 
     | 
    
         
             
            #     {
         
     | 
| 
       585 
680 
     | 
    
         
             
            #       "position": 8,
         
     | 
| 
      
 681 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       586 
682 
     | 
    
         
             
            #       "value": "se"
         
     | 
| 
       587 
683 
     | 
    
         
             
            #     },
         
     | 
| 
       588 
684 
     | 
    
         
             
            #     {
         
     | 
| 
       589 
685 
     | 
    
         
             
            #       "position": 9,
         
     | 
| 
      
 686 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       590 
687 
     | 
    
         
             
            #       "value": "ea"
         
     | 
| 
       591 
688 
     | 
    
         
             
            #     },
         
     | 
| 
       592 
689 
     | 
    
         
             
            #     {
         
     | 
| 
       593 
690 
     | 
    
         
             
            #       "position": 10,
         
     | 
| 
      
 691 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       594 
692 
     | 
    
         
             
            #       "value": "ar"
         
     | 
| 
       595 
693 
     | 
    
         
             
            #     },
         
     | 
| 
       596 
694 
     | 
    
         
             
            #     {
         
     | 
| 
       597 
695 
     | 
    
         
             
            #       "position": 11,
         
     | 
| 
      
 696 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       598 
697 
     | 
    
         
             
            #       "value": "rc"
         
     | 
| 
       599 
698 
     | 
    
         
             
            #     },
         
     | 
| 
       600 
699 
     | 
    
         
             
            #     {
         
     | 
| 
       601 
700 
     | 
    
         
             
            #       "position": 12,
         
     | 
| 
      
 701 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       602 
702 
     | 
    
         
             
            #       "value": "ch"
         
     | 
| 
       603 
703 
     | 
    
         
             
            #     },
         
     | 
| 
       604 
704 
     | 
    
         
             
            #     {
         
     | 
| 
       605 
705 
     | 
    
         
             
            #       "position": 13,
         
     | 
| 
      
 706 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       606 
707 
     | 
    
         
             
            #       "value": "h"
         
     | 
| 
       607 
708 
     | 
    
         
             
            #     }
         
     | 
| 
       608 
709 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -612,7 +713,7 @@ 
     | 
|
| 
       612 
713 
     | 
    
         
             
            <p>すべてのアルファベットが2文字ごとにトークナイズされています。そして、すべての文字が小文字にノーマライズされています。例えば、 <tt class="docutils literal"><span class="pre">fu</span></tt> で1つのトークンになっています。</p>
         
     | 
| 
       613 
714 
     | 
    
         
             
            </div>
         
     | 
| 
       614 
715 
     | 
    
         
             
            <div class="section" id="flags">
         
     | 
| 
       615 
     | 
    
         
            -
            <span id="tokenize-flags"></span><h4>7.3. 
     | 
| 
      
 716 
     | 
    
         
            +
            <span id="tokenize-flags"></span><h4>7.3.44.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt><a class="headerlink" href="#flags" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       616 
717 
     | 
    
         
             
            <p>トークナイズ処理をカスタマイズするオプションを指定します。「 <tt class="docutils literal"><span class="pre">|</span></tt> 」で区切って複数のオプションを指定することができます。例えば、 <tt class="docutils literal"><span class="pre">NONE|ENABLE_TOKENIZED_DELIMITER</span></tt> というように指定できます。</p>
         
     | 
| 
       617 
718 
     | 
    
         
             
            <p>指定可能なフラグは以下の通りです。</p>
         
     | 
| 
       618 
719 
     | 
    
         
             
            <table border="1" class="docutils">
         
     | 
| 
         @@ -650,14 +751,17 @@ 
     | 
|
| 
       650 
751 
     | 
    
         
             
            #   [
         
     | 
| 
       651 
752 
     | 
    
         
             
            #     {
         
     | 
| 
       652 
753 
     | 
    
         
             
            #       "position": 0,
         
     | 
| 
      
 754 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       653 
755 
     | 
    
         
             
            #       "value": "full"
         
     | 
| 
       654 
756 
     | 
    
         
             
            #     },
         
     | 
| 
       655 
757 
     | 
    
         
             
            #     {
         
     | 
| 
       656 
758 
     | 
    
         
             
            #       "position": 1,
         
     | 
| 
      
 759 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       657 
760 
     | 
    
         
             
            #       "value": "text sea"
         
     | 
| 
       658 
761 
     | 
    
         
             
            #     },
         
     | 
| 
       659 
762 
     | 
    
         
             
            #     {
         
     | 
| 
       660 
763 
     | 
    
         
             
            #       "position": 2,
         
     | 
| 
      
 764 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
       661 
765 
     | 
    
         
             
            #       "value": "crch"
         
     | 
| 
       662 
766 
     | 
    
         
             
            #     }
         
     | 
| 
       663 
767 
     | 
    
         
             
            #   ]
         
     | 
| 
         @@ -667,7 +771,7 @@ 
     | 
|
| 
       667 
771 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">TokenDelimit</span></tt> トークナイザーはトークナイズ済み区切り文字をサポートしているトークナイザーの1つです。 <tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt> でトークナイズ済み区切り文字を有効に出来ます。トークナイズ済み区切り文字はトークンの区切りを表す特別な文字です。この文字は <tt class="docutils literal"><span class="pre">U+FFFE</span></tt> です。この文字コードはどの文字にも割り当てられていません。つまり、通常の文字列にはこの文字は現れません。よって、トークンの区切りを表すという目的には適切な文字です。 <tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt> が有効のときは、対象文字列はすでにトークナイズ済みであると扱われます。トークナイザーは単にトークナイズ済み区切り文字で区切ってトークナイズします。</p>
         
     | 
| 
       668 
772 
     | 
    
         
             
            </div>
         
     | 
| 
       669 
773 
     | 
    
         
             
            <div class="section" id="mode">
         
     | 
| 
       670 
     | 
    
         
            -
            <span id="tokenize-mode"></span><h4>7.3. 
     | 
| 
      
 774 
     | 
    
         
            +
            <span id="tokenize-mode"></span><h4>7.3.44.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt><a class="headerlink" href="#mode" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       671 
775 
     | 
    
         
             
            <p>トークナイズモードを指定します。 <tt class="docutils literal"><span class="pre">ADD</span></tt> を指定すると、ドキュメント追加時と同じルールでトークナイズされます。 <tt class="docutils literal"><span class="pre">GET</span></tt> を指定すると、ドキュメント検索時と同じルールでトークナイズされます。省略された場合、 <tt class="docutils literal"><span class="pre">ADD</span></tt> モードでトークナイズされます。</p>
         
     | 
| 
       672 
776 
     | 
    
         
             
            <p>デフォルトのモードは <tt class="docutils literal"><span class="pre">ADD</span></tt> です。</p>
         
     | 
| 
       673 
777 
     | 
    
         
             
            <p>以下は <tt class="docutils literal"><span class="pre">ADD</span></tt> モードの例です。</p>
         
     | 
| 
         @@ -676,69 +780,84 @@ 
     | 
|
| 
       676 
780 
     | 
    
         
             
            # [
         
     | 
| 
       677 
781 
     | 
    
         
             
            #   [
         
     | 
| 
       678 
782 
     | 
    
         
             
            #     0,
         
     | 
| 
       679 
     | 
    
         
            -
            #      
     | 
| 
       680 
     | 
    
         
            -
            #     0. 
     | 
| 
      
 783 
     | 
    
         
            +
            #     1337566253.89858,
         
     | 
| 
      
 784 
     | 
    
         
            +
            #     0.000355720520019531
         
     | 
| 
       681 
785 
     | 
    
         
             
            #   ],
         
     | 
| 
       682 
786 
     | 
    
         
             
            #   [
         
     | 
| 
       683 
787 
     | 
    
         
             
            #     {
         
     | 
| 
       684 
     | 
    
         
            -
            #       " 
     | 
| 
       685 
     | 
    
         
            -
            #       " 
     | 
| 
      
 788 
     | 
    
         
            +
            #       "position": 0,
         
     | 
| 
      
 789 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 790 
     | 
    
         
            +
            #       "value": "Fu"
         
     | 
| 
       686 
791 
     | 
    
         
             
            #     },
         
     | 
| 
       687 
792 
     | 
    
         
             
            #     {
         
     | 
| 
       688 
     | 
    
         
            -
            #       " 
     | 
| 
       689 
     | 
    
         
            -
            #       " 
     | 
| 
      
 793 
     | 
    
         
            +
            #       "position": 1,
         
     | 
| 
      
 794 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 795 
     | 
    
         
            +
            #       "value": "ul"
         
     | 
| 
       690 
796 
     | 
    
         
             
            #     },
         
     | 
| 
       691 
797 
     | 
    
         
             
            #     {
         
     | 
| 
       692 
     | 
    
         
            -
            #       " 
     | 
| 
       693 
     | 
    
         
            -
            #       " 
     | 
| 
      
 798 
     | 
    
         
            +
            #       "position": 2,
         
     | 
| 
      
 799 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 800 
     | 
    
         
            +
            #       "value": "ll"
         
     | 
| 
       694 
801 
     | 
    
         
             
            #     },
         
     | 
| 
       695 
802 
     | 
    
         
             
            #     {
         
     | 
| 
       696 
     | 
    
         
            -
            #       " 
     | 
| 
       697 
     | 
    
         
            -
            #       " 
     | 
| 
      
 803 
     | 
    
         
            +
            #       "position": 3,
         
     | 
| 
      
 804 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 805 
     | 
    
         
            +
            #       "value": "lt"
         
     | 
| 
       698 
806 
     | 
    
         
             
            #     },
         
     | 
| 
       699 
807 
     | 
    
         
             
            #     {
         
     | 
| 
       700 
     | 
    
         
            -
            #       " 
     | 
| 
       701 
     | 
    
         
            -
            #       " 
     | 
| 
      
 808 
     | 
    
         
            +
            #       "position": 4,
         
     | 
| 
      
 809 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 810 
     | 
    
         
            +
            #       "value": "te"
         
     | 
| 
       702 
811 
     | 
    
         
             
            #     },
         
     | 
| 
       703 
812 
     | 
    
         
             
            #     {
         
     | 
| 
       704 
     | 
    
         
            -
            #       " 
     | 
| 
       705 
     | 
    
         
            -
            #       " 
     | 
| 
      
 813 
     | 
    
         
            +
            #       "position": 5,
         
     | 
| 
      
 814 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 815 
     | 
    
         
            +
            #       "value": "ex"
         
     | 
| 
       706 
816 
     | 
    
         
             
            #     },
         
     | 
| 
       707 
817 
     | 
    
         
             
            #     {
         
     | 
| 
       708 
     | 
    
         
            -
            #       " 
     | 
| 
       709 
     | 
    
         
            -
            #       " 
     | 
| 
      
 818 
     | 
    
         
            +
            #       "position": 6,
         
     | 
| 
      
 819 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 820 
     | 
    
         
            +
            #       "value": "xt"
         
     | 
| 
       710 
821 
     | 
    
         
             
            #     },
         
     | 
| 
       711 
822 
     | 
    
         
             
            #     {
         
     | 
| 
       712 
     | 
    
         
            -
            #       " 
     | 
| 
       713 
     | 
    
         
            -
            #       " 
     | 
| 
      
 823 
     | 
    
         
            +
            #       "position": 7,
         
     | 
| 
      
 824 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 825 
     | 
    
         
            +
            #       "value": "t "
         
     | 
| 
       714 
826 
     | 
    
         
             
            #     },
         
     | 
| 
       715 
827 
     | 
    
         
             
            #     {
         
     | 
| 
       716 
     | 
    
         
            -
            #       " 
     | 
| 
       717 
     | 
    
         
            -
            #       " 
     | 
| 
      
 828 
     | 
    
         
            +
            #       "position": 8,
         
     | 
| 
      
 829 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 830 
     | 
    
         
            +
            #       "value": " S"
         
     | 
| 
       718 
831 
     | 
    
         
             
            #     },
         
     | 
| 
       719 
832 
     | 
    
         
             
            #     {
         
     | 
| 
       720 
     | 
    
         
            -
            #       " 
     | 
| 
       721 
     | 
    
         
            -
            #       " 
     | 
| 
      
 833 
     | 
    
         
            +
            #       "position": 9,
         
     | 
| 
      
 834 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 835 
     | 
    
         
            +
            #       "value": "Se"
         
     | 
| 
       722 
836 
     | 
    
         
             
            #     },
         
     | 
| 
       723 
837 
     | 
    
         
             
            #     {
         
     | 
| 
       724 
     | 
    
         
            -
            #       " 
     | 
| 
       725 
     | 
    
         
            -
            #       " 
     | 
| 
      
 838 
     | 
    
         
            +
            #       "position": 10,
         
     | 
| 
      
 839 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 840 
     | 
    
         
            +
            #       "value": "ea"
         
     | 
| 
       726 
841 
     | 
    
         
             
            #     },
         
     | 
| 
       727 
842 
     | 
    
         
             
            #     {
         
     | 
| 
       728 
     | 
    
         
            -
            #       " 
     | 
| 
       729 
     | 
    
         
            -
            #       " 
     | 
| 
      
 843 
     | 
    
         
            +
            #       "position": 11,
         
     | 
| 
      
 844 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 845 
     | 
    
         
            +
            #       "value": "ar"
         
     | 
| 
       730 
846 
     | 
    
         
             
            #     },
         
     | 
| 
       731 
847 
     | 
    
         
             
            #     {
         
     | 
| 
       732 
     | 
    
         
            -
            #       " 
     | 
| 
       733 
     | 
    
         
            -
            #       " 
     | 
| 
      
 848 
     | 
    
         
            +
            #       "position": 12,
         
     | 
| 
      
 849 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 850 
     | 
    
         
            +
            #       "value": "rc"
         
     | 
| 
       734 
851 
     | 
    
         
             
            #     },
         
     | 
| 
       735 
852 
     | 
    
         
             
            #     {
         
     | 
| 
       736 
     | 
    
         
            -
            #       " 
     | 
| 
       737 
     | 
    
         
            -
            #       " 
     | 
| 
      
 853 
     | 
    
         
            +
            #       "position": 13,
         
     | 
| 
      
 854 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 855 
     | 
    
         
            +
            #       "value": "ch"
         
     | 
| 
       738 
856 
     | 
    
         
             
            #     },
         
     | 
| 
       739 
857 
     | 
    
         
             
            #     {
         
     | 
| 
       740 
     | 
    
         
            -
            #       " 
     | 
| 
       741 
     | 
    
         
            -
            #       " 
     | 
| 
      
 858 
     | 
    
         
            +
            #       "position": 14,
         
     | 
| 
      
 859 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 860 
     | 
    
         
            +
            #       "value": "h"
         
     | 
| 
       742 
861 
     | 
    
         
             
            #     }
         
     | 
| 
       743 
862 
     | 
    
         
             
            #   ]
         
     | 
| 
       744 
863 
     | 
    
         
             
            # ]
         
     | 
| 
         @@ -751,65 +870,79 @@ 
     | 
|
| 
       751 
870 
     | 
    
         
             
            # [
         
     | 
| 
       752 
871 
     | 
    
         
             
            #   [
         
     | 
| 
       753 
872 
     | 
    
         
             
            #     0,
         
     | 
| 
       754 
     | 
    
         
            -
            #      
     | 
| 
       755 
     | 
    
         
            -
            #     0. 
     | 
| 
      
 873 
     | 
    
         
            +
            #     1337566253.89858,
         
     | 
| 
      
 874 
     | 
    
         
            +
            #     0.000355720520019531
         
     | 
| 
       756 
875 
     | 
    
         
             
            #   ],
         
     | 
| 
       757 
876 
     | 
    
         
             
            #   [
         
     | 
| 
       758 
877 
     | 
    
         
             
            #     {
         
     | 
| 
       759 
     | 
    
         
            -
            #       " 
     | 
| 
       760 
     | 
    
         
            -
            #       " 
     | 
| 
      
 878 
     | 
    
         
            +
            #       "position": 0,
         
     | 
| 
      
 879 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 880 
     | 
    
         
            +
            #       "value": "Fu"
         
     | 
| 
       761 
881 
     | 
    
         
             
            #     },
         
     | 
| 
       762 
882 
     | 
    
         
             
            #     {
         
     | 
| 
       763 
     | 
    
         
            -
            #       " 
     | 
| 
       764 
     | 
    
         
            -
            #       " 
     | 
| 
      
 883 
     | 
    
         
            +
            #       "position": 1,
         
     | 
| 
      
 884 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 885 
     | 
    
         
            +
            #       "value": "ul"
         
     | 
| 
       765 
886 
     | 
    
         
             
            #     },
         
     | 
| 
       766 
887 
     | 
    
         
             
            #     {
         
     | 
| 
       767 
     | 
    
         
            -
            #       " 
     | 
| 
       768 
     | 
    
         
            -
            #       " 
     | 
| 
      
 888 
     | 
    
         
            +
            #       "position": 2,
         
     | 
| 
      
 889 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 890 
     | 
    
         
            +
            #       "value": "ll"
         
     | 
| 
       769 
891 
     | 
    
         
             
            #     },
         
     | 
| 
       770 
892 
     | 
    
         
             
            #     {
         
     | 
| 
       771 
     | 
    
         
            -
            #       " 
     | 
| 
       772 
     | 
    
         
            -
            #       " 
     | 
| 
      
 893 
     | 
    
         
            +
            #       "position": 3,
         
     | 
| 
      
 894 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 895 
     | 
    
         
            +
            #       "value": "lt"
         
     | 
| 
       773 
896 
     | 
    
         
             
            #     },
         
     | 
| 
       774 
897 
     | 
    
         
             
            #     {
         
     | 
| 
       775 
     | 
    
         
            -
            #       " 
     | 
| 
       776 
     | 
    
         
            -
            #       " 
     | 
| 
      
 898 
     | 
    
         
            +
            #       "position": 4,
         
     | 
| 
      
 899 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 900 
     | 
    
         
            +
            #       "value": "te"
         
     | 
| 
       777 
901 
     | 
    
         
             
            #     },
         
     | 
| 
       778 
902 
     | 
    
         
             
            #     {
         
     | 
| 
       779 
     | 
    
         
            -
            #       " 
     | 
| 
       780 
     | 
    
         
            -
            #       " 
     | 
| 
      
 903 
     | 
    
         
            +
            #       "position": 5,
         
     | 
| 
      
 904 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 905 
     | 
    
         
            +
            #       "value": "ex"
         
     | 
| 
       781 
906 
     | 
    
         
             
            #     },
         
     | 
| 
       782 
907 
     | 
    
         
             
            #     {
         
     | 
| 
       783 
     | 
    
         
            -
            #       " 
     | 
| 
       784 
     | 
    
         
            -
            #       " 
     | 
| 
      
 908 
     | 
    
         
            +
            #       "position": 6,
         
     | 
| 
      
 909 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 910 
     | 
    
         
            +
            #       "value": "xt"
         
     | 
| 
       785 
911 
     | 
    
         
             
            #     },
         
     | 
| 
       786 
912 
     | 
    
         
             
            #     {
         
     | 
| 
       787 
     | 
    
         
            -
            #       " 
     | 
| 
       788 
     | 
    
         
            -
            #       " 
     | 
| 
      
 913 
     | 
    
         
            +
            #       "position": 7,
         
     | 
| 
      
 914 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 915 
     | 
    
         
            +
            #       "value": "t "
         
     | 
| 
       789 
916 
     | 
    
         
             
            #     },
         
     | 
| 
       790 
917 
     | 
    
         
             
            #     {
         
     | 
| 
       791 
     | 
    
         
            -
            #       " 
     | 
| 
       792 
     | 
    
         
            -
            #       " 
     | 
| 
      
 918 
     | 
    
         
            +
            #       "position": 8,
         
     | 
| 
      
 919 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 920 
     | 
    
         
            +
            #       "value": " S"
         
     | 
| 
       793 
921 
     | 
    
         
             
            #     },
         
     | 
| 
       794 
922 
     | 
    
         
             
            #     {
         
     | 
| 
       795 
     | 
    
         
            -
            #       " 
     | 
| 
       796 
     | 
    
         
            -
            #       " 
     | 
| 
      
 923 
     | 
    
         
            +
            #       "position": 9,
         
     | 
| 
      
 924 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 925 
     | 
    
         
            +
            #       "value": "Se"
         
     | 
| 
       797 
926 
     | 
    
         
             
            #     },
         
     | 
| 
       798 
927 
     | 
    
         
             
            #     {
         
     | 
| 
       799 
     | 
    
         
            -
            #       " 
     | 
| 
       800 
     | 
    
         
            -
            #       " 
     | 
| 
      
 928 
     | 
    
         
            +
            #       "position": 10,
         
     | 
| 
      
 929 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 930 
     | 
    
         
            +
            #       "value": "ea"
         
     | 
| 
       801 
931 
     | 
    
         
             
            #     },
         
     | 
| 
       802 
932 
     | 
    
         
             
            #     {
         
     | 
| 
       803 
     | 
    
         
            -
            #       " 
     | 
| 
       804 
     | 
    
         
            -
            #       " 
     | 
| 
      
 933 
     | 
    
         
            +
            #       "position": 11,
         
     | 
| 
      
 934 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 935 
     | 
    
         
            +
            #       "value": "ar"
         
     | 
| 
       805 
936 
     | 
    
         
             
            #     },
         
     | 
| 
       806 
937 
     | 
    
         
             
            #     {
         
     | 
| 
       807 
     | 
    
         
            -
            #       " 
     | 
| 
       808 
     | 
    
         
            -
            #       " 
     | 
| 
      
 938 
     | 
    
         
            +
            #       "position": 12,
         
     | 
| 
      
 939 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 940 
     | 
    
         
            +
            #       "value": "rc"
         
     | 
| 
       809 
941 
     | 
    
         
             
            #     },
         
     | 
| 
       810 
942 
     | 
    
         
             
            #     {
         
     | 
| 
       811 
     | 
    
         
            -
            #       " 
     | 
| 
       812 
     | 
    
         
            -
            #       " 
     | 
| 
      
 943 
     | 
    
         
            +
            #       "position": 13,
         
     | 
| 
      
 944 
     | 
    
         
            +
            #       "force_prefix": false,
         
     | 
| 
      
 945 
     | 
    
         
            +
            #       "value": "ch"
         
     | 
| 
       813 
946 
     | 
    
         
             
            #     }
         
     | 
| 
       814 
947 
     | 
    
         
             
            #   ]
         
     | 
| 
       815 
948 
     | 
    
         
             
            # ]
         
     | 
| 
         @@ -818,14 +951,14 @@ 
     | 
|
| 
       818 
951 
     | 
    
         
             
            <p>最後のアルファベットは2文字でトークナイズされています。</p>
         
     | 
| 
       819 
952 
     | 
    
         
             
            </div>
         
     | 
| 
       820 
953 
     | 
    
         
             
            <div class="section" id="token-filters">
         
     | 
| 
       821 
     | 
    
         
            -
            <span id="tokenize-token-filters"></span><h4>7.3. 
     | 
| 
      
 954 
     | 
    
         
            +
            <span id="tokenize-token-filters"></span><h4>7.3.44.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt><a class="headerlink" href="#token-filters" title="このヘッドラインへのパーマリンク">¶</a></h4>
         
     | 
| 
       822 
955 
     | 
    
         
             
            <p>トークンフィルターを指定します。 <tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドは <tt class="docutils literal"><span class="pre">token_filters</span></tt> という名前のトークンフィルターを使います。</p>
         
     | 
| 
       823 
956 
     | 
    
         
             
            <p>トークンフィルターについては <a class="reference internal" href="../token_filters.html"><em>トークンフィルター</em></a> を参照してください。</p>
         
     | 
| 
       824 
957 
     | 
    
         
             
            </div>
         
     | 
| 
       825 
958 
     | 
    
         
             
            </div>
         
     | 
| 
       826 
959 
     | 
    
         
             
            </div>
         
     | 
| 
       827 
960 
     | 
    
         
             
            <div class="section" id="return-value">
         
     | 
| 
       828 
     | 
    
         
            -
            <span id="tokenize-return-value"></span><h2>7.3. 
     | 
| 
      
 961 
     | 
    
         
            +
            <span id="tokenize-return-value"></span><h2>7.3.44.5. 戻り値<a class="headerlink" href="#return-value" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       829 
962 
     | 
    
         
             
            <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> コマンドはトークナイズしたトークンをすべて返します。各トークンはトークン自身の文字列情報以外にいくつかの属性を持ちます。属性は今後増えていく可能性があります:</p>
         
     | 
| 
       830 
963 
     | 
    
         
             
            <div class="highlight-none"><div class="highlight"><pre>[HEADER, tokens]
         
     | 
| 
       831 
964 
     | 
    
         
             
            </pre></div>
         
     | 
| 
         @@ -863,7 +996,7 @@ 
     | 
|
| 
       863 
996 
     | 
    
         
             
            </div></blockquote>
         
     | 
| 
       864 
997 
     | 
    
         
             
            </div>
         
     | 
| 
       865 
998 
     | 
    
         
             
            <div class="section" id="see-also">
         
     | 
| 
       866 
     | 
    
         
            -
            <h2>7.3. 
     | 
| 
      
 999 
     | 
    
         
            +
            <h2>7.3.44.6. 参考<a class="headerlink" href="#see-also" title="このヘッドラインへのパーマリンク">¶</a></h2>
         
     | 
| 
       867 
1000 
     | 
    
         
             
            <ul class="simple">
         
     | 
| 
       868 
1001 
     | 
    
         
             
            <li><a class="reference internal" href="../tokenizers.html"><em>トークナイザー</em></a></li>
         
     | 
| 
       869 
1002 
     | 
    
         
             
            </ul>
         
     | 
| 
         @@ -878,37 +1011,37 @@ 
     | 
|
| 
       878 
1011 
     | 
    
         
             
                    <div class="sphinxsidebarwrapper">
         
     | 
| 
       879 
1012 
     | 
    
         
             
              <h3><a href="../../index.html">目次</a></h3>
         
     | 
| 
       880 
1013 
     | 
    
         
             
              <ul>
         
     | 
| 
       881 
     | 
    
         
            -
            <li><a class="reference internal" href="#">7.3. 
     | 
| 
       882 
     | 
    
         
            -
            <li><a class="reference internal" href="#summary">7.3. 
     | 
| 
       883 
     | 
    
         
            -
            <li><a class="reference internal" href="#syntax">7.3. 
     | 
| 
       884 
     | 
    
         
            -
            <li><a class="reference internal" href="#usage">7.3. 
     | 
| 
       885 
     | 
    
         
            -
            <li><a class="reference internal" href="#parameters">7.3. 
     | 
| 
       886 
     | 
    
         
            -
            <li><a class="reference internal" href="#required-parameters">7.3. 
     | 
| 
       887 
     | 
    
         
            -
            <li><a class="reference internal" href="#tokenizer">7.3. 
     | 
| 
       888 
     | 
    
         
            -
            <li><a class="reference internal" href="#string">7.3. 
     | 
| 
      
 1014 
     | 
    
         
            +
            <li><a class="reference internal" href="#">7.3.44. <tt class="docutils literal"><span class="pre">tokenize</span></tt></a><ul>
         
     | 
| 
      
 1015 
     | 
    
         
            +
            <li><a class="reference internal" href="#summary">7.3.44.1. 概要</a></li>
         
     | 
| 
      
 1016 
     | 
    
         
            +
            <li><a class="reference internal" href="#syntax">7.3.44.2. 構文</a></li>
         
     | 
| 
      
 1017 
     | 
    
         
            +
            <li><a class="reference internal" href="#usage">7.3.44.3. 使い方</a></li>
         
     | 
| 
      
 1018 
     | 
    
         
            +
            <li><a class="reference internal" href="#parameters">7.3.44.4. 引数</a><ul>
         
     | 
| 
      
 1019 
     | 
    
         
            +
            <li><a class="reference internal" href="#required-parameters">7.3.44.4.1. 必須引数</a><ul>
         
     | 
| 
      
 1020 
     | 
    
         
            +
            <li><a class="reference internal" href="#tokenizer">7.3.44.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt></a></li>
         
     | 
| 
      
 1021 
     | 
    
         
            +
            <li><a class="reference internal" href="#string">7.3.44.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt></a></li>
         
     | 
| 
       889 
1022 
     | 
    
         
             
            </ul>
         
     | 
| 
       890 
1023 
     | 
    
         
             
            </li>
         
     | 
| 
       891 
     | 
    
         
            -
            <li><a class="reference internal" href="#optional-parameters">7.3. 
     | 
| 
       892 
     | 
    
         
            -
            <li><a class="reference internal" href="#normalizer">7.3. 
     | 
| 
       893 
     | 
    
         
            -
            <li><a class="reference internal" href="#flags">7.3. 
     | 
| 
       894 
     | 
    
         
            -
            <li><a class="reference internal" href="#mode">7.3. 
     | 
| 
       895 
     | 
    
         
            -
            <li><a class="reference internal" href="#token-filters">7.3. 
     | 
| 
      
 1024 
     | 
    
         
            +
            <li><a class="reference internal" href="#optional-parameters">7.3.44.4.2. 省略可能引数</a><ul>
         
     | 
| 
      
 1025 
     | 
    
         
            +
            <li><a class="reference internal" href="#normalizer">7.3.44.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt></a></li>
         
     | 
| 
      
 1026 
     | 
    
         
            +
            <li><a class="reference internal" href="#flags">7.3.44.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt></a></li>
         
     | 
| 
      
 1027 
     | 
    
         
            +
            <li><a class="reference internal" href="#mode">7.3.44.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt></a></li>
         
     | 
| 
      
 1028 
     | 
    
         
            +
            <li><a class="reference internal" href="#token-filters">7.3.44.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt></a></li>
         
     | 
| 
       896 
1029 
     | 
    
         
             
            </ul>
         
     | 
| 
       897 
1030 
     | 
    
         
             
            </li>
         
     | 
| 
       898 
1031 
     | 
    
         
             
            </ul>
         
     | 
| 
       899 
1032 
     | 
    
         
             
            </li>
         
     | 
| 
       900 
     | 
    
         
            -
            <li><a class="reference internal" href="#return-value">7.3. 
     | 
| 
       901 
     | 
    
         
            -
            <li><a class="reference internal" href="#see-also">7.3. 
     | 
| 
      
 1033 
     | 
    
         
            +
            <li><a class="reference internal" href="#return-value">7.3.44.5. 戻り値</a></li>
         
     | 
| 
      
 1034 
     | 
    
         
            +
            <li><a class="reference internal" href="#see-also">7.3.44.6. 参考</a></li>
         
     | 
| 
       902 
1035 
     | 
    
         
             
            </ul>
         
     | 
| 
       903 
1036 
     | 
    
         
             
            </li>
         
     | 
| 
       904 
1037 
     | 
    
         
             
            </ul>
         
     | 
| 
       905 
1038 
     | 
    
         | 
| 
       906 
1039 
     | 
    
         
             
              <h4>前のトピックへ</h4>
         
     | 
| 
       907 
1040 
     | 
    
         
             
              <p class="topless"><a href="table_tokenize.html"
         
     | 
| 
       908 
     | 
    
         
            -
                                    title="前の章へ">7.3. 
     | 
| 
      
 1041 
     | 
    
         
            +
                                    title="前の章へ">7.3.43. <tt class="docutils literal"><span class="pre">table_tokenize</span></tt></a></p>
         
     | 
| 
       909 
1042 
     | 
    
         
             
              <h4>次のトピックへ</h4>
         
     | 
| 
       910 
1043 
     | 
    
         
             
              <p class="topless"><a href="tokenizer_list.html"
         
     | 
| 
       911 
     | 
    
         
            -
                                    title="次の章へ">7.3. 
     | 
| 
      
 1044 
     | 
    
         
            +
                                    title="次の章へ">7.3.45. <tt class="docutils literal"><span class="pre">tokenizer_list</span></tt></a></p>
         
     | 
| 
       912 
1045 
     | 
    
         
             
              <h3>このページ</h3>
         
     | 
| 
       913 
1046 
     | 
    
         
             
              <ul class="this-page-menu">
         
     | 
| 
       914 
1047 
     | 
    
         
             
                <li><a href="../../_sources/reference/commands/tokenize.txt"
         
     | 
| 
         @@ -938,12 +1071,12 @@ 
     | 
|
| 
       938 
1071 
     | 
    
         
             
                      <a href="../../genindex.html" title="総合索引"
         
     | 
| 
       939 
1072 
     | 
    
         
             
                         >索引</a></li>
         
     | 
| 
       940 
1073 
     | 
    
         
             
                    <li class="right" >
         
     | 
| 
       941 
     | 
    
         
            -
                      <a href="tokenizer_list.html" title="7.3. 
     | 
| 
      
 1074 
     | 
    
         
            +
                      <a href="tokenizer_list.html" title="7.3.45. tokenizer_list"
         
     | 
| 
       942 
1075 
     | 
    
         
             
                         >次へ</a> |</li>
         
     | 
| 
       943 
1076 
     | 
    
         
             
                    <li class="right" >
         
     | 
| 
       944 
     | 
    
         
            -
                      <a href="table_tokenize.html" title="7.3. 
     | 
| 
      
 1077 
     | 
    
         
            +
                      <a href="table_tokenize.html" title="7.3.43. table_tokenize"
         
     | 
| 
       945 
1078 
     | 
    
         
             
                         >前へ</a> |</li>
         
     | 
| 
       946 
     | 
    
         
            -
                    <li><a href="../../index.html">Groonga v5.0.4ドキュメント</a> »</li>
         
     | 
| 
      
 1079 
     | 
    
         
            +
                    <li><a href="../../index.html">Groonga v5.0.4-139-g6629adbドキュメント</a> »</li>
         
     | 
| 
       947 
1080 
     | 
    
         
             
                      <li><a href="../../reference.html" >7. リファレンスマニュアル</a> »</li>
         
     | 
| 
       948 
1081 
     | 
    
         
             
                      <li><a href="../command.html" >7.3. コマンド</a> »</li> 
         
     | 
| 
       949 
1082 
     | 
    
         
             
                  </ul>
         
     |