rroonga 5.0.4-x86-mingw32 → 5.0.5-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +2 -2
- data/example/measure-data-column-disk-usage.rb +124 -0
- data/example/measure-index-column-disk-usage.rb +81 -0
- data/example/measure-table-disk-usage.rb +100 -0
- data/ext/groonga/rb-grn-database.c +31 -0
- data/ext/groonga/rb-grn-double-array-trie.c +1 -8
- data/ext/groonga/rb-grn-logger.c +45 -0
- data/ext/groonga/rb-grn-object.c +29 -1
- data/ext/groonga/rb-grn-patricia-trie.c +1 -8
- data/ext/groonga/rb-grn-table-cursor.c +8 -3
- data/ext/groonga/rb-grn-table.c +10 -5
- data/ext/groonga/rb-grn-thread.c +160 -0
- data/ext/groonga/rb-grn-windows-event-logger.c +79 -0
- data/ext/groonga/rb-grn.h +3 -1
- data/ext/groonga/rb-groonga.c +3 -1
- data/lib/1.9/groonga.so +0 -0
- data/lib/2.0/groonga.so +0 -0
- data/lib/2.1/groonga.so +0 -0
- data/lib/2.2/groonga.so +0 -0
- data/lib/groonga/dumper.rb +6 -1
- data/rroonga-build.rb +4 -4
- data/test/groonga-test-utils.rb +5 -1
- data/test/test-database.rb +11 -0
- data/test/test-logger.rb +6 -0
- data/test/test-operator.rb +6 -6
- data/test/test-procedure.rb +15 -0
- data/test/test-table-dumper.rb +170 -1
- data/test/test-thread.rb +42 -0
- data/test/test-windows-event-logger.rb +28 -0
- data/vendor/local/bin/grndb.exe +0 -0
- data/vendor/local/bin/groonga-benchmark.exe +0 -0
- data/vendor/local/bin/groonga.exe +0 -0
- data/vendor/local/bin/libgcc_s_sjlj-1.dll +0 -0
- data/vendor/local/bin/libgroonga-0.dll +0 -0
- data/vendor/local/bin/libmecab-1.dll +0 -0
- data/vendor/local/bin/libmsgpack-4.dll +0 -0
- data/vendor/local/bin/libmsgpackc-2.dll +0 -0
- data/vendor/local/bin/libonig-5.dll +0 -0
- data/vendor/local/bin/libstdc++-6.dll +0 -0
- data/vendor/local/bin/libwinpthread-1.dll +0 -0
- data/vendor/local/bin/lz4.exe +0 -0
- data/vendor/local/bin/lz4c.exe +0 -0
- data/vendor/local/bin/lz4cat +0 -0
- data/vendor/local/bin/mecab-config +2 -2
- data/vendor/local/bin/mecab.exe +0 -0
- data/vendor/local/bin/onig-config +1 -1
- data/vendor/local/bin/zlib1.dll +0 -0
- data/vendor/local/etc/groonga/httpd/groonga-httpd.conf +2 -2
- data/vendor/local/etc/groonga/windows_event_log/provider.man +38 -0
- data/vendor/local/include/groonga/groonga.h +2 -0
- data/vendor/local/include/groonga/groonga/command.h +2 -0
- data/vendor/local/include/groonga/groonga/groonga.h +5 -0
- data/vendor/local/include/groonga/groonga/obj.h +1 -0
- data/vendor/local/include/groonga/groonga/portability.h +16 -0
- data/vendor/local/include/groonga/groonga/thread.h +42 -0
- data/vendor/local/include/groonga/groonga/windows_event_logger.h +33 -0
- data/vendor/local/lib/groonga/plugins/functions/vector.a +0 -0
- data/vendor/local/lib/groonga/plugins/functions/vector.dll +0 -0
- data/vendor/local/lib/groonga/plugins/functions/vector.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/functions/vector.la +2 -2
- data/vendor/local/lib/groonga/plugins/query_expanders/tsv.a +0 -0
- data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll +0 -0
- data/vendor/local/lib/groonga/plugins/query_expanders/tsv.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/query_expanders/tsv.la +2 -2
- data/vendor/local/lib/groonga/plugins/ruby/eval.a +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/eval.dll +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/eval.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/eval.la +2 -2
- data/vendor/local/lib/groonga/plugins/ruby/load.a +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/load.dll +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/load.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/ruby/load.la +2 -2
- data/vendor/local/lib/groonga/plugins/sharding.rb +5 -0
- data/vendor/local/lib/groonga/plugins/sharding/logical_count.rb +43 -6
- data/vendor/local/lib/groonga/plugins/sharding/logical_enumerator.rb +32 -25
- data/vendor/local/lib/groonga/plugins/sharding/logical_parameters.rb +44 -0
- data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb +217 -49
- data/vendor/local/lib/groonga/plugins/sharding/logical_select.rb +507 -45
- data/vendor/local/lib/groonga/plugins/sharding/logical_shard_list.rb +28 -0
- data/vendor/local/lib/groonga/plugins/sharding/logical_table_remove.rb +11 -6
- data/vendor/local/lib/groonga/plugins/sharding/parameters.rb +10 -0
- data/vendor/local/lib/groonga/plugins/suggest/suggest.a +0 -0
- data/vendor/local/lib/groonga/plugins/suggest/suggest.dll +0 -0
- data/vendor/local/lib/groonga/plugins/suggest/suggest.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/suggest/suggest.la +2 -2
- data/vendor/local/lib/groonga/plugins/table/table.a +0 -0
- data/vendor/local/lib/groonga/plugins/table/table.dll +0 -0
- data/vendor/local/lib/groonga/plugins/table/table.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/table/table.la +2 -2
- data/vendor/local/lib/groonga/plugins/token_filters/stop_word.a +0 -0
- data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll +0 -0
- data/vendor/local/lib/groonga/plugins/token_filters/stop_word.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/token_filters/stop_word.la +2 -2
- data/vendor/local/lib/groonga/plugins/tokenizers/mecab.a +0 -0
- data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll +0 -0
- data/vendor/local/lib/groonga/plugins/tokenizers/mecab.dll.a +0 -0
- data/vendor/local/lib/groonga/plugins/tokenizers/mecab.la +2 -2
- data/vendor/local/lib/groonga/scripts/ruby/command.rb +31 -1
- data/vendor/local/lib/groonga/scripts/ruby/context.rb +18 -2
- data/vendor/local/lib/groonga/scripts/ruby/database.rb +12 -4
- data/vendor/local/lib/groonga/scripts/ruby/expression_size_estimator.rb +31 -28
- data/vendor/local/lib/groonga/scripts/ruby/initialize/post.rb +1 -0
- data/vendor/local/lib/groonga/scripts/ruby/logger/level.rb +4 -2
- data/vendor/local/lib/groonga/scripts/ruby/query_logger.rb +9 -0
- data/vendor/local/lib/groonga/scripts/ruby/query_logger/flag.rb +39 -0
- data/vendor/local/lib/groonga/scripts/ruby/record.rb +12 -0
- data/vendor/local/lib/groonga/scripts/ruby/table.rb +35 -1
- data/vendor/local/lib/libgroonga.a +0 -0
- data/vendor/local/lib/libgroonga.dll.a +0 -0
- data/vendor/local/lib/libgroonga.la +2 -2
- data/vendor/local/lib/liblz4.dll +0 -0
- data/vendor/local/lib/liblz4.dll.1 +0 -0
- data/vendor/local/lib/liblz4.dll.1.5.0 +0 -0
- data/vendor/local/lib/libmecab.a +0 -0
- data/vendor/local/lib/libmecab.dll.a +0 -0
- data/vendor/local/lib/libmecab.la +2 -2
- data/vendor/local/lib/libmsgpack.a +0 -0
- data/vendor/local/lib/libmsgpack.dll.a +0 -0
- data/vendor/local/lib/libmsgpack.la +2 -2
- data/vendor/local/lib/libmsgpackc.a +0 -0
- data/vendor/local/lib/libmsgpackc.dll.a +0 -0
- data/vendor/local/lib/libmsgpackc.la +2 -2
- data/vendor/local/lib/libonig.a +0 -0
- data/vendor/local/lib/libonig.dll.a +0 -0
- data/vendor/local/lib/libonig.la +2 -2
- data/vendor/local/lib/libz.a +0 -0
- data/vendor/local/lib/libz.dll.a +0 -0
- data/vendor/local/lib/pkgconfig/groonga.pc +3 -3
- data/vendor/local/lib/pkgconfig/liblz4.pc +5 -5
- data/vendor/local/lib/pkgconfig/msgpack.pc +1 -1
- data/vendor/local/lib/pkgconfig/oniguruma.pc +6 -6
- data/vendor/local/lib/pkgconfig/zlib.pc +3 -3
- data/vendor/local/libexec/mecab/mecab-cost-train.exe +0 -0
- data/vendor/local/libexec/mecab/mecab-dict-gen.exe +0 -0
- data/vendor/local/libexec/mecab/mecab-dict-index.exe +0 -0
- data/vendor/local/libexec/mecab/mecab-system-eval.exe +0 -0
- data/vendor/local/libexec/mecab/mecab-test-gen.exe +0 -0
- data/vendor/local/sbin/groonga-httpd-restart +1 -1
- data/vendor/local/sbin/groonga-httpd.exe +0 -0
- data/vendor/local/share/doc/groonga/en/html/.buildinfo +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development.txt +3 -2
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/build.txt +19 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/build/unix_autotools.txt +101 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/build/unix_cmake.txt +94 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/build/windows_cmake.txt +93 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/release.txt +16 -7
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/repository.txt +7 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/contribution/development/test.txt +4 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/install/centos.txt +3 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/install/debian.txt +4 -4
- data/vendor/local/share/doc/groonga/en/html/_sources/install/fedora.txt +3 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/install/mac_os_x.txt +3 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/install/others.txt +4 -4
- data/vendor/local/share/doc/groonga/en/html/_sources/install/solaris.txt +3 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/install/ubuntu.txt +3 -4
- data/vendor/local/share/doc/groonga/en/html/_sources/install/windows.txt +9 -9
- data/vendor/local/share/doc/groonga/en/html/_sources/news.txt +319 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference.txt +1 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/api/grn_ctx.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/api/grn_db.txt +23 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/api/grn_thread.txt +122 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/cache_limit.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_copy.txt +381 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_list.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/column_rename.txt +3 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/database_unmap.txt +85 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/io_flush.txt +218 -9
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/lock_clear.txt +1 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/log_level.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_count.txt +3 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_parameters.txt +138 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_range_filter.txt +97 -10
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_select.txt +745 -23
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_shard_list.txt +107 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/logical_table_remove.txt +3 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/normalize.txt +2 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/normalizer_list.txt +1 -2
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/object_exist.txt +90 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/plugin_register.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/plugin_unregister.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/register.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/request_cancel.txt +1 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/ruby_eval.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/ruby_load.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/select.txt +240 -56
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/table_create.txt +33 -7
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/table_rename.txt +90 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/table_tokenize.txt +2 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/thread_limit.txt +110 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/tokenize.txt +2 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/tokenizer_list.txt +1 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/commands/truncate.txt +1 -3
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/executables/groonga-httpd.txt +3 -4
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/highlight_full.txt +0 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/highlight_html.txt +0 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/query.txt +2 -2
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/functions/snippet_html.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/grn_expr/query_syntax.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/grn_expr/script_syntax.txt +34 -14
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/indexing.txt +2 -2
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/query_expanders/tsv.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/regular_expression.txt +3 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/scoring_note.txt +2 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/sharding.txt +108 -0
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/tokenizers.txt +0 -21
- data/vendor/local/share/doc/groonga/en/html/_sources/reference/tuning.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/spec/search.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_sources/troubleshooting/different_results_with_the_same_keyword.txt +4 -4
- data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/introduction.txt +24 -18
- data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/match_columns.txt +19 -19
- data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/micro_blog.txt +9 -9
- data/vendor/local/share/doc/groonga/en/html/_sources/tutorial/query_expansion.txt +1 -1
- data/vendor/local/share/doc/groonga/en/html/_static/basic.css +68 -6
- data/vendor/local/share/doc/groonga/en/html/_static/doctools.js +27 -2
- data/vendor/local/share/doc/groonga/en/html/_static/down-pressed.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/down.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/file.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/jquery-1.11.1.js +10308 -0
- data/vendor/local/share/doc/groonga/en/html/_static/jquery.js +4 -9404
- data/vendor/local/share/doc/groonga/en/html/_static/minus.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/plus.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/searchtools.js +2 -2
- data/vendor/local/share/doc/groonga/en/html/_static/underscore-1.3.1.js +999 -0
- data/vendor/local/share/doc/groonga/en/html/_static/underscore.js +31 -1415
- data/vendor/local/share/doc/groonga/en/html/_static/up-pressed.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/up.png +0 -0
- data/vendor/local/share/doc/groonga/en/html/_static/websupport.js +15 -15
- data/vendor/local/share/doc/groonga/en/html/characteristic.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/client.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/community.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/contribution.html +78 -70
- data/vendor/local/share/doc/groonga/en/html/contribution/development.html +30 -27
- data/vendor/local/share/doc/groonga/en/html/contribution/development/build.html +146 -0
- data/vendor/local/share/doc/groonga/en/html/contribution/development/build/unix_autotools.html +237 -0
- data/vendor/local/share/doc/groonga/en/html/contribution/development/build/unix_cmake.html +227 -0
- data/vendor/local/share/doc/groonga/en/html/contribution/development/build/windows_cmake.html +231 -0
- data/vendor/local/share/doc/groonga/en/html/contribution/development/com.html +37 -35
- data/vendor/local/share/doc/groonga/en/html/contribution/development/cooperation.html +54 -52
- data/vendor/local/share/doc/groonga/en/html/contribution/development/query.html +80 -78
- data/vendor/local/share/doc/groonga/en/html/contribution/development/release.html +135 -122
- data/vendor/local/share/doc/groonga/en/html/contribution/development/repository.html +38 -34
- data/vendor/local/share/doc/groonga/en/html/contribution/development/test.html +58 -54
- data/vendor/local/share/doc/groonga/en/html/contribution/documentation.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/contribution/documentation/c-api.html +27 -25
- data/vendor/local/share/doc/groonga/en/html/contribution/documentation/i18n.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/contribution/documentation/introduction.html +30 -28
- data/vendor/local/share/doc/groonga/en/html/contribution/report.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/development.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/development/travis-ci.html +30 -28
- data/vendor/local/share/doc/groonga/en/html/genindex.html +48 -20
- data/vendor/local/share/doc/groonga/en/html/index.html +123 -105
- data/vendor/local/share/doc/groonga/en/html/install.html +33 -31
- data/vendor/local/share/doc/groonga/en/html/install/centos.html +32 -30
- data/vendor/local/share/doc/groonga/en/html/install/debian.html +31 -29
- data/vendor/local/share/doc/groonga/en/html/install/fedora.html +29 -27
- data/vendor/local/share/doc/groonga/en/html/install/mac_os_x.html +26 -24
- data/vendor/local/share/doc/groonga/en/html/install/others.html +92 -90
- data/vendor/local/share/doc/groonga/en/html/install/solaris.html +26 -24
- data/vendor/local/share/doc/groonga/en/html/install/ubuntu.html +29 -28
- data/vendor/local/share/doc/groonga/en/html/install/windows.html +34 -32
- data/vendor/local/share/doc/groonga/en/html/limitations.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/news.html +509 -142
- data/vendor/local/share/doc/groonga/en/html/news/0.x.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/news/1.0.x.html +20 -18
- data/vendor/local/share/doc/groonga/en/html/news/1.1.x.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/news/1.2.x.html +34 -32
- data/vendor/local/share/doc/groonga/en/html/news/1.3.x.html +29 -27
- data/vendor/local/share/doc/groonga/en/html/news/2.x.html +110 -108
- data/vendor/local/share/doc/groonga/en/html/news/3.x.html +73 -71
- data/vendor/local/share/doc/groonga/en/html/news/4.x.html +111 -109
- data/vendor/local/share/doc/groonga/en/html/news/senna.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/objects.inv +0 -0
- data/vendor/local/share/doc/groonga/en/html/reference.html +111 -94
- data/vendor/local/share/doc/groonga/en/html/reference/api.html +55 -52
- data/vendor/local/share/doc/groonga/en/html/reference/api/global_configurations.html +51 -49
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_cache.html +63 -61
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_column.html +84 -82
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_command_version.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_content_type.html +41 -39
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ctx.html +89 -87
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_db.html +88 -50
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_encoding.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_expr.html +83 -81
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_geo.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_hook.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_ii.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_index_cursor.html +45 -43
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_info.html +45 -43
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_match_escalation.html +44 -42
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_obj.html +93 -91
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_proc.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_search.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table.html +79 -77
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_table_cursor.html +69 -67
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_thread.html +296 -0
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_type.html +45 -43
- data/vendor/local/share/doc/groonga/en/html/reference/api/grn_user_data.html +38 -36
- data/vendor/local/share/doc/groonga/en/html/reference/api/overview.html +51 -49
- data/vendor/local/share/doc/groonga/en/html/reference/api/plugin.html +60 -58
- data/vendor/local/share/doc/groonga/en/html/reference/cast.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/reference/column.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/reference/columns/index.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/columns/pseudo.html +28 -26
- data/vendor/local/share/doc/groonga/en/html/reference/columns/scalar.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/columns/vector.html +72 -70
- data/vendor/local/share/doc/groonga/en/html/reference/command.html +70 -61
- data/vendor/local/share/doc/groonga/en/html/reference/command/command_version.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/command/output_format.html +59 -57
- data/vendor/local/share/doc/groonga/en/html/reference/command/request_id.html +28 -26
- data/vendor/local/share/doc/groonga/en/html/reference/command/return_code.html +100 -98
- data/vendor/local/share/doc/groonga/en/html/reference/commands/cache_limit.html +44 -42
- data/vendor/local/share/doc/groonga/en/html/reference/commands/check.html +49 -47
- data/vendor/local/share/doc/groonga/en/html/reference/commands/clearlock.html +33 -31
- data/vendor/local/share/doc/groonga/en/html/reference/commands/column_copy.html +796 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/column_create.html +62 -60
- data/vendor/local/share/doc/groonga/en/html/reference/commands/column_list.html +96 -94
- data/vendor/local/share/doc/groonga/en/html/reference/commands/column_remove.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/commands/column_rename.html +67 -64
- data/vendor/local/share/doc/groonga/en/html/reference/commands/database_unmap.html +236 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/define_selector.html +62 -60
- data/vendor/local/share/doc/groonga/en/html/reference/commands/defrag.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/commands/delete.html +59 -57
- data/vendor/local/share/doc/groonga/en/html/reference/commands/dump.html +63 -61
- data/vendor/local/share/doc/groonga/en/html/reference/commands/io_flush.html +281 -54
- data/vendor/local/share/doc/groonga/en/html/reference/commands/load.html +66 -64
- data/vendor/local/share/doc/groonga/en/html/reference/commands/lock_clear.html +53 -52
- data/vendor/local/share/doc/groonga/en/html/reference/commands/log_level.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/commands/log_put.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/commands/log_reopen.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_count.html +78 -75
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_parameters.html +283 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_range_filter.html +160 -85
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_select.html +2071 -83
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_shard_list.html +287 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/logical_table_remove.html +71 -68
- data/vendor/local/share/doc/groonga/en/html/reference/commands/normalize.html +86 -84
- data/vendor/local/share/doc/groonga/en/html/reference/commands/normalizer_list.html +52 -50
- data/vendor/local/share/doc/groonga/en/html/reference/commands/object_exist.html +227 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_register.html +57 -55
- data/vendor/local/share/doc/groonga/en/html/reference/commands/plugin_unregister.html +53 -51
- data/vendor/local/share/doc/groonga/en/html/reference/commands/quit.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/reference/commands/range_filter.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/reference/commands/register.html +56 -54
- data/vendor/local/share/doc/groonga/en/html/reference/commands/request_cancel.html +75 -74
- data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_eval.html +59 -57
- data/vendor/local/share/doc/groonga/en/html/reference/commands/ruby_load.html +59 -57
- data/vendor/local/share/doc/groonga/en/html/reference/commands/select.html +898 -647
- data/vendor/local/share/doc/groonga/en/html/reference/commands/shutdown.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/reference/commands/status.html +50 -48
- data/vendor/local/share/doc/groonga/en/html/reference/commands/suggest.html +87 -85
- data/vendor/local/share/doc/groonga/en/html/reference/commands/table_create.html +175 -152
- data/vendor/local/share/doc/groonga/en/html/reference/commands/table_list.html +55 -53
- data/vendor/local/share/doc/groonga/en/html/reference/commands/table_remove.html +46 -44
- data/vendor/local/share/doc/groonga/en/html/reference/commands/table_rename.html +327 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/table_tokenize.html +77 -75
- data/vendor/local/share/doc/groonga/en/html/reference/commands/thread_limit.html +241 -0
- data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenize.html +108 -106
- data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenizer_list.html +56 -51
- data/vendor/local/share/doc/groonga/en/html/reference/commands/truncate.html +56 -55
- data/vendor/local/share/doc/groonga/en/html/reference/executables.html +24 -22
- data/vendor/local/share/doc/groonga/en/html/reference/executables/grndb.html +41 -39
- data/vendor/local/share/doc/groonga/en/html/reference/executables/grnslap.html +28 -26
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-benchmark.html +33 -31
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-httpd.html +83 -81
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-server-http.html +29 -27
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-create-dataset.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-httpd.html +48 -46
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga-suggest-learner.html +40 -38
- data/vendor/local/share/doc/groonga/en/html/reference/executables/groonga.html +84 -82
- data/vendor/local/share/doc/groonga/en/html/reference/function.html +22 -20
- data/vendor/local/share/doc/groonga/en/html/reference/functions/between.html +47 -45
- data/vendor/local/share/doc/groonga/en/html/reference/functions/edit_distance.html +25 -23
- data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_distance.html +85 -83
- data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_circle.html +36 -34
- data/vendor/local/share/doc/groonga/en/html/reference/functions/geo_in_rectangle.html +26 -24
- data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_full.html +66 -64
- data/vendor/local/share/doc/groonga/en/html/reference/functions/highlight_html.html +54 -52
- data/vendor/local/share/doc/groonga/en/html/reference/functions/html_untag.html +35 -33
- data/vendor/local/share/doc/groonga/en/html/reference/functions/in_values.html +34 -32
- data/vendor/local/share/doc/groonga/en/html/reference/functions/now.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/functions/query.html +55 -53
- data/vendor/local/share/doc/groonga/en/html/reference/functions/rand.html +24 -22
- data/vendor/local/share/doc/groonga/en/html/reference/functions/snippet_html.html +49 -47
- data/vendor/local/share/doc/groonga/en/html/reference/functions/sub_filter.html +45 -43
- data/vendor/local/share/doc/groonga/en/html/reference/functions/vector_size.html +33 -31
- data/vendor/local/share/doc/groonga/en/html/reference/grn_expr.html +29 -27
- data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/query_syntax.html +244 -242
- data/vendor/local/share/doc/groonga/en/html/reference/grn_expr/script_syntax.html +327 -325
- data/vendor/local/share/doc/groonga/en/html/reference/indexing.html +35 -29
- data/vendor/local/share/doc/groonga/en/html/reference/log.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/reference/normalizers.html +49 -47
- data/vendor/local/share/doc/groonga/en/html/reference/operations.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/reference/operations/geolocation_search.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/output.html +36 -34
- data/vendor/local/share/doc/groonga/en/html/reference/query_expanders.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/reference/query_expanders/tsv.html +62 -60
- data/vendor/local/share/doc/groonga/en/html/reference/regular_expression.html +107 -103
- data/vendor/local/share/doc/groonga/en/html/reference/scorer.html +50 -40
- data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_at_most.html +27 -25
- data/vendor/local/share/doc/groonga/en/html/reference/scorers/scorer_tf_idf.html +41 -31
- data/vendor/local/share/doc/groonga/en/html/reference/scoring_note.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/reference/sharding.html +241 -0
- data/vendor/local/share/doc/groonga/en/html/reference/suggest.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/reference/suggest/completion.html +30 -28
- data/vendor/local/share/doc/groonga/en/html/reference/suggest/correction.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/suggest/introduction.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/suggest/suggestion.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/reference/tables.html +56 -54
- data/vendor/local/share/doc/groonga/en/html/reference/token_filters.html +41 -39
- data/vendor/local/share/doc/groonga/en/html/reference/tokenizers.html +341 -289
- data/vendor/local/share/doc/groonga/en/html/reference/tuning.html +68 -66
- data/vendor/local/share/doc/groonga/en/html/reference/types.html +43 -41
- data/vendor/local/share/doc/groonga/en/html/search.html +11 -11
- data/vendor/local/share/doc/groonga/en/html/searchindex.js +1 -1
- data/vendor/local/share/doc/groonga/en/html/server.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/server/gqtp.html +22 -20
- data/vendor/local/share/doc/groonga/en/html/server/http.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/server/http/comparison.html +37 -35
- data/vendor/local/share/doc/groonga/en/html/server/http/groonga-httpd.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/server/http/groonga.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/server/memcached.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/server/package.html +39 -37
- data/vendor/local/share/doc/groonga/en/html/spec.html +23 -21
- data/vendor/local/share/doc/groonga/en/html/spec/gqtp.html +129 -127
- data/vendor/local/share/doc/groonga/en/html/spec/search.html +22 -20
- data/vendor/local/share/doc/groonga/en/html/troubleshooting.html +19 -17
- data/vendor/local/share/doc/groonga/en/html/troubleshooting/different_results_with_the_same_keyword.html +25 -23
- data/vendor/local/share/doc/groonga/en/html/troubleshooting/mmap_cannot_allocate_memory.html +22 -20
- data/vendor/local/share/doc/groonga/en/html/tutorial.html +20 -18
- data/vendor/local/share/doc/groonga/en/html/tutorial/data.html +29 -23
- data/vendor/local/share/doc/groonga/en/html/tutorial/drilldown.html +35 -33
- data/vendor/local/share/doc/groonga/en/html/tutorial/index.html +25 -23
- data/vendor/local/share/doc/groonga/en/html/tutorial/introduction.html +46 -39
- data/vendor/local/share/doc/groonga/en/html/tutorial/lexicon.html +21 -19
- data/vendor/local/share/doc/groonga/en/html/tutorial/match_columns.html +65 -63
- data/vendor/local/share/doc/groonga/en/html/tutorial/micro_blog.html +55 -53
- data/vendor/local/share/doc/groonga/en/html/tutorial/network.html +25 -23
- data/vendor/local/share/doc/groonga/en/html/tutorial/patricia_trie.html +22 -20
- data/vendor/local/share/doc/groonga/en/html/tutorial/query_expansion.html +25 -23
- data/vendor/local/share/doc/groonga/en/html/tutorial/search.html +37 -35
- data/vendor/local/share/doc/groonga/ja/html/.buildinfo +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development.txt +3 -2
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/build.txt +19 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/build/unix_autotools.txt +101 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/build/unix_cmake.txt +94 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/build/windows_cmake.txt +93 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/release.txt +16 -7
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/repository.txt +7 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/contribution/development/test.txt +4 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/centos.txt +3 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/debian.txt +4 -4
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/fedora.txt +3 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/mac_os_x.txt +3 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/others.txt +4 -4
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/solaris.txt +3 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/ubuntu.txt +3 -4
- data/vendor/local/share/doc/groonga/ja/html/_sources/install/windows.txt +9 -9
- data/vendor/local/share/doc/groonga/ja/html/_sources/news.txt +319 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference.txt +1 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api/grn_ctx.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api/grn_db.txt +23 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/api/grn_thread.txt +122 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/cache_limit.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_copy.txt +381 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_list.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/column_rename.txt +3 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/database_unmap.txt +85 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/io_flush.txt +218 -9
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/lock_clear.txt +1 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/log_level.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_count.txt +3 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_parameters.txt +138 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_range_filter.txt +97 -10
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_select.txt +745 -23
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_shard_list.txt +107 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/logical_table_remove.txt +3 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/normalize.txt +2 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/normalizer_list.txt +1 -2
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/object_exist.txt +90 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/plugin_register.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/plugin_unregister.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/register.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/request_cancel.txt +1 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/ruby_eval.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/ruby_load.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/select.txt +240 -56
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/table_create.txt +33 -7
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/table_rename.txt +90 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/table_tokenize.txt +2 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/thread_limit.txt +110 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/tokenize.txt +2 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/tokenizer_list.txt +1 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/commands/truncate.txt +1 -3
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/executables/groonga-httpd.txt +3 -4
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/highlight_full.txt +0 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/highlight_html.txt +0 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/query.txt +2 -2
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/functions/snippet_html.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/grn_expr/query_syntax.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/grn_expr/script_syntax.txt +34 -14
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/indexing.txt +2 -2
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/query_expanders/tsv.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/regular_expression.txt +3 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/scoring_note.txt +2 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/sharding.txt +108 -0
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/tokenizers.txt +0 -21
- data/vendor/local/share/doc/groonga/ja/html/_sources/reference/tuning.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/spec/search.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_sources/troubleshooting/different_results_with_the_same_keyword.txt +4 -4
- data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/introduction.txt +24 -18
- data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/match_columns.txt +19 -19
- data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/micro_blog.txt +9 -9
- data/vendor/local/share/doc/groonga/ja/html/_sources/tutorial/query_expansion.txt +1 -1
- data/vendor/local/share/doc/groonga/ja/html/_static/basic.css +68 -6
- data/vendor/local/share/doc/groonga/ja/html/_static/doctools.js +27 -2
- data/vendor/local/share/doc/groonga/ja/html/_static/down-pressed.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/down.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/file.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/jquery-1.11.1.js +10308 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/jquery.js +4 -9404
- data/vendor/local/share/doc/groonga/ja/html/_static/minus.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/plus.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/searchtools.js +2 -2
- data/vendor/local/share/doc/groonga/ja/html/_static/underscore-1.3.1.js +999 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/underscore.js +31 -1415
- data/vendor/local/share/doc/groonga/ja/html/_static/up-pressed.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/up.png +0 -0
- data/vendor/local/share/doc/groonga/ja/html/_static/websupport.js +15 -15
- data/vendor/local/share/doc/groonga/ja/html/characteristic.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/client.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/community.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/contribution.html +77 -69
- data/vendor/local/share/doc/groonga/ja/html/contribution/development.html +30 -27
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/build.html +144 -0
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/build/unix_autotools.html +226 -0
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/build/unix_cmake.html +215 -0
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/build/windows_cmake.html +229 -0
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/com.html +36 -34
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/cooperation.html +53 -51
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/query.html +79 -77
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/release.html +134 -121
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/repository.html +29 -27
- data/vendor/local/share/doc/groonga/ja/html/contribution/development/test.html +57 -53
- data/vendor/local/share/doc/groonga/ja/html/contribution/documentation.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/c-api.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/i18n.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/contribution/documentation/introduction.html +30 -28
- data/vendor/local/share/doc/groonga/ja/html/contribution/report.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/development.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/development/travis-ci.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/genindex.html +48 -20
- data/vendor/local/share/doc/groonga/ja/html/index.html +122 -104
- data/vendor/local/share/doc/groonga/ja/html/install.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/install/centos.html +35 -33
- data/vendor/local/share/doc/groonga/ja/html/install/debian.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/install/fedora.html +30 -28
- data/vendor/local/share/doc/groonga/ja/html/install/mac_os_x.html +26 -24
- data/vendor/local/share/doc/groonga/ja/html/install/others.html +83 -81
- data/vendor/local/share/doc/groonga/ja/html/install/solaris.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/install/ubuntu.html +30 -29
- data/vendor/local/share/doc/groonga/ja/html/install/windows.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/limitations.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/news.html +460 -126
- data/vendor/local/share/doc/groonga/ja/html/news/0.x.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/news/1.0.x.html +20 -18
- data/vendor/local/share/doc/groonga/ja/html/news/1.1.x.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/news/1.2.x.html +34 -32
- data/vendor/local/share/doc/groonga/ja/html/news/1.3.x.html +29 -27
- data/vendor/local/share/doc/groonga/ja/html/news/2.x.html +102 -100
- data/vendor/local/share/doc/groonga/ja/html/news/3.x.html +66 -64
- data/vendor/local/share/doc/groonga/ja/html/news/4.x.html +91 -89
- data/vendor/local/share/doc/groonga/ja/html/news/senna.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/objects.inv +0 -0
- data/vendor/local/share/doc/groonga/ja/html/reference.html +111 -94
- data/vendor/local/share/doc/groonga/ja/html/reference/api.html +55 -52
- data/vendor/local/share/doc/groonga/ja/html/reference/api/global_configurations.html +51 -49
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_cache.html +58 -56
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_column.html +84 -82
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_command_version.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_content_type.html +41 -39
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ctx.html +85 -83
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_db.html +88 -50
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_encoding.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_expr.html +78 -76
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_geo.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_hook.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_ii.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_index_cursor.html +45 -43
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_info.html +45 -43
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_match_escalation.html +44 -42
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_obj.html +93 -91
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_proc.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_search.html +43 -41
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table.html +79 -77
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_table_cursor.html +69 -67
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_thread.html +286 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_type.html +45 -43
- data/vendor/local/share/doc/groonga/ja/html/reference/api/grn_user_data.html +38 -36
- data/vendor/local/share/doc/groonga/ja/html/reference/api/overview.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/api/plugin.html +60 -58
- data/vendor/local/share/doc/groonga/ja/html/reference/cast.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/reference/column.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/reference/columns/index.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/columns/pseudo.html +28 -26
- data/vendor/local/share/doc/groonga/ja/html/reference/columns/scalar.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/columns/vector.html +61 -59
- data/vendor/local/share/doc/groonga/ja/html/reference/command.html +70 -61
- data/vendor/local/share/doc/groonga/ja/html/reference/command/command_version.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/command/output_format.html +49 -47
- data/vendor/local/share/doc/groonga/ja/html/reference/command/request_id.html +28 -26
- data/vendor/local/share/doc/groonga/ja/html/reference/command/return_code.html +99 -97
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/cache_limit.html +41 -39
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/check.html +49 -47
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/clearlock.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_copy.html +781 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_create.html +62 -60
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_list.html +88 -86
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_remove.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/column_rename.html +66 -63
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/database_unmap.html +229 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/define_selector.html +62 -60
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/defrag.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/delete.html +58 -56
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/dump.html +63 -61
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/io_flush.html +266 -54
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/load.html +59 -57
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/lock_clear.html +53 -52
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_level.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_put.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/log_reopen.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_count.html +78 -75
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_parameters.html +276 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_range_filter.html +158 -85
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_select.html +2008 -80
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_shard_list.html +285 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/logical_table_remove.html +71 -68
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalize.html +82 -79
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/normalizer_list.html +52 -50
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/object_exist.html +220 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_register.html +53 -51
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/plugin_unregister.html +51 -49
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/quit.html +43 -41
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/range_filter.html +43 -41
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/register.html +52 -50
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/request_cancel.html +68 -67
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_eval.html +59 -57
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/ruby_load.html +59 -57
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/select.html +680 -448
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/shutdown.html +43 -41
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/status.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/suggest.html +84 -82
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_create.html +146 -126
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_list.html +55 -53
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_remove.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_rename.html +322 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/table_tokenize.html +73 -70
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/thread_limit.html +229 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenize.html +94 -91
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/tokenizer_list.html +56 -51
- data/vendor/local/share/doc/groonga/ja/html/reference/commands/truncate.html +56 -55
- data/vendor/local/share/doc/groonga/ja/html/reference/executables.html +24 -22
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/grndb.html +39 -37
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/grnslap.html +28 -26
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-benchmark.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-httpd.html +73 -72
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-server-http.html +29 -27
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-create-dataset.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-httpd.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga-suggest-learner.html +40 -38
- data/vendor/local/share/doc/groonga/ja/html/reference/executables/groonga.html +84 -82
- data/vendor/local/share/doc/groonga/ja/html/reference/function.html +22 -20
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/between.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/edit_distance.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_distance.html +69 -67
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_circle.html +36 -34
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/geo_in_rectangle.html +26 -24
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_full.html +56 -54
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/highlight_html.html +45 -43
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/html_untag.html +34 -32
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/in_values.html +35 -33
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/now.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/query.html +48 -46
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/rand.html +24 -22
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/snippet_html.html +35 -33
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/sub_filter.html +37 -35
- data/vendor/local/share/doc/groonga/ja/html/reference/functions/vector_size.html +33 -31
- data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr.html +26 -24
- data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/query_syntax.html +143 -141
- data/vendor/local/share/doc/groonga/ja/html/reference/grn_expr/script_syntax.html +236 -234
- data/vendor/local/share/doc/groonga/ja/html/reference/indexing.html +34 -28
- data/vendor/local/share/doc/groonga/ja/html/reference/log.html +43 -41
- data/vendor/local/share/doc/groonga/ja/html/reference/normalizers.html +39 -37
- data/vendor/local/share/doc/groonga/ja/html/reference/operations.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/reference/operations/geolocation_search.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/output.html +32 -30
- data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/reference/query_expanders/tsv.html +42 -39
- data/vendor/local/share/doc/groonga/ja/html/reference/regular_expression.html +91 -88
- data/vendor/local/share/doc/groonga/ja/html/reference/scorer.html +50 -40
- data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_at_most.html +27 -25
- data/vendor/local/share/doc/groonga/ja/html/reference/scorers/scorer_tf_idf.html +41 -31
- data/vendor/local/share/doc/groonga/ja/html/reference/scoring_note.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/reference/sharding.html +223 -0
- data/vendor/local/share/doc/groonga/ja/html/reference/suggest.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/reference/suggest/completion.html +27 -25
- data/vendor/local/share/doc/groonga/ja/html/reference/suggest/correction.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/suggest/introduction.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/suggest/suggestion.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/reference/tables.html +46 -44
- data/vendor/local/share/doc/groonga/ja/html/reference/token_filters.html +38 -36
- data/vendor/local/share/doc/groonga/ja/html/reference/tokenizers.html +303 -243
- data/vendor/local/share/doc/groonga/ja/html/reference/tuning.html +62 -60
- data/vendor/local/share/doc/groonga/ja/html/reference/types.html +42 -40
- data/vendor/local/share/doc/groonga/ja/html/search.html +11 -11
- data/vendor/local/share/doc/groonga/ja/html/searchindex.js +1 -1
- data/vendor/local/share/doc/groonga/ja/html/server.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/server/gqtp.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/server/http.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/server/http/comparison.html +31 -29
- data/vendor/local/share/doc/groonga/ja/html/server/http/groonga-httpd.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/server/http/groonga.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/server/memcached.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/server/package.html +38 -36
- data/vendor/local/share/doc/groonga/ja/html/spec.html +23 -21
- data/vendor/local/share/doc/groonga/ja/html/spec/gqtp.html +128 -126
- data/vendor/local/share/doc/groonga/ja/html/spec/search.html +22 -20
- data/vendor/local/share/doc/groonga/ja/html/troubleshooting.html +19 -17
- data/vendor/local/share/doc/groonga/ja/html/troubleshooting/different_results_with_the_same_keyword.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/troubleshooting/mmap_cannot_allocate_memory.html +22 -20
- data/vendor/local/share/doc/groonga/ja/html/tutorial.html +20 -18
- data/vendor/local/share/doc/groonga/ja/html/tutorial/data.html +29 -23
- data/vendor/local/share/doc/groonga/ja/html/tutorial/drilldown.html +34 -32
- data/vendor/local/share/doc/groonga/ja/html/tutorial/index.html +22 -20
- data/vendor/local/share/doc/groonga/ja/html/tutorial/introduction.html +46 -39
- data/vendor/local/share/doc/groonga/ja/html/tutorial/lexicon.html +21 -19
- data/vendor/local/share/doc/groonga/ja/html/tutorial/match_columns.html +66 -64
- data/vendor/local/share/doc/groonga/ja/html/tutorial/micro_blog.html +52 -50
- data/vendor/local/share/doc/groonga/ja/html/tutorial/network.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/tutorial/patricia_trie.html +22 -20
- data/vendor/local/share/doc/groonga/ja/html/tutorial/query_expansion.html +25 -23
- data/vendor/local/share/doc/groonga/ja/html/tutorial/search.html +35 -33
- data/vendor/local/share/license/groonga/README.md +6 -0
- data/vendor/local/share/license/mruby/AUTHORS +1 -0
- data/vendor/local/share/license/mruby/MITL +1 -1
- data/vendor/local/share/license/mruby/README.md +6 -5
- data/vendor/local/share/license/msgpack/README +219 -0
- data/vendor/local/share/man/ja/man1/groonga.1 +23512 -15126
- data/vendor/local/share/man/man1/groonga.1 +26542 -17745
- metadata +77 -3
- data/vendor/local/share/license/msgpack/AUTHORS +0 -0
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
<head>
|
|
8
8
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
9
9
|
|
|
10
|
-
<title>7.9. トークンフィルター — Groonga v5.0.
|
|
10
|
+
<title>7.9. トークンフィルター — Groonga v5.0.6-226-gd7da7e7ドキュメント</title>
|
|
11
11
|
|
|
12
12
|
<link rel="stylesheet" href="../_static/groonga.css" type="text/css" />
|
|
13
13
|
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<script type="text/javascript">
|
|
16
16
|
var DOCUMENTATION_OPTIONS = {
|
|
17
17
|
URL_ROOT: '../',
|
|
18
|
-
VERSION: '5.0.
|
|
18
|
+
VERSION: '5.0.6-226-gd7da7e7',
|
|
19
19
|
COLLAPSE_INDEX: false,
|
|
20
20
|
FILE_SUFFIX: '.html',
|
|
21
21
|
HAS_SOURCE: true
|
|
@@ -26,12 +26,12 @@
|
|
|
26
26
|
<script type="text/javascript" src="../_static/doctools.js"></script>
|
|
27
27
|
<script type="text/javascript" src="../_static/translations.js"></script>
|
|
28
28
|
<link rel="shortcut icon" href="../_static/favicon.ico"/>
|
|
29
|
-
<link rel="top" title="Groonga v5.0.
|
|
29
|
+
<link rel="top" title="Groonga v5.0.6-226-gd7da7e7ドキュメント" href="../index.html" />
|
|
30
30
|
<link rel="up" title="7. リファレンスマニュアル" href="../reference.html" />
|
|
31
31
|
<link rel="next" title="7.10. クエリー展開オブジェクト一覧" href="query_expanders.html" />
|
|
32
32
|
<link rel="prev" title="7.8. トークナイザー" href="tokenizers.html" />
|
|
33
33
|
</head>
|
|
34
|
-
<body>
|
|
34
|
+
<body role="document">
|
|
35
35
|
<div class="header">
|
|
36
36
|
<h1 class="title">
|
|
37
37
|
<a id="top-link" href="../index.html">
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
</div>
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
<div class="related">
|
|
52
|
+
<div class="related" role="navigation" aria-label="related navigation">
|
|
53
53
|
<h3>ナビゲーション</h3>
|
|
54
54
|
<ul>
|
|
55
55
|
<li class="right" style="margin-right: 10px">
|
|
@@ -61,15 +61,15 @@
|
|
|
61
61
|
<li class="right" >
|
|
62
62
|
<a href="tokenizers.html" title="7.8. トークナイザー"
|
|
63
63
|
accesskey="P">前へ</a> |</li>
|
|
64
|
-
<li><a href="../index.html">Groonga v5.0.
|
|
65
|
-
<li><a href="../reference.html" accesskey="U">7. リファレンスマニュアル</a> »</li>
|
|
64
|
+
<li class="nav-item nav-item-0"><a href="../index.html">Groonga v5.0.6-226-gd7da7e7ドキュメント</a> »</li>
|
|
65
|
+
<li class="nav-item nav-item-1"><a href="../reference.html" accesskey="U">7. リファレンスマニュアル</a> »</li>
|
|
66
66
|
</ul>
|
|
67
67
|
</div>
|
|
68
68
|
|
|
69
69
|
<div class="document">
|
|
70
70
|
<div class="documentwrapper">
|
|
71
71
|
<div class="bodywrapper">
|
|
72
|
-
<div class="body">
|
|
72
|
+
<div class="body" role="main">
|
|
73
73
|
|
|
74
74
|
<div class="section" id="token-filters">
|
|
75
75
|
<h1>7.9. トークンフィルター<a class="headerlink" href="#token-filters" title="このヘッドラインへのパーマリンク">¶</a></h1>
|
|
@@ -78,8 +78,8 @@
|
|
|
78
78
|
<p>Groongaにはトークナイズされたトークンに所定の処理を行うトークンフィルターモジュールがあります。</p>
|
|
79
79
|
<p>トークンフィルターモジュールはプラグインとして追加できます。</p>
|
|
80
80
|
<p>トークンフィルタープラグインをGroongaに追加することでトークナイズされたトークンをカスタマイズできます。</p>
|
|
81
|
-
<p>テーブルは0個以上のトークンフィルターを持てます。テーブルにトークンフィルターを付けるには <a class="reference internal" href="commands/table_create.html"><em>table_create</em></a> の <a class="reference internal" href="commands/table_create.html#table-create-token-filters"><
|
|
82
|
-
<p>以下は <
|
|
81
|
+
<p>テーブルは0個以上のトークンフィルターを持てます。テーブルにトークンフィルターを付けるには <a class="reference internal" href="commands/table_create.html"><em>table_create</em></a> の <a class="reference internal" href="commands/table_create.html#table-create-token-filters"><span>token_filters</span></a> オプションを使います。</p>
|
|
82
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> トークンフィルターモジュールを使う <code class="docutils literal"><span class="pre">table_create</span></code> の例です。</p>
|
|
83
83
|
<p>実行例:</p>
|
|
84
84
|
<div class="highlight-none"><div class="highlight"><pre>register token_filters/stop_word
|
|
85
85
|
# [[0, 1337566253.89858, 0.000355720520019531], true]
|
|
@@ -95,15 +95,15 @@ table_create Terms TABLE_PAT_KEY ShortText \
|
|
|
95
95
|
<h2>7.9.2. 利用可能なトークンフィルター<a class="headerlink" href="#available-token-filters" title="このヘッドラインへのパーマリンク">¶</a></h2>
|
|
96
96
|
<p>以下は組み込みのトークンフィルターのリストです。</p>
|
|
97
97
|
<ul class="simple">
|
|
98
|
-
<li><
|
|
99
|
-
<li><
|
|
98
|
+
<li><code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code></li>
|
|
99
|
+
<li><code class="docutils literal"><span class="pre">TokenFilterStem</span></code></li>
|
|
100
100
|
</ul>
|
|
101
101
|
<div class="section" id="tokenfilterstopword">
|
|
102
|
-
<span id="token-filter-stop-word"></span><h3>7.9.2.1. <
|
|
103
|
-
<p><
|
|
104
|
-
<p><
|
|
105
|
-
<p>ストップワードは、語彙表の <
|
|
106
|
-
<p>以下は <
|
|
102
|
+
<span id="token-filter-stop-word"></span><h3>7.9.2.1. <code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code><a class="headerlink" href="#tokenfilterstopword" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
103
|
+
<p><code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> は、文書を検索する時にトークナイズされたトークンからストップワードを除去します。</p>
|
|
104
|
+
<p><code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> は、文書を検索する時のみトークン除去するため、文書を追加した後でストップワードを指定することもできます。</p>
|
|
105
|
+
<p>ストップワードは、語彙表の <code class="docutils literal"><span class="pre">is_stop_word</span></code> カラムで指定します。</p>
|
|
106
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> トークンフィルターを使う例です。</p>
|
|
107
107
|
<p>実行例:</p>
|
|
108
108
|
<div class="highlight-none"><div class="highlight"><pre>register token_filters/stop_word
|
|
109
109
|
# [[0, 1337566253.89858, 0.000355720520019531], true]
|
|
@@ -167,13 +167,13 @@ select Memos --match_columns content --query "Hello and"
|
|
|
167
167
|
# ]
|
|
168
168
|
</pre></div>
|
|
169
169
|
</div>
|
|
170
|
-
<p><
|
|
171
|
-
<p><
|
|
170
|
+
<p><code class="docutils literal"><span class="pre">and</span></code> というトークンは <code class="docutils literal"><span class="pre">Terms</span></code> テーブルでストップワードと指定されています。</p>
|
|
171
|
+
<p><code class="docutils literal"><span class="pre">"Hello"</span></code> は文書内に <code class="docutils literal"><span class="pre">and</span></code> がありませんがマッチしています。なぜなら、 <code class="docutils literal"><span class="pre">and</span></code> はストップワードと指定されているため、クエリーから除去されているからです。</p>
|
|
172
172
|
</div>
|
|
173
173
|
<div class="section" id="tokenfilterstem">
|
|
174
|
-
<span id="token-filter-stem"></span><h3>7.9.2.2. <
|
|
175
|
-
<p><
|
|
176
|
-
<p>以下は <
|
|
174
|
+
<span id="token-filter-stem"></span><h3>7.9.2.2. <code class="docutils literal"><span class="pre">TokenFilterStem</span></code><a class="headerlink" href="#tokenfilterstem" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
175
|
+
<p><code class="docutils literal"><span class="pre">TokenFilterStem</span></code> は、トークナイズされたトークンをステミングします。</p>
|
|
176
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenFilterStem</span></code> トークンフィルターを使う例です。</p>
|
|
177
177
|
<p>実行例:</p>
|
|
178
178
|
<div class="highlight-none"><div class="highlight"><pre>register token_filters/stem
|
|
179
179
|
# [[0, 1337566253.89858, 0.000355720520019531], true]
|
|
@@ -234,7 +234,7 @@ select Memos --match_columns content --query "develops"
|
|
|
234
234
|
# ]
|
|
235
235
|
</pre></div>
|
|
236
236
|
</div>
|
|
237
|
-
<p><
|
|
237
|
+
<p><code class="docutils literal"><span class="pre">develop</span></code> も <code class="docutils literal"><span class="pre">developing</span></code> も <code class="docutils literal"><span class="pre">developed</span></code> も <code class="docutils literal"><span class="pre">develops</span></code> も、すべてステミングすると <code class="docutils literal"><span class="pre">develop</span></code> になります。そのため、 <code class="docutils literal"><span class="pre">develops</span></code> というクエリーで <code class="docutils literal"><span class="pre">develop</span></code> も <code class="docutils literal"><span class="pre">developing</span></code> も <code class="docutils literal"><span class="pre">developed</span></code> も検索できます。</p>
|
|
238
238
|
</div>
|
|
239
239
|
</div>
|
|
240
240
|
<div class="section" id="see-also">
|
|
@@ -249,15 +249,15 @@ select Memos --match_columns content --query "develops"
|
|
|
249
249
|
</div>
|
|
250
250
|
</div>
|
|
251
251
|
</div>
|
|
252
|
-
<div class="sphinxsidebar">
|
|
252
|
+
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
|
253
253
|
<div class="sphinxsidebarwrapper">
|
|
254
254
|
<h3><a href="../index.html">目次</a></h3>
|
|
255
255
|
<ul>
|
|
256
256
|
<li><a class="reference internal" href="#">7.9. トークンフィルター</a><ul>
|
|
257
257
|
<li><a class="reference internal" href="#summary">7.9.1. 概要</a></li>
|
|
258
258
|
<li><a class="reference internal" href="#available-token-filters">7.9.2. 利用可能なトークンフィルター</a><ul>
|
|
259
|
-
<li><a class="reference internal" href="#tokenfilterstopword">7.9.2.1. <
|
|
260
|
-
<li><a class="reference internal" href="#tokenfilterstem">7.9.2.2. <
|
|
259
|
+
<li><a class="reference internal" href="#tokenfilterstopword">7.9.2.1. <code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code></a></li>
|
|
260
|
+
<li><a class="reference internal" href="#tokenfilterstem">7.9.2.2. <code class="docutils literal"><span class="pre">TokenFilterStem</span></code></a></li>
|
|
261
261
|
</ul>
|
|
262
262
|
</li>
|
|
263
263
|
<li><a class="reference internal" href="#see-also">7.9.3. 参考</a></li>
|
|
@@ -271,12 +271,14 @@ select Memos --match_columns content --query "develops"
|
|
|
271
271
|
<h4>次のトピックへ</h4>
|
|
272
272
|
<p class="topless"><a href="query_expanders.html"
|
|
273
273
|
title="次の章へ">7.10. クエリー展開オブジェクト一覧</a></p>
|
|
274
|
-
<
|
|
275
|
-
|
|
276
|
-
<
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
274
|
+
<div role="note" aria-label="source link">
|
|
275
|
+
<h3>このページ</h3>
|
|
276
|
+
<ul class="this-page-menu">
|
|
277
|
+
<li><a href="../_sources/reference/token_filters.txt"
|
|
278
|
+
rel="nofollow">ソースコードを表示</a></li>
|
|
279
|
+
</ul>
|
|
280
|
+
</div>
|
|
281
|
+
<div id="searchbox" style="display: none" role="search">
|
|
280
282
|
<h3>クイック検索</h3>
|
|
281
283
|
<form class="search" action="../search.html" method="get">
|
|
282
284
|
<input type="text" name="q" />
|
|
@@ -293,7 +295,7 @@ select Memos --match_columns content --query "develops"
|
|
|
293
295
|
</div>
|
|
294
296
|
<div class="clearer"></div>
|
|
295
297
|
</div>
|
|
296
|
-
<div class="related">
|
|
298
|
+
<div class="related" role="navigation" aria-label="related navigation">
|
|
297
299
|
<h3>ナビゲーション</h3>
|
|
298
300
|
<ul>
|
|
299
301
|
<li class="right" style="margin-right: 10px">
|
|
@@ -305,11 +307,11 @@ select Memos --match_columns content --query "develops"
|
|
|
305
307
|
<li class="right" >
|
|
306
308
|
<a href="tokenizers.html" title="7.8. トークナイザー"
|
|
307
309
|
>前へ</a> |</li>
|
|
308
|
-
<li><a href="../index.html">Groonga v5.0.
|
|
309
|
-
<li><a href="../reference.html" >7. リファレンスマニュアル</a> »</li>
|
|
310
|
+
<li class="nav-item nav-item-0"><a href="../index.html">Groonga v5.0.6-226-gd7da7e7ドキュメント</a> »</li>
|
|
311
|
+
<li class="nav-item nav-item-1"><a href="../reference.html" >7. リファレンスマニュアル</a> »</li>
|
|
310
312
|
</ul>
|
|
311
313
|
</div>
|
|
312
|
-
<div class="footer">
|
|
314
|
+
<div class="footer" role="contentinfo">
|
|
313
315
|
© Copyright 2009-2015, Brazil, Inc.
|
|
314
316
|
</div>
|
|
315
317
|
</body>
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
<head>
|
|
8
8
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
9
9
|
|
|
10
|
-
<title>7.8. トークナイザー — Groonga v5.0.
|
|
10
|
+
<title>7.8. トークナイザー — Groonga v5.0.6-226-gd7da7e7ドキュメント</title>
|
|
11
11
|
|
|
12
12
|
<link rel="stylesheet" href="../_static/groonga.css" type="text/css" />
|
|
13
13
|
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<script type="text/javascript">
|
|
16
16
|
var DOCUMENTATION_OPTIONS = {
|
|
17
17
|
URL_ROOT: '../',
|
|
18
|
-
VERSION: '5.0.
|
|
18
|
+
VERSION: '5.0.6-226-gd7da7e7',
|
|
19
19
|
COLLAPSE_INDEX: false,
|
|
20
20
|
FILE_SUFFIX: '.html',
|
|
21
21
|
HAS_SOURCE: true
|
|
@@ -26,12 +26,12 @@
|
|
|
26
26
|
<script type="text/javascript" src="../_static/doctools.js"></script>
|
|
27
27
|
<script type="text/javascript" src="../_static/translations.js"></script>
|
|
28
28
|
<link rel="shortcut icon" href="../_static/favicon.ico"/>
|
|
29
|
-
<link rel="top" title="Groonga v5.0.
|
|
29
|
+
<link rel="top" title="Groonga v5.0.6-226-gd7da7e7ドキュメント" href="../index.html" />
|
|
30
30
|
<link rel="up" title="7. リファレンスマニュアル" href="../reference.html" />
|
|
31
31
|
<link rel="next" title="7.9. トークンフィルター" href="token_filters.html" />
|
|
32
32
|
<link rel="prev" title="7.7. ノーマライザー" href="normalizers.html" />
|
|
33
33
|
</head>
|
|
34
|
-
<body>
|
|
34
|
+
<body role="document">
|
|
35
35
|
<div class="header">
|
|
36
36
|
<h1 class="title">
|
|
37
37
|
<a id="top-link" href="../index.html">
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
</div>
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
<div class="related">
|
|
52
|
+
<div class="related" role="navigation" aria-label="related navigation">
|
|
53
53
|
<h3>ナビゲーション</h3>
|
|
54
54
|
<ul>
|
|
55
55
|
<li class="right" style="margin-right: 10px">
|
|
@@ -61,15 +61,15 @@
|
|
|
61
61
|
<li class="right" >
|
|
62
62
|
<a href="normalizers.html" title="7.7. ノーマライザー"
|
|
63
63
|
accesskey="P">前へ</a> |</li>
|
|
64
|
-
<li><a href="../index.html">Groonga v5.0.
|
|
65
|
-
<li><a href="../reference.html" accesskey="U">7. リファレンスマニュアル</a> »</li>
|
|
64
|
+
<li class="nav-item nav-item-0"><a href="../index.html">Groonga v5.0.6-226-gd7da7e7ドキュメント</a> »</li>
|
|
65
|
+
<li class="nav-item nav-item-1"><a href="../reference.html" accesskey="U">7. リファレンスマニュアル</a> »</li>
|
|
66
66
|
</ul>
|
|
67
67
|
</div>
|
|
68
68
|
|
|
69
69
|
<div class="document">
|
|
70
70
|
<div class="documentwrapper">
|
|
71
71
|
<div class="bodywrapper">
|
|
72
|
-
<div class="body">
|
|
72
|
+
<div class="body" role="main">
|
|
73
73
|
|
|
74
74
|
<div class="section" id="tokenizers">
|
|
75
75
|
<h1>7.8. トークナイザー<a class="headerlink" href="#tokenizers" title="このヘッドラインへのパーマリンク">¶</a></h1>
|
|
@@ -79,22 +79,22 @@
|
|
|
79
79
|
<blockquote>
|
|
80
80
|
<div><ul>
|
|
81
81
|
<li><p class="first">テキストのインデックスを構築するとき</p>
|
|
82
|
-
<div class="figure align-center">
|
|
82
|
+
<div class="figure align-center" id="id1">
|
|
83
83
|
<a class="reference internal image-reference" href="../_images/used-when-indexing.png"><img alt="../_images/used-when-indexing.png" src="../_images/used-when-indexing.png" style="width: 80%;" /></a>
|
|
84
|
-
<p class="caption">テキストのインデックスを構築するときにトークナイザーを使います。</p>
|
|
84
|
+
<p class="caption"><span class="caption-text">テキストのインデックスを構築するときにトークナイザーを使います。</span></p>
|
|
85
85
|
</div>
|
|
86
86
|
</li>
|
|
87
87
|
<li><p class="first">クエリーで検索するとき</p>
|
|
88
|
-
<div class="figure align-center">
|
|
88
|
+
<div class="figure align-center" id="id2">
|
|
89
89
|
<a class="reference internal image-reference" href="../_images/used-when-searching.png"><img alt="../_images/used-when-searching.png" src="../_images/used-when-searching.png" style="width: 80%;" /></a>
|
|
90
|
-
<p class="caption">クエリーで検索するときにトークナイザーを使います。</p>
|
|
90
|
+
<p class="caption"><span class="caption-text">クエリーで検索するときにトークナイザーを使います。</span></p>
|
|
91
91
|
</div>
|
|
92
92
|
</li>
|
|
93
93
|
</ul>
|
|
94
94
|
</div></blockquote>
|
|
95
95
|
<p>全文検索ではトークナイザーは重要なモジュールです。トークナイザーを変えることで <a class="reference external" href="http://ja.wikipedia.org/wiki/%E6%83%85%E5%A0%B1%E6%A4%9C%E7%B4%A2">適合率と再現率</a> のトレードオフを調整することができます。</p>
|
|
96
|
-
<p>一般的に <a class="reference internal" href="#token-bigram"><
|
|
97
|
-
<p><a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a> コマンドと <a class="reference internal" href="commands/table_tokenize.html"><em>table_tokenize</em></a> コマンドを使うことでトークナイザーを試すことができます。 <a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a> コマンドを使って <a class="reference internal" href="#token-bigram"><
|
|
96
|
+
<p>一般的に <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> が適切なトークナイザーです。トークナイザーについてよく知らない場合は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> を使うことをオススメします。</p>
|
|
97
|
+
<p><a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a> コマンドと <a class="reference internal" href="commands/table_tokenize.html"><em>table_tokenize</em></a> コマンドを使うことでトークナイザーを試すことができます。 <a class="reference internal" href="commands/tokenize.html"><em>tokenize</em></a> コマンドを使って <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> トークナイザーを試す例を以下に示します。</p>
|
|
98
98
|
<p>実行例:</p>
|
|
99
99
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Hello World"
|
|
100
100
|
# [
|
|
@@ -106,46 +106,57 @@
|
|
|
106
106
|
# [
|
|
107
107
|
# {
|
|
108
108
|
# "position": 0,
|
|
109
|
+
# "force_prefix": false,
|
|
109
110
|
# "value": "He"
|
|
110
111
|
# },
|
|
111
112
|
# {
|
|
112
113
|
# "position": 1,
|
|
114
|
+
# "force_prefix": false,
|
|
113
115
|
# "value": "el"
|
|
114
116
|
# },
|
|
115
117
|
# {
|
|
116
118
|
# "position": 2,
|
|
119
|
+
# "force_prefix": false,
|
|
117
120
|
# "value": "ll"
|
|
118
121
|
# },
|
|
119
122
|
# {
|
|
120
123
|
# "position": 3,
|
|
124
|
+
# "force_prefix": false,
|
|
121
125
|
# "value": "lo"
|
|
122
126
|
# },
|
|
123
127
|
# {
|
|
124
128
|
# "position": 4,
|
|
129
|
+
# "force_prefix": false,
|
|
125
130
|
# "value": "o "
|
|
126
131
|
# },
|
|
127
132
|
# {
|
|
128
133
|
# "position": 5,
|
|
134
|
+
# "force_prefix": false,
|
|
129
135
|
# "value": " W"
|
|
130
136
|
# },
|
|
131
137
|
# {
|
|
132
138
|
# "position": 6,
|
|
139
|
+
# "force_prefix": false,
|
|
133
140
|
# "value": "Wo"
|
|
134
141
|
# },
|
|
135
142
|
# {
|
|
136
143
|
# "position": 7,
|
|
144
|
+
# "force_prefix": false,
|
|
137
145
|
# "value": "or"
|
|
138
146
|
# },
|
|
139
147
|
# {
|
|
140
148
|
# "position": 8,
|
|
149
|
+
# "force_prefix": false,
|
|
141
150
|
# "value": "rl"
|
|
142
151
|
# },
|
|
143
152
|
# {
|
|
144
153
|
# "position": 9,
|
|
154
|
+
# "force_prefix": false,
|
|
145
155
|
# "value": "ld"
|
|
146
156
|
# },
|
|
147
157
|
# {
|
|
148
158
|
# "position": 10,
|
|
159
|
+
# "force_prefix": false,
|
|
149
160
|
# "value": "d"
|
|
150
161
|
# }
|
|
151
162
|
# ]
|
|
@@ -156,75 +167,75 @@
|
|
|
156
167
|
<div class="section" id="what-is-tokenize">
|
|
157
168
|
<h2>7.8.2. 「トークナイズ」とはなにか<a class="headerlink" href="#what-is-tokenize" title="このヘッドラインへのパーマリンク">¶</a></h2>
|
|
158
169
|
<p>「トークナイズ」はテキストから0個以上のトークンを抽出する処理です。「トークナイズ」する方法はいくつかあります。</p>
|
|
159
|
-
<p>例えば、バイグラムというトークナイズ方法では <
|
|
170
|
+
<p>例えば、バイグラムというトークナイズ方法では <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> は次のトークンにトークナイズされます。</p>
|
|
160
171
|
<blockquote>
|
|
161
172
|
<div><ul class="simple">
|
|
162
|
-
<li><
|
|
163
|
-
<li><
|
|
164
|
-
<li><
|
|
165
|
-
<li><
|
|
166
|
-
<li><p class="first"><
|
|
173
|
+
<li><code class="docutils literal"><span class="pre">He</span></code></li>
|
|
174
|
+
<li><code class="docutils literal"><span class="pre">el</span></code></li>
|
|
175
|
+
<li><code class="docutils literal"><span class="pre">ll</span></code></li>
|
|
176
|
+
<li><code class="docutils literal"><span class="pre">lo</span></code></li>
|
|
177
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">o_</span></code> （ <code class="docutils literal"><span class="pre">_</span></code> は空白文字という意味）</p>
|
|
167
178
|
</li>
|
|
168
|
-
<li><p class="first"><
|
|
179
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">_W</span></code> （ <code class="docutils literal"><span class="pre">_</span></code> は空白文字という意味）</p>
|
|
169
180
|
</li>
|
|
170
|
-
<li><
|
|
171
|
-
<li><
|
|
172
|
-
<li><
|
|
173
|
-
<li><
|
|
181
|
+
<li><code class="docutils literal"><span class="pre">Wo</span></code></li>
|
|
182
|
+
<li><code class="docutils literal"><span class="pre">or</span></code></li>
|
|
183
|
+
<li><code class="docutils literal"><span class="pre">rl</span></code></li>
|
|
184
|
+
<li><code class="docutils literal"><span class="pre">ld</span></code></li>
|
|
174
185
|
</ul>
|
|
175
186
|
</div></blockquote>
|
|
176
|
-
<p>上記の例では、 <
|
|
177
|
-
<p>例えば、空白区切りのトークナイズ方法では <
|
|
187
|
+
<p>上記の例では、 <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> から10個のトークンを抽出しました。</p>
|
|
188
|
+
<p>例えば、空白区切りのトークナイズ方法では <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> は次のトークンにトークナイズされます。</p>
|
|
178
189
|
<blockquote>
|
|
179
190
|
<div><ul class="simple">
|
|
180
|
-
<li><
|
|
181
|
-
<li><
|
|
191
|
+
<li><code class="docutils literal"><span class="pre">Hello</span></code></li>
|
|
192
|
+
<li><code class="docutils literal"><span class="pre">World</span></code></li>
|
|
182
193
|
</ul>
|
|
183
194
|
</div></blockquote>
|
|
184
|
-
<p>上記の例では、<
|
|
185
|
-
<p>トークンは検索時のキーとして使われます。使用したトークナイズ方法で抽出したトークンでしかインデックス化されたドキュメントを探すことはできません。例えば、トークナイズ方法としてバイグラムを使った場合は <
|
|
195
|
+
<p>上記の例では、<code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> から2つのトークンを抽出しました。</p>
|
|
196
|
+
<p>トークンは検索時のキーとして使われます。使用したトークナイズ方法で抽出したトークンでしかインデックス化されたドキュメントを探すことはできません。例えば、トークナイズ方法としてバイグラムを使った場合は <code class="docutils literal"><span class="pre">ll</span></code> で <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> を見つけることができます。しかし、空白区切りのトークナイズ方法を使ったときは <code class="docutils literal"><span class="pre">ll</span></code> で <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> を見つけることはできません。なぜなら、空白区切りのトークナイズ方法は <code class="docutils literal"><span class="pre">ll</span></code> というトークンを抽出していないからです。空白区切りのトークナイズ方法は <code class="docutils literal"><span class="pre">Hello</span></code> というトークンと <code class="docutils literal"><span class="pre">World</span></code> というトークンしか抽出していません。</p>
|
|
186
197
|
<p>一般的に、小さいトークンを生成するトークナイズ方法は再現率が高い代わりに適合率が低くなりがちです。一方、大きいトークンを生成するトークナイズ方法は適合率が高い代わりに再現率が低くなりがちです。</p>
|
|
187
|
-
<p>例えば、バイグラムというトークナイズ方法では <
|
|
188
|
-
<p>空白区切りのトークナイズ方法を使った場合は <
|
|
198
|
+
<p>例えば、バイグラムというトークナイズ方法では <code class="docutils literal"><span class="pre">or</span></code> で <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> と <code class="docutils literal"><span class="pre">A</span> <span class="pre">or</span> <span class="pre">B</span></code> を検索できます。しかし、「論理和」を検索したい人にとっては <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> は不要な結果です。これは、適合率が下がったということです。しかし、再現率は上がっています。</p>
|
|
199
|
+
<p>空白区切りのトークナイズ方法を使った場合は <code class="docutils literal"><span class="pre">or</span></code> で <code class="docutils literal"><span class="pre">A</span> <span class="pre">or</span> <span class="pre">B</span></code> だけが見つかります。なぜなら、空白区切りのトークナイズ方法では <code class="docutils literal"><span class="pre">World</span></code> は <code class="docutils literal"><span class="pre">World</span></code> という1つのトークンだけにトークナイズされるからです。これは、「論理和」を探したい人にとっては適合率が挙がっています。しかし、 <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> も <code class="docutils literal"><span class="pre">or</span></code> を含んでいるのに見つかっていないので再現率が下がっています。</p>
|
|
189
200
|
</div>
|
|
190
201
|
<div class="section" id="built-in-tokenizsers">
|
|
191
202
|
<h2>7.8.3. 組み込みトークナイザー<a class="headerlink" href="#built-in-tokenizsers" title="このヘッドラインへのパーマリンク">¶</a></h2>
|
|
192
203
|
<p>以下は組み込みのトークナイザーのリストです。</p>
|
|
193
204
|
<blockquote>
|
|
194
205
|
<div><ul class="simple">
|
|
195
|
-
<li><
|
|
196
|
-
<li><
|
|
197
|
-
<li><
|
|
198
|
-
<li><
|
|
199
|
-
<li><
|
|
200
|
-
<li><
|
|
201
|
-
<li><
|
|
202
|
-
<li><
|
|
203
|
-
<li><
|
|
204
|
-
<li><
|
|
205
|
-
<li><
|
|
206
|
-
<li><
|
|
207
|
-
<li><
|
|
208
|
-
<li><
|
|
206
|
+
<li><code class="docutils literal"><span class="pre">TokenBigram</span></code></li>
|
|
207
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></code></li>
|
|
208
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code></li>
|
|
209
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code></li>
|
|
210
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code></li>
|
|
211
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code></li>
|
|
212
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitAlpha</span></code></li>
|
|
213
|
+
<li><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitAlphaDigit</span></code></li>
|
|
214
|
+
<li><code class="docutils literal"><span class="pre">TokenUnigram</span></code></li>
|
|
215
|
+
<li><code class="docutils literal"><span class="pre">TokenTrigram</span></code></li>
|
|
216
|
+
<li><code class="docutils literal"><span class="pre">TokenDelimit</span></code></li>
|
|
217
|
+
<li><code class="docutils literal"><span class="pre">TokenDelimitNull</span></code></li>
|
|
218
|
+
<li><code class="docutils literal"><span class="pre">TokenMecab</span></code></li>
|
|
219
|
+
<li><code class="docutils literal"><span class="pre">TokenRegexp</span></code></li>
|
|
209
220
|
</ul>
|
|
210
221
|
</div></blockquote>
|
|
211
222
|
<div class="section" id="tokenbigram">
|
|
212
|
-
<span id="token-bigram"></span><h3>7.8.3.1. <
|
|
213
|
-
<p><
|
|
214
|
-
<p>バイグラムというトークナイズ方法は、隣り合った2つの文字を1つのトークンとしてテキストをトークナイズします。例えば、 <
|
|
223
|
+
<span id="token-bigram"></span><h3>7.8.3.1. <code class="docutils literal"><span class="pre">TokenBigram</span></code><a class="headerlink" href="#tokenbigram" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
224
|
+
<p><code class="docutils literal"><span class="pre">TokenBigram</span></code> はバイグラムベースのトークナイザーです。多くのケースでは、このトークナイザーを使うことをオススメします。</p>
|
|
225
|
+
<p>バイグラムというトークナイズ方法は、隣り合った2つの文字を1つのトークンとしてテキストをトークナイズします。例えば、 <code class="docutils literal"><span class="pre">Hello</span></code> は次のトークンにトークナイズします。</p>
|
|
215
226
|
<blockquote>
|
|
216
227
|
<div><ul class="simple">
|
|
217
|
-
<li><
|
|
218
|
-
<li><
|
|
219
|
-
<li><
|
|
220
|
-
<li><
|
|
228
|
+
<li><code class="docutils literal"><span class="pre">He</span></code></li>
|
|
229
|
+
<li><code class="docutils literal"><span class="pre">el</span></code></li>
|
|
230
|
+
<li><code class="docutils literal"><span class="pre">ll</span></code></li>
|
|
231
|
+
<li><code class="docutils literal"><span class="pre">lo</span></code></li>
|
|
221
232
|
</ul>
|
|
222
233
|
</div></blockquote>
|
|
223
234
|
<p>バイグラムというトークナイズ方法は再現性に優れています。なぜなら、2文字以上の文字のクエリーに対してはすべてのテキストを見つけることができるからです。</p>
|
|
224
|
-
<p>一般的に、1文字のクエリーではすべてのテキストを見つけることはできません。なぜなら、1つの文字のトークンが存在しないからです。しかし、Groongaでは1文字のクエリーでもすべてのテキストを見つけることができます。なぜなら、Groongaは前方一致検索によりクエリーで指定した文字で始まるトークンをすべて見つけることができるからです。例えば、Groongaは <
|
|
225
|
-
<p>バイグラムというトークナイズ方法は適合率はそれほど優れていません。なぜなら、単語の一部にクエリーが含まれていればすべてのテキストが見つかってしまうからです。例えば、 <
|
|
226
|
-
<p><
|
|
227
|
-
<p>ノーマライザーを使っていない場合は <
|
|
235
|
+
<p>一般的に、1文字のクエリーではすべてのテキストを見つけることはできません。なぜなら、1つの文字のトークンが存在しないからです。しかし、Groongaでは1文字のクエリーでもすべてのテキストを見つけることができます。なぜなら、Groongaは前方一致検索によりクエリーで指定した文字で始まるトークンをすべて見つけることができるからです。例えば、Groongaは <code class="docutils literal"><span class="pre">l</span></code> というクエリーから <code class="docutils literal"><span class="pre">ll</span></code> というトークンと <code class="docutils literal"><span class="pre">lo</span></code> というトークンを見つけることができます。</p>
|
|
236
|
+
<p>バイグラムというトークナイズ方法は適合率はそれほど優れていません。なぜなら、単語の一部にクエリーが含まれていればすべてのテキストが見つかってしまうからです。例えば、 <code class="docutils literal"><span class="pre">or</span></code> で <code class="docutils literal"><span class="pre">world</span></code> が見つかります。これは非ASCIIを使う言語よりASCIIのみを使う言語で顕著です。以降の説明で触れる通り、 <code class="docutils literal"><span class="pre">TokenBigram</span></code> はこの問題を解決しています。</p>
|
|
237
|
+
<p><code class="docutils literal"><span class="pre">TokenBigram</span></code> の挙動は <a class="reference internal" href="normalizers.html"><em>ノーマライザー</em></a> を使うかどうかで変わります。</p>
|
|
238
|
+
<p>ノーマライザーを使っていない場合は <code class="docutils literal"><span class="pre">TokenBigram</span></code> は純粋なバイグラム（最後のトークンをのぞいてすべてのトークンを2文字にする）のトークナイズ方法を使います。</p>
|
|
228
239
|
<p>実行例:</p>
|
|
229
240
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Hello World"
|
|
230
241
|
# [
|
|
@@ -236,60 +247,71 @@
|
|
|
236
247
|
# [
|
|
237
248
|
# {
|
|
238
249
|
# "position": 0,
|
|
250
|
+
# "force_prefix": false,
|
|
239
251
|
# "value": "He"
|
|
240
252
|
# },
|
|
241
253
|
# {
|
|
242
254
|
# "position": 1,
|
|
255
|
+
# "force_prefix": false,
|
|
243
256
|
# "value": "el"
|
|
244
257
|
# },
|
|
245
258
|
# {
|
|
246
259
|
# "position": 2,
|
|
260
|
+
# "force_prefix": false,
|
|
247
261
|
# "value": "ll"
|
|
248
262
|
# },
|
|
249
263
|
# {
|
|
250
264
|
# "position": 3,
|
|
265
|
+
# "force_prefix": false,
|
|
251
266
|
# "value": "lo"
|
|
252
267
|
# },
|
|
253
268
|
# {
|
|
254
269
|
# "position": 4,
|
|
270
|
+
# "force_prefix": false,
|
|
255
271
|
# "value": "o "
|
|
256
272
|
# },
|
|
257
273
|
# {
|
|
258
274
|
# "position": 5,
|
|
275
|
+
# "force_prefix": false,
|
|
259
276
|
# "value": " W"
|
|
260
277
|
# },
|
|
261
278
|
# {
|
|
262
279
|
# "position": 6,
|
|
280
|
+
# "force_prefix": false,
|
|
263
281
|
# "value": "Wo"
|
|
264
282
|
# },
|
|
265
283
|
# {
|
|
266
284
|
# "position": 7,
|
|
285
|
+
# "force_prefix": false,
|
|
267
286
|
# "value": "or"
|
|
268
287
|
# },
|
|
269
288
|
# {
|
|
270
289
|
# "position": 8,
|
|
290
|
+
# "force_prefix": false,
|
|
271
291
|
# "value": "rl"
|
|
272
292
|
# },
|
|
273
293
|
# {
|
|
274
294
|
# "position": 9,
|
|
295
|
+
# "force_prefix": false,
|
|
275
296
|
# "value": "ld"
|
|
276
297
|
# },
|
|
277
298
|
# {
|
|
278
299
|
# "position": 10,
|
|
300
|
+
# "force_prefix": false,
|
|
279
301
|
# "value": "d"
|
|
280
302
|
# }
|
|
281
303
|
# ]
|
|
282
304
|
# ]
|
|
283
305
|
</pre></div>
|
|
284
306
|
</div>
|
|
285
|
-
<p>ノーマライザーを使っている場合は <
|
|
307
|
+
<p>ノーマライザーを使っている場合は <code class="docutils literal"><span class="pre">TokenBigram</span></code> はASCIIの文字には空白区切りのようなトークナイズ方法を使います。非ASCII文字にはバイグラムのトークナイズ方法を使います。</p>
|
|
286
308
|
<p>もしかしたら、複数の方法が混ざったこの挙動はわかりにくいかもしれません。しかし、英語のテキスト（ASCII文字列のみ）や日本語テキスト（ASCII文字列と非ASCII文字列が混ざっている）ような多くのユースケースでは合理的な方法です。</p>
|
|
287
309
|
<p>ASCII文字しか使わない多くの言語は単語の区切りに空白文字を使っています。このようなケースに空白区切りのトークナイズ方法は適切です。</p>
|
|
288
310
|
<p>非ASCII文字を使う言語では単語の区切りに空白文字を使いません。このケースにはバイグラムなトークナイズ方法は適切です。</p>
|
|
289
311
|
<p>複数の言語が混ざっている場合は、複数の方法を組み合わせたトークナイズ方法が適切です。</p>
|
|
290
|
-
<p>ASCII文字にバイグラムなトークナイズ方法を使いたい場合は <a class="reference internal" href="#token-bigram-split-symbol-alpha"><
|
|
291
|
-
<p>例を使いながら <
|
|
292
|
-
<p><
|
|
312
|
+
<p>ASCII文字にバイグラムなトークナイズ方法を使いたい場合は <a class="reference internal" href="#token-bigram-split-symbol-alpha"><span>TokenBigramSplitSymbolAlpha</span></a> のような <code class="docutils literal"><span class="pre">TokenBigramSplitXXX</span></code> というトークナイザーを参照してください。</p>
|
|
313
|
+
<p>例を使いながら <code class="docutils literal"><span class="pre">TokenBigram</span></code> の挙動を確認しましょう。</p>
|
|
314
|
+
<p><code class="docutils literal"><span class="pre">TokenBigram</span></code> はASCII文字には1つ以上の空白文字をトークンの区切りとして使います。</p>
|
|
293
315
|
<p>実行例:</p>
|
|
294
316
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Hello World" NormalizerAuto
|
|
295
317
|
# [
|
|
@@ -301,24 +323,26 @@
|
|
|
301
323
|
# [
|
|
302
324
|
# {
|
|
303
325
|
# "position": 0,
|
|
326
|
+
# "force_prefix": false,
|
|
304
327
|
# "value": "hello"
|
|
305
328
|
# },
|
|
306
329
|
# {
|
|
307
330
|
# "position": 1,
|
|
331
|
+
# "force_prefix": false,
|
|
308
332
|
# "value": "world"
|
|
309
333
|
# }
|
|
310
334
|
# ]
|
|
311
335
|
# ]
|
|
312
336
|
</pre></div>
|
|
313
337
|
</div>
|
|
314
|
-
<p><
|
|
338
|
+
<p><code class="docutils literal"><span class="pre">TokenBigram</span></code> はASCII文字には文字の種類が変わったところをトークンの区切りとします。文字の種類は次のどれかです。</p>
|
|
315
339
|
<blockquote>
|
|
316
340
|
<div><ul class="simple">
|
|
317
341
|
<li><p class="first">アルファベット</p>
|
|
318
342
|
</li>
|
|
319
343
|
<li><p class="first">数字</p>
|
|
320
344
|
</li>
|
|
321
|
-
<li><p class="first">記号（たとえば <
|
|
345
|
+
<li><p class="first">記号（たとえば <code class="docutils literal"><span class="pre">(</span></code> 、 <code class="docutils literal"><span class="pre">)</span></code> 、 <code class="docutils literal"><span class="pre">!</span></code> など）</p>
|
|
322
346
|
</li>
|
|
323
347
|
<li><p class="first">ひらがな</p>
|
|
324
348
|
</li>
|
|
@@ -333,9 +357,9 @@
|
|
|
333
357
|
<p>次の例は2つのトークン区切りを示しています。</p>
|
|
334
358
|
<blockquote>
|
|
335
359
|
<div><ul class="simple">
|
|
336
|
-
<li><p class="first"><
|
|
360
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">100</span></code> （数字）と <code class="docutils literal"><span class="pre">cents</span></code> （アルファベット）の間のところ</p>
|
|
337
361
|
</li>
|
|
338
|
-
<li><p class="first"><
|
|
362
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">cents</span></code> （アルファベット）と <code class="docutils literal"><span class="pre">!!!</span></code> （記号）の間のところ</p>
|
|
339
363
|
</li>
|
|
340
364
|
</ul>
|
|
341
365
|
</div></blockquote>
|
|
@@ -350,21 +374,24 @@
|
|
|
350
374
|
# [
|
|
351
375
|
# {
|
|
352
376
|
# "position": 0,
|
|
377
|
+
# "force_prefix": false,
|
|
353
378
|
# "value": "100"
|
|
354
379
|
# },
|
|
355
380
|
# {
|
|
356
381
|
# "position": 1,
|
|
382
|
+
# "force_prefix": false,
|
|
357
383
|
# "value": "cents"
|
|
358
384
|
# },
|
|
359
385
|
# {
|
|
360
386
|
# "position": 2,
|
|
387
|
+
# "force_prefix": false,
|
|
361
388
|
# "value": "!!!"
|
|
362
389
|
# }
|
|
363
390
|
# ]
|
|
364
391
|
# ]
|
|
365
392
|
</pre></div>
|
|
366
393
|
</div>
|
|
367
|
-
<p>以下は <
|
|
394
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenBigram</span></code> が非ASCII文字にはトークナイズ方法としてバイグラムを使う例です。</p>
|
|
368
395
|
<p>実行例:</p>
|
|
369
396
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "日本語の勉強" NormalizerAuto
|
|
370
397
|
# [
|
|
@@ -376,26 +403,32 @@
|
|
|
376
403
|
# [
|
|
377
404
|
# {
|
|
378
405
|
# "position": 0,
|
|
406
|
+
# "force_prefix": false,
|
|
379
407
|
# "value": "日本"
|
|
380
408
|
# },
|
|
381
409
|
# {
|
|
382
410
|
# "position": 1,
|
|
411
|
+
# "force_prefix": false,
|
|
383
412
|
# "value": "本語"
|
|
384
413
|
# },
|
|
385
414
|
# {
|
|
386
415
|
# "position": 2,
|
|
416
|
+
# "force_prefix": false,
|
|
387
417
|
# "value": "語の"
|
|
388
418
|
# },
|
|
389
419
|
# {
|
|
390
420
|
# "position": 3,
|
|
421
|
+
# "force_prefix": false,
|
|
391
422
|
# "value": "の勉"
|
|
392
423
|
# },
|
|
393
424
|
# {
|
|
394
425
|
# "position": 4,
|
|
426
|
+
# "force_prefix": false,
|
|
395
427
|
# "value": "勉強"
|
|
396
428
|
# },
|
|
397
429
|
# {
|
|
398
430
|
# "position": 5,
|
|
431
|
+
# "force_prefix": false,
|
|
399
432
|
# "value": "強"
|
|
400
433
|
# }
|
|
401
434
|
# ]
|
|
@@ -404,8 +437,8 @@
|
|
|
404
437
|
</div>
|
|
405
438
|
</div>
|
|
406
439
|
<div class="section" id="tokenbigramsplitsymbol">
|
|
407
|
-
<span id="token-bigram-split-symbol"></span><h3>7.8.3.2. <
|
|
408
|
-
<p><
|
|
440
|
+
<span id="token-bigram-split-symbol"></span><h3>7.8.3.2. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></code><a class="headerlink" href="#tokenbigramsplitsymbol" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
441
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは記号の扱いです。 <code class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></code> は記号のトークナイズ方法にバイグラムを使います。</p>
|
|
409
442
|
<p>実行例:</p>
|
|
410
443
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbol "100cents!!!" NormalizerAuto
|
|
411
444
|
# [
|
|
@@ -417,22 +450,27 @@
|
|
|
417
450
|
# [
|
|
418
451
|
# {
|
|
419
452
|
# "position": 0,
|
|
453
|
+
# "force_prefix": false,
|
|
420
454
|
# "value": "100"
|
|
421
455
|
# },
|
|
422
456
|
# {
|
|
423
457
|
# "position": 1,
|
|
458
|
+
# "force_prefix": false,
|
|
424
459
|
# "value": "cents"
|
|
425
460
|
# },
|
|
426
461
|
# {
|
|
427
462
|
# "position": 2,
|
|
463
|
+
# "force_prefix": false,
|
|
428
464
|
# "value": "!!"
|
|
429
465
|
# },
|
|
430
466
|
# {
|
|
431
467
|
# "position": 3,
|
|
468
|
+
# "force_prefix": false,
|
|
432
469
|
# "value": "!!"
|
|
433
470
|
# },
|
|
434
471
|
# {
|
|
435
472
|
# "position": 4,
|
|
473
|
+
# "force_prefix": false,
|
|
436
474
|
# "value": "!"
|
|
437
475
|
# }
|
|
438
476
|
# ]
|
|
@@ -441,8 +479,8 @@
|
|
|
441
479
|
</div>
|
|
442
480
|
</div>
|
|
443
481
|
<div class="section" id="tokenbigramsplitsymbolalpha">
|
|
444
|
-
<span id="token-bigram-split-symbol-alpha"></span><h3>7.8.3.3. <
|
|
445
|
-
<p><
|
|
482
|
+
<span id="token-bigram-split-symbol-alpha"></span><h3>7.8.3.3. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code><a class="headerlink" href="#tokenbigramsplitsymbolalpha" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
483
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは記号とアルファベットの扱いです。 <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code> は記号とアルファベットのトークナイズ方法にバイグラムを使います。</p>
|
|
446
484
|
<p>実行例:</p>
|
|
447
485
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbolAlpha "100cents!!!" NormalizerAuto
|
|
448
486
|
# [
|
|
@@ -454,38 +492,47 @@
|
|
|
454
492
|
# [
|
|
455
493
|
# {
|
|
456
494
|
# "position": 0,
|
|
495
|
+
# "force_prefix": false,
|
|
457
496
|
# "value": "100"
|
|
458
497
|
# },
|
|
459
498
|
# {
|
|
460
499
|
# "position": 1,
|
|
500
|
+
# "force_prefix": false,
|
|
461
501
|
# "value": "ce"
|
|
462
502
|
# },
|
|
463
503
|
# {
|
|
464
504
|
# "position": 2,
|
|
505
|
+
# "force_prefix": false,
|
|
465
506
|
# "value": "en"
|
|
466
507
|
# },
|
|
467
508
|
# {
|
|
468
509
|
# "position": 3,
|
|
510
|
+
# "force_prefix": false,
|
|
469
511
|
# "value": "nt"
|
|
470
512
|
# },
|
|
471
513
|
# {
|
|
472
514
|
# "position": 4,
|
|
515
|
+
# "force_prefix": false,
|
|
473
516
|
# "value": "ts"
|
|
474
517
|
# },
|
|
475
518
|
# {
|
|
476
519
|
# "position": 5,
|
|
520
|
+
# "force_prefix": false,
|
|
477
521
|
# "value": "s!"
|
|
478
522
|
# },
|
|
479
523
|
# {
|
|
480
524
|
# "position": 6,
|
|
525
|
+
# "force_prefix": false,
|
|
481
526
|
# "value": "!!"
|
|
482
527
|
# },
|
|
483
528
|
# {
|
|
484
529
|
# "position": 7,
|
|
530
|
+
# "force_prefix": false,
|
|
485
531
|
# "value": "!!"
|
|
486
532
|
# },
|
|
487
533
|
# {
|
|
488
534
|
# "position": 8,
|
|
535
|
+
# "force_prefix": false,
|
|
489
536
|
# "value": "!"
|
|
490
537
|
# }
|
|
491
538
|
# ]
|
|
@@ -494,8 +541,8 @@
|
|
|
494
541
|
</div>
|
|
495
542
|
</div>
|
|
496
543
|
<div class="section" id="tokenbigramsplitsymbolalphadigit">
|
|
497
|
-
<span id="token-bigram-split-symbol-alpha-digit"></span><h3>7.8.3.4. <
|
|
498
|
-
<p><
|
|
544
|
+
<span id="token-bigram-split-symbol-alpha-digit"></span><h3>7.8.3.4. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code><a class="headerlink" href="#tokenbigramsplitsymbolalphadigit" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
545
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは記号とアルファベットと数字の扱いです。 <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code> は記号とアルファベット数字のトークナイズ方法にバイグラムを使います。つまり、すべての文字をバイグラムでトークナイズします。</p>
|
|
499
546
|
<p>実行例:</p>
|
|
500
547
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbolAlphaDigit "100cents!!!" NormalizerAuto
|
|
501
548
|
# [
|
|
@@ -507,46 +554,57 @@
|
|
|
507
554
|
# [
|
|
508
555
|
# {
|
|
509
556
|
# "position": 0,
|
|
557
|
+
# "force_prefix": false,
|
|
510
558
|
# "value": "10"
|
|
511
559
|
# },
|
|
512
560
|
# {
|
|
513
561
|
# "position": 1,
|
|
562
|
+
# "force_prefix": false,
|
|
514
563
|
# "value": "00"
|
|
515
564
|
# },
|
|
516
565
|
# {
|
|
517
566
|
# "position": 2,
|
|
567
|
+
# "force_prefix": false,
|
|
518
568
|
# "value": "0c"
|
|
519
569
|
# },
|
|
520
570
|
# {
|
|
521
571
|
# "position": 3,
|
|
572
|
+
# "force_prefix": false,
|
|
522
573
|
# "value": "ce"
|
|
523
574
|
# },
|
|
524
575
|
# {
|
|
525
576
|
# "position": 4,
|
|
577
|
+
# "force_prefix": false,
|
|
526
578
|
# "value": "en"
|
|
527
579
|
# },
|
|
528
580
|
# {
|
|
529
581
|
# "position": 5,
|
|
582
|
+
# "force_prefix": false,
|
|
530
583
|
# "value": "nt"
|
|
531
584
|
# },
|
|
532
585
|
# {
|
|
533
586
|
# "position": 6,
|
|
587
|
+
# "force_prefix": false,
|
|
534
588
|
# "value": "ts"
|
|
535
589
|
# },
|
|
536
590
|
# {
|
|
537
591
|
# "position": 7,
|
|
592
|
+
# "force_prefix": false,
|
|
538
593
|
# "value": "s!"
|
|
539
594
|
# },
|
|
540
595
|
# {
|
|
541
596
|
# "position": 8,
|
|
597
|
+
# "force_prefix": false,
|
|
542
598
|
# "value": "!!"
|
|
543
599
|
# },
|
|
544
600
|
# {
|
|
545
601
|
# "position": 9,
|
|
602
|
+
# "force_prefix": false,
|
|
546
603
|
# "value": "!!"
|
|
547
604
|
# },
|
|
548
605
|
# {
|
|
549
606
|
# "position": 10,
|
|
607
|
+
# "force_prefix": false,
|
|
550
608
|
# "value": "!"
|
|
551
609
|
# }
|
|
552
610
|
# ]
|
|
@@ -555,10 +613,10 @@
|
|
|
555
613
|
</div>
|
|
556
614
|
</div>
|
|
557
615
|
<div class="section" id="tokenbigramignoreblank">
|
|
558
|
-
<span id="token-bigram-ignore-blank"></span><h3>7.8.3.5. <
|
|
559
|
-
<p><
|
|
560
|
-
<p><
|
|
561
|
-
<p><a class="reference internal" href="#token-bigram"><
|
|
616
|
+
<span id="token-bigram-ignore-blank"></span><h3>7.8.3.5. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code><a class="headerlink" href="#tokenbigramignoreblank" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
617
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは空白文字の扱いです。 <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code> は連続する記号と非ASCII文字の間にある空白文字を無視します。</p>
|
|
618
|
+
<p><code class="docutils literal"><span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></code> というテキストを使うと違いがわかります。なぜならこのテキストは記号と非ASCII文字を両方含んでいるからです。</p>
|
|
619
|
+
<p><a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> での実行結果です。</p>
|
|
562
620
|
<p>実行例:</p>
|
|
563
621
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "日 本 語 ! ! !" NormalizerAuto
|
|
564
622
|
# [
|
|
@@ -570,33 +628,39 @@
|
|
|
570
628
|
# [
|
|
571
629
|
# {
|
|
572
630
|
# "position": 0,
|
|
631
|
+
# "force_prefix": false,
|
|
573
632
|
# "value": "日"
|
|
574
633
|
# },
|
|
575
634
|
# {
|
|
576
635
|
# "position": 1,
|
|
636
|
+
# "force_prefix": false,
|
|
577
637
|
# "value": "本"
|
|
578
638
|
# },
|
|
579
639
|
# {
|
|
580
640
|
# "position": 2,
|
|
641
|
+
# "force_prefix": false,
|
|
581
642
|
# "value": "語"
|
|
582
643
|
# },
|
|
583
644
|
# {
|
|
584
645
|
# "position": 3,
|
|
646
|
+
# "force_prefix": false,
|
|
585
647
|
# "value": "!"
|
|
586
648
|
# },
|
|
587
649
|
# {
|
|
588
650
|
# "position": 4,
|
|
651
|
+
# "force_prefix": false,
|
|
589
652
|
# "value": "!"
|
|
590
653
|
# },
|
|
591
654
|
# {
|
|
592
655
|
# "position": 5,
|
|
656
|
+
# "force_prefix": false,
|
|
593
657
|
# "value": "!"
|
|
594
658
|
# }
|
|
595
659
|
# ]
|
|
596
660
|
# ]
|
|
597
661
|
</pre></div>
|
|
598
662
|
</div>
|
|
599
|
-
<p><
|
|
663
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code> での実行結果です。</p>
|
|
600
664
|
<p>実行例:</p>
|
|
601
665
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlank "日 本 語 ! ! !" NormalizerAuto
|
|
602
666
|
# [
|
|
@@ -608,18 +672,22 @@
|
|
|
608
672
|
# [
|
|
609
673
|
# {
|
|
610
674
|
# "position": 0,
|
|
675
|
+
# "force_prefix": false,
|
|
611
676
|
# "value": "日本"
|
|
612
677
|
# },
|
|
613
678
|
# {
|
|
614
679
|
# "position": 1,
|
|
680
|
+
# "force_prefix": false,
|
|
615
681
|
# "value": "本語"
|
|
616
682
|
# },
|
|
617
683
|
# {
|
|
618
684
|
# "position": 2,
|
|
685
|
+
# "force_prefix": false,
|
|
619
686
|
# "value": "語"
|
|
620
687
|
# },
|
|
621
688
|
# {
|
|
622
689
|
# "position": 3,
|
|
690
|
+
# "force_prefix": false,
|
|
623
691
|
# "value": "!!!"
|
|
624
692
|
# }
|
|
625
693
|
# ]
|
|
@@ -628,8 +696,8 @@
|
|
|
628
696
|
</div>
|
|
629
697
|
</div>
|
|
630
698
|
<div class="section" id="tokenbigramignoreblanksplitsymbol">
|
|
631
|
-
<span id="token-bigram-ignore-blank-split-symbol"></span><h3>7.8.3.6. <
|
|
632
|
-
<p><
|
|
699
|
+
<span id="token-bigram-ignore-blank-split-symbol"></span><h3>7.8.3.6. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code><a class="headerlink" href="#tokenbigramignoreblanksplitsymbol" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
700
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは次の通りです。</p>
|
|
633
701
|
<blockquote>
|
|
634
702
|
<div><ul class="simple">
|
|
635
703
|
<li><p class="first">空白文字の扱い</p>
|
|
@@ -638,10 +706,10 @@
|
|
|
638
706
|
</li>
|
|
639
707
|
</ul>
|
|
640
708
|
</div></blockquote>
|
|
641
|
-
<p><
|
|
642
|
-
<p><
|
|
643
|
-
<p><
|
|
644
|
-
<p><a class="reference internal" href="#token-bigram"><
|
|
709
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code> は連続した記号と非ASCII文字の間の空白文字を無視します。</p>
|
|
710
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code> は記号をバイグラムでトークナイズします。</p>
|
|
711
|
+
<p><code class="docutils literal"><span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></code> というテキストを使うと違いがわかります。なぜならこのテキストは記号と非ASCII文字を両方含んでいるからです。</p>
|
|
712
|
+
<p><a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> での実行結果です。</p>
|
|
645
713
|
<p>実行例:</p>
|
|
646
714
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "日 本 語 ! ! !" NormalizerAuto
|
|
647
715
|
# [
|
|
@@ -653,33 +721,39 @@
|
|
|
653
721
|
# [
|
|
654
722
|
# {
|
|
655
723
|
# "position": 0,
|
|
724
|
+
# "force_prefix": false,
|
|
656
725
|
# "value": "日"
|
|
657
726
|
# },
|
|
658
727
|
# {
|
|
659
728
|
# "position": 1,
|
|
729
|
+
# "force_prefix": false,
|
|
660
730
|
# "value": "本"
|
|
661
731
|
# },
|
|
662
732
|
# {
|
|
663
733
|
# "position": 2,
|
|
734
|
+
# "force_prefix": false,
|
|
664
735
|
# "value": "語"
|
|
665
736
|
# },
|
|
666
737
|
# {
|
|
667
738
|
# "position": 3,
|
|
739
|
+
# "force_prefix": false,
|
|
668
740
|
# "value": "!"
|
|
669
741
|
# },
|
|
670
742
|
# {
|
|
671
743
|
# "position": 4,
|
|
744
|
+
# "force_prefix": false,
|
|
672
745
|
# "value": "!"
|
|
673
746
|
# },
|
|
674
747
|
# {
|
|
675
748
|
# "position": 5,
|
|
749
|
+
# "force_prefix": false,
|
|
676
750
|
# "value": "!"
|
|
677
751
|
# }
|
|
678
752
|
# ]
|
|
679
753
|
# ]
|
|
680
754
|
</pre></div>
|
|
681
755
|
</div>
|
|
682
|
-
<p><
|
|
756
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code> の実行結果です。</p>
|
|
683
757
|
<p>実行例:</p>
|
|
684
758
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbol "日 本 語 ! ! !" NormalizerAuto
|
|
685
759
|
# [
|
|
@@ -691,26 +765,32 @@
|
|
|
691
765
|
# [
|
|
692
766
|
# {
|
|
693
767
|
# "position": 0,
|
|
768
|
+
# "force_prefix": false,
|
|
694
769
|
# "value": "日本"
|
|
695
770
|
# },
|
|
696
771
|
# {
|
|
697
772
|
# "position": 1,
|
|
773
|
+
# "force_prefix": false,
|
|
698
774
|
# "value": "本語"
|
|
699
775
|
# },
|
|
700
776
|
# {
|
|
701
777
|
# "position": 2,
|
|
778
|
+
# "force_prefix": false,
|
|
702
779
|
# "value": "語!"
|
|
703
780
|
# },
|
|
704
781
|
# {
|
|
705
782
|
# "position": 3,
|
|
783
|
+
# "force_prefix": false,
|
|
706
784
|
# "value": "!!"
|
|
707
785
|
# },
|
|
708
786
|
# {
|
|
709
787
|
# "position": 4,
|
|
788
|
+
# "force_prefix": false,
|
|
710
789
|
# "value": "!!"
|
|
711
790
|
# },
|
|
712
791
|
# {
|
|
713
792
|
# "position": 5,
|
|
793
|
+
# "force_prefix": false,
|
|
714
794
|
# "value": "!"
|
|
715
795
|
# }
|
|
716
796
|
# ]
|
|
@@ -719,8 +799,8 @@
|
|
|
719
799
|
</div>
|
|
720
800
|
</div>
|
|
721
801
|
<div class="section" id="tokenbigramignoreblanksplitsymbolalpha">
|
|
722
|
-
<span id="token-bigram-ignore-blank-split-symbol-alpha"></span><h3>7.8.3.7. <
|
|
723
|
-
<p><
|
|
802
|
+
<span id="token-bigram-ignore-blank-split-symbol-alpha"></span><h3>7.8.3.7. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code><a class="headerlink" href="#tokenbigramignoreblanksplitsymbolalpha" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
803
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは次の通りです。</p>
|
|
724
804
|
<blockquote>
|
|
725
805
|
<div><ul class="simple">
|
|
726
806
|
<li><p class="first">空白文字の扱い</p>
|
|
@@ -729,10 +809,10 @@
|
|
|
729
809
|
</li>
|
|
730
810
|
</ul>
|
|
731
811
|
</div></blockquote>
|
|
732
|
-
<p><
|
|
733
|
-
<p><
|
|
734
|
-
<p><
|
|
735
|
-
<p><a class="reference internal" href="#token-bigram"><
|
|
812
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code> は連続した記号と非ASCII文字の間の空白文字を無視します。</p>
|
|
813
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code> は記号とアルファベットをバイグラムでトークナイズします。</p>
|
|
814
|
+
<p><code class="docutils literal"><span class="pre">Hello</span> <span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span></code> というテキストを使うと違いがわかります。なぜなら空白文字入りの記号と非ASCII文字だけでなく、アルファベットも含んでいるからです。</p>
|
|
815
|
+
<p><a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> での実行結果です。</p>
|
|
736
816
|
<p>実行例:</p>
|
|
737
817
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Hello 日 本 語 ! ! !" NormalizerAuto
|
|
738
818
|
# [
|
|
@@ -744,37 +824,44 @@
|
|
|
744
824
|
# [
|
|
745
825
|
# {
|
|
746
826
|
# "position": 0,
|
|
827
|
+
# "force_prefix": false,
|
|
747
828
|
# "value": "hello"
|
|
748
829
|
# },
|
|
749
830
|
# {
|
|
750
831
|
# "position": 1,
|
|
832
|
+
# "force_prefix": false,
|
|
751
833
|
# "value": "日"
|
|
752
834
|
# },
|
|
753
835
|
# {
|
|
754
836
|
# "position": 2,
|
|
837
|
+
# "force_prefix": false,
|
|
755
838
|
# "value": "本"
|
|
756
839
|
# },
|
|
757
840
|
# {
|
|
758
841
|
# "position": 3,
|
|
842
|
+
# "force_prefix": false,
|
|
759
843
|
# "value": "語"
|
|
760
844
|
# },
|
|
761
845
|
# {
|
|
762
846
|
# "position": 4,
|
|
847
|
+
# "force_prefix": false,
|
|
763
848
|
# "value": "!"
|
|
764
849
|
# },
|
|
765
850
|
# {
|
|
766
851
|
# "position": 5,
|
|
852
|
+
# "force_prefix": false,
|
|
767
853
|
# "value": "!"
|
|
768
854
|
# },
|
|
769
855
|
# {
|
|
770
856
|
# "position": 6,
|
|
857
|
+
# "force_prefix": false,
|
|
771
858
|
# "value": "!"
|
|
772
859
|
# }
|
|
773
860
|
# ]
|
|
774
861
|
# ]
|
|
775
862
|
</pre></div>
|
|
776
863
|
</div>
|
|
777
|
-
<p><
|
|
864
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code> の実行結果です。</p>
|
|
778
865
|
<p>実行例:</p>
|
|
779
866
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbolAlpha "Hello 日 本 語 ! ! !" NormalizerAuto
|
|
780
867
|
# [
|
|
@@ -786,46 +873,57 @@
|
|
|
786
873
|
# [
|
|
787
874
|
# {
|
|
788
875
|
# "position": 0,
|
|
876
|
+
# "force_prefix": false,
|
|
789
877
|
# "value": "he"
|
|
790
878
|
# },
|
|
791
879
|
# {
|
|
792
880
|
# "position": 1,
|
|
881
|
+
# "force_prefix": false,
|
|
793
882
|
# "value": "el"
|
|
794
883
|
# },
|
|
795
884
|
# {
|
|
796
885
|
# "position": 2,
|
|
886
|
+
# "force_prefix": false,
|
|
797
887
|
# "value": "ll"
|
|
798
888
|
# },
|
|
799
889
|
# {
|
|
800
890
|
# "position": 3,
|
|
891
|
+
# "force_prefix": false,
|
|
801
892
|
# "value": "lo"
|
|
802
893
|
# },
|
|
803
894
|
# {
|
|
804
895
|
# "position": 4,
|
|
896
|
+
# "force_prefix": false,
|
|
805
897
|
# "value": "o日"
|
|
806
898
|
# },
|
|
807
899
|
# {
|
|
808
900
|
# "position": 5,
|
|
901
|
+
# "force_prefix": false,
|
|
809
902
|
# "value": "日本"
|
|
810
903
|
# },
|
|
811
904
|
# {
|
|
812
905
|
# "position": 6,
|
|
906
|
+
# "force_prefix": false,
|
|
813
907
|
# "value": "本語"
|
|
814
908
|
# },
|
|
815
909
|
# {
|
|
816
910
|
# "position": 7,
|
|
911
|
+
# "force_prefix": false,
|
|
817
912
|
# "value": "語!"
|
|
818
913
|
# },
|
|
819
914
|
# {
|
|
820
915
|
# "position": 8,
|
|
916
|
+
# "force_prefix": false,
|
|
821
917
|
# "value": "!!"
|
|
822
918
|
# },
|
|
823
919
|
# {
|
|
824
920
|
# "position": 9,
|
|
921
|
+
# "force_prefix": false,
|
|
825
922
|
# "value": "!!"
|
|
826
923
|
# },
|
|
827
924
|
# {
|
|
828
925
|
# "position": 10,
|
|
926
|
+
# "force_prefix": false,
|
|
829
927
|
# "value": "!"
|
|
830
928
|
# }
|
|
831
929
|
# ]
|
|
@@ -834,8 +932,8 @@
|
|
|
834
932
|
</div>
|
|
835
933
|
</div>
|
|
836
934
|
<div class="section" id="tokenbigramignoreblanksplitsymbolalphadigit">
|
|
837
|
-
<span id="token-bigram-ignore-blank-split-symbol-alpha-digit"></span><h3>7.8.3.8. <
|
|
838
|
-
<p><
|
|
935
|
+
<span id="token-bigram-ignore-blank-split-symbol-alpha-digit"></span><h3>7.8.3.8. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code><a class="headerlink" href="#tokenbigramignoreblanksplitsymbolalphadigit" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
936
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> と似ています。違いは次の通りです。</p>
|
|
839
937
|
<blockquote>
|
|
840
938
|
<div><ul class="simple">
|
|
841
939
|
<li><p class="first">空白文字の扱い</p>
|
|
@@ -844,10 +942,10 @@
|
|
|
844
942
|
</li>
|
|
845
943
|
</ul>
|
|
846
944
|
</div></blockquote>
|
|
847
|
-
<p><
|
|
848
|
-
<p><
|
|
849
|
-
<p><
|
|
850
|
-
<p><a class="reference internal" href="#token-bigram"><
|
|
945
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code> は連続した記号と非ASCII文字の間の空白文字を無視します。</p>
|
|
946
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code> は記号、アルファベット、数字をバイグラムでトークナイズします。つまり、すべての文字をバイグラムでトークナイズします。</p>
|
|
947
|
+
<p><code class="docutils literal"><span class="pre">Hello</span> <span class="pre">日</span> <span class="pre">本</span> <span class="pre">語</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">!</span> <span class="pre">777</span></code> というテキストを使うと違いがわかります。なぜなら、このテキストは空白文字入りの記号と非ASCII文字だけでなく、アルファベットと数字も含んでいるからです。</p>
|
|
948
|
+
<p><a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> での実行結果です。</p>
|
|
851
949
|
<p>実行例:</p>
|
|
852
950
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram "Hello 日 本 語 ! ! ! 777" NormalizerAuto
|
|
853
951
|
# [
|
|
@@ -859,41 +957,49 @@
|
|
|
859
957
|
# [
|
|
860
958
|
# {
|
|
861
959
|
# "position": 0,
|
|
960
|
+
# "force_prefix": false,
|
|
862
961
|
# "value": "hello"
|
|
863
962
|
# },
|
|
864
963
|
# {
|
|
865
964
|
# "position": 1,
|
|
965
|
+
# "force_prefix": false,
|
|
866
966
|
# "value": "日"
|
|
867
967
|
# },
|
|
868
968
|
# {
|
|
869
969
|
# "position": 2,
|
|
970
|
+
# "force_prefix": false,
|
|
870
971
|
# "value": "本"
|
|
871
972
|
# },
|
|
872
973
|
# {
|
|
873
974
|
# "position": 3,
|
|
975
|
+
# "force_prefix": false,
|
|
874
976
|
# "value": "語"
|
|
875
977
|
# },
|
|
876
978
|
# {
|
|
877
979
|
# "position": 4,
|
|
980
|
+
# "force_prefix": false,
|
|
878
981
|
# "value": "!"
|
|
879
982
|
# },
|
|
880
983
|
# {
|
|
881
984
|
# "position": 5,
|
|
985
|
+
# "force_prefix": false,
|
|
882
986
|
# "value": "!"
|
|
883
987
|
# },
|
|
884
988
|
# {
|
|
885
989
|
# "position": 6,
|
|
990
|
+
# "force_prefix": false,
|
|
886
991
|
# "value": "!"
|
|
887
992
|
# },
|
|
888
993
|
# {
|
|
889
994
|
# "position": 7,
|
|
995
|
+
# "force_prefix": false,
|
|
890
996
|
# "value": "777"
|
|
891
997
|
# }
|
|
892
998
|
# ]
|
|
893
999
|
# ]
|
|
894
1000
|
</pre></div>
|
|
895
1001
|
</div>
|
|
896
|
-
<p><
|
|
1002
|
+
<p><code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code> の実行結果です。</p>
|
|
897
1003
|
<p>実行例:</p>
|
|
898
1004
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramIgnoreBlankSplitSymbolAlphaDigit "Hello 日 本 語 ! ! ! 777" NormalizerAuto
|
|
899
1005
|
# [
|
|
@@ -905,58 +1011,72 @@
|
|
|
905
1011
|
# [
|
|
906
1012
|
# {
|
|
907
1013
|
# "position": 0,
|
|
1014
|
+
# "force_prefix": false,
|
|
908
1015
|
# "value": "he"
|
|
909
1016
|
# },
|
|
910
1017
|
# {
|
|
911
1018
|
# "position": 1,
|
|
1019
|
+
# "force_prefix": false,
|
|
912
1020
|
# "value": "el"
|
|
913
1021
|
# },
|
|
914
1022
|
# {
|
|
915
1023
|
# "position": 2,
|
|
1024
|
+
# "force_prefix": false,
|
|
916
1025
|
# "value": "ll"
|
|
917
1026
|
# },
|
|
918
1027
|
# {
|
|
919
1028
|
# "position": 3,
|
|
1029
|
+
# "force_prefix": false,
|
|
920
1030
|
# "value": "lo"
|
|
921
1031
|
# },
|
|
922
1032
|
# {
|
|
923
1033
|
# "position": 4,
|
|
1034
|
+
# "force_prefix": false,
|
|
924
1035
|
# "value": "o日"
|
|
925
1036
|
# },
|
|
926
1037
|
# {
|
|
927
1038
|
# "position": 5,
|
|
1039
|
+
# "force_prefix": false,
|
|
928
1040
|
# "value": "日本"
|
|
929
1041
|
# },
|
|
930
1042
|
# {
|
|
931
1043
|
# "position": 6,
|
|
1044
|
+
# "force_prefix": false,
|
|
932
1045
|
# "value": "本語"
|
|
933
1046
|
# },
|
|
934
1047
|
# {
|
|
935
1048
|
# "position": 7,
|
|
1049
|
+
# "force_prefix": false,
|
|
936
1050
|
# "value": "語!"
|
|
937
1051
|
# },
|
|
938
1052
|
# {
|
|
939
1053
|
# "position": 8,
|
|
1054
|
+
# "force_prefix": false,
|
|
940
1055
|
# "value": "!!"
|
|
941
1056
|
# },
|
|
942
1057
|
# {
|
|
943
1058
|
# "position": 9,
|
|
1059
|
+
# "force_prefix": false,
|
|
944
1060
|
# "value": "!!"
|
|
945
1061
|
# },
|
|
946
1062
|
# {
|
|
947
1063
|
# "position": 10,
|
|
1064
|
+
# "force_prefix": false,
|
|
948
1065
|
# "value": "!7"
|
|
949
1066
|
# },
|
|
950
1067
|
# {
|
|
951
1068
|
# "position": 11,
|
|
1069
|
+
# "force_prefix": false,
|
|
952
1070
|
# "value": "77"
|
|
953
1071
|
# },
|
|
954
1072
|
# {
|
|
955
1073
|
# "position": 12,
|
|
1074
|
+
# "force_prefix": false,
|
|
956
1075
|
# "value": "77"
|
|
957
1076
|
# },
|
|
958
1077
|
# {
|
|
959
1078
|
# "position": 13,
|
|
1079
|
+
# "force_prefix": false,
|
|
960
1080
|
# "value": "7"
|
|
961
1081
|
# }
|
|
962
1082
|
# ]
|
|
@@ -965,8 +1085,8 @@
|
|
|
965
1085
|
</div>
|
|
966
1086
|
</div>
|
|
967
1087
|
<div class="section" id="tokenunigram">
|
|
968
|
-
<span id="token-unigram"></span><h3>7.8.3.9. <
|
|
969
|
-
<p><
|
|
1088
|
+
<span id="token-unigram"></span><h3>7.8.3.9. <code class="docutils literal"><span class="pre">TokenUnigram</span></code><a class="headerlink" href="#tokenunigram" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1089
|
+
<p><code class="docutils literal"><span class="pre">TokenUnigram</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> に似ています。違いはトークンの単位です。 <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> は各トークンが2文字ですが、 <code class="docutils literal"><span class="pre">TokenUnigram</span></code> は各トークンが1文字です。</p>
|
|
970
1090
|
<p>実行例:</p>
|
|
971
1091
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenUnigram "100cents!!!" NormalizerAuto
|
|
972
1092
|
# [
|
|
@@ -978,14 +1098,17 @@
|
|
|
978
1098
|
# [
|
|
979
1099
|
# {
|
|
980
1100
|
# "position": 0,
|
|
1101
|
+
# "force_prefix": false,
|
|
981
1102
|
# "value": "100"
|
|
982
1103
|
# },
|
|
983
1104
|
# {
|
|
984
1105
|
# "position": 1,
|
|
1106
|
+
# "force_prefix": false,
|
|
985
1107
|
# "value": "cents"
|
|
986
1108
|
# },
|
|
987
1109
|
# {
|
|
988
1110
|
# "position": 2,
|
|
1111
|
+
# "force_prefix": false,
|
|
989
1112
|
# "value": "!!!"
|
|
990
1113
|
# }
|
|
991
1114
|
# ]
|
|
@@ -994,8 +1117,8 @@
|
|
|
994
1117
|
</div>
|
|
995
1118
|
</div>
|
|
996
1119
|
<div class="section" id="tokentrigram">
|
|
997
|
-
<span id="token-trigram"></span><h3>7.8.3.10. <
|
|
998
|
-
<p><
|
|
1120
|
+
<span id="token-trigram"></span><h3>7.8.3.10. <code class="docutils literal"><span class="pre">TokenTrigram</span></code><a class="headerlink" href="#tokentrigram" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1121
|
+
<p><code class="docutils literal"><span class="pre">TokenTrigram</span></code> は <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> に似ています。違いはトークンの単位です。 <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> は各トークンが2文字ですが、 <code class="docutils literal"><span class="pre">TokenTrigram</span></code> は各トークンが3文字です。</p>
|
|
999
1122
|
<p>実行例:</p>
|
|
1000
1123
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenTrigram "10000cents!!!!!" NormalizerAuto
|
|
1001
1124
|
# [
|
|
@@ -1007,14 +1130,17 @@
|
|
|
1007
1130
|
# [
|
|
1008
1131
|
# {
|
|
1009
1132
|
# "position": 0,
|
|
1133
|
+
# "force_prefix": false,
|
|
1010
1134
|
# "value": "10000"
|
|
1011
1135
|
# },
|
|
1012
1136
|
# {
|
|
1013
1137
|
# "position": 1,
|
|
1138
|
+
# "force_prefix": false,
|
|
1014
1139
|
# "value": "cents"
|
|
1015
1140
|
# },
|
|
1016
1141
|
# {
|
|
1017
1142
|
# "position": 2,
|
|
1143
|
+
# "force_prefix": false,
|
|
1018
1144
|
# "value": "!!!!!"
|
|
1019
1145
|
# }
|
|
1020
1146
|
# ]
|
|
@@ -1023,10 +1149,10 @@
|
|
|
1023
1149
|
</div>
|
|
1024
1150
|
</div>
|
|
1025
1151
|
<div class="section" id="tokendelimit">
|
|
1026
|
-
<span id="token-delimit"></span><h3>7.8.3.11. <
|
|
1027
|
-
<p><
|
|
1028
|
-
<p><
|
|
1029
|
-
<p>以下は <
|
|
1152
|
+
<span id="token-delimit"></span><h3>7.8.3.11. <code class="docutils literal"><span class="pre">TokenDelimit</span></code><a class="headerlink" href="#tokendelimit" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1153
|
+
<p><code class="docutils literal"><span class="pre">TokenDelimit</span></code> は1つ以上の空白文字（ <code class="docutils literal"><span class="pre">U+0020</span></code> ）で分割してトークンを抽出します。たとえば、 <code class="docutils literal"><span class="pre">Hello</span> <span class="pre">World</span></code> は <code class="docutils literal"><span class="pre">Hello</span></code> と <code class="docutils literal"><span class="pre">World</span></code> にトークナイズされます。</p>
|
|
1154
|
+
<p><code class="docutils literal"><span class="pre">TokenDelimit</span></code> はタグテキストに適切です。 <code class="docutils literal"><span class="pre">groonga</span> <span class="pre">full-text-search</span> <span class="pre">http</span></code> というテキストから <code class="docutils literal"><span class="pre">groonga</span></code> 、 <code class="docutils literal"><span class="pre">full-text-search</span></code> 、 <code class="docutils literal"><span class="pre">http</span></code> を抽出します。</p>
|
|
1155
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenDelimit</span></code> の例です。</p>
|
|
1030
1156
|
<p>実行例:</p>
|
|
1031
1157
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenDelimit "Groonga full-text-search HTTP" NormalizerAuto
|
|
1032
1158
|
# [
|
|
@@ -1038,14 +1164,17 @@
|
|
|
1038
1164
|
# [
|
|
1039
1165
|
# {
|
|
1040
1166
|
# "position": 0,
|
|
1167
|
+
# "force_prefix": false,
|
|
1041
1168
|
# "value": "groonga"
|
|
1042
1169
|
# },
|
|
1043
1170
|
# {
|
|
1044
1171
|
# "position": 1,
|
|
1172
|
+
# "force_prefix": false,
|
|
1045
1173
|
# "value": "full-text-search"
|
|
1046
1174
|
# },
|
|
1047
1175
|
# {
|
|
1048
1176
|
# "position": 2,
|
|
1177
|
+
# "force_prefix": false,
|
|
1049
1178
|
# "value": "http"
|
|
1050
1179
|
# }
|
|
1051
1180
|
# ]
|
|
@@ -1054,10 +1183,10 @@
|
|
|
1054
1183
|
</div>
|
|
1055
1184
|
</div>
|
|
1056
1185
|
<div class="section" id="tokendelimitnull">
|
|
1057
|
-
<span id="token-delimit-null"></span><h3>7.8.3.12. <
|
|
1058
|
-
<p><
|
|
1059
|
-
<p><
|
|
1060
|
-
<p>以下は <
|
|
1186
|
+
<span id="token-delimit-null"></span><h3>7.8.3.12. <code class="docutils literal"><span class="pre">TokenDelimitNull</span></code><a class="headerlink" href="#tokendelimitnull" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1187
|
+
<p><code class="docutils literal"><span class="pre">TokenDelimitNull</span></code> は <a class="reference internal" href="#token-delimit"><span>TokenDelimit</span></a> に似ています。違いは区切り文字です。 <a class="reference internal" href="#token-delimit"><span>TokenDelimit</span></a> は空白文字（ <code class="docutils literal"><span class="pre">U+0020</span></code> ）を使いますが、 <code class="docutils literal"><span class="pre">TokenDelimitNull</span></code> はNUL文字（ <code class="docutils literal"><span class="pre">U+0000</span></code> ）を使います。</p>
|
|
1188
|
+
<p><code class="docutils literal"><span class="pre">TokenDelimitNull</span></code> もタグテキストに適切です。</p>
|
|
1189
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenDelimitNull</span></code> の例です。</p>
|
|
1061
1190
|
<p>実行例:</p>
|
|
1062
1191
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenDelimitNull "Groonga\u0000full-text-search\u0000HTTP" NormalizerAuto
|
|
1063
1192
|
# [
|
|
@@ -1069,6 +1198,7 @@
|
|
|
1069
1198
|
# [
|
|
1070
1199
|
# {
|
|
1071
1200
|
# "position": 0,
|
|
1201
|
+
# "force_prefix": false,
|
|
1072
1202
|
# "value": "groongau0000full-text-searchu0000http"
|
|
1073
1203
|
# }
|
|
1074
1204
|
# ]
|
|
@@ -1077,12 +1207,12 @@
|
|
|
1077
1207
|
</div>
|
|
1078
1208
|
</div>
|
|
1079
1209
|
<div class="section" id="tokenmecab">
|
|
1080
|
-
<span id="token-mecab"></span><h3>7.8.3.13. <
|
|
1081
|
-
<p><
|
|
1210
|
+
<span id="token-mecab"></span><h3>7.8.3.13. <code class="docutils literal"><span class="pre">TokenMecab</span></code><a class="headerlink" href="#tokenmecab" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1211
|
+
<p><code class="docutils literal"><span class="pre">TokenMecab</span></code> は <a class="reference external" href="http://mecab.sourceforge.net/">MeCab</a> 形態素解析器をベースにしたトークナイザーです。</p>
|
|
1082
1212
|
<p>MeCabは日本語に依存していません。その言語用の辞書を用意すれば日本語以外でもMeCabを使えます。日本語用の辞書には <a class="reference external" href="http://sourceforge.jp/projects/naist-jdic/">NAIST Japanese Dictionary</a> を使えます。</p>
|
|
1083
|
-
<p><
|
|
1084
|
-
<p>新語をサポートしたい場合は、MeCabの辞書を更新し続ける必要があります。これはメンテナンスコストがかかります。（ <a class="reference internal" href="#token-bigram"><
|
|
1085
|
-
<p>以下は <
|
|
1213
|
+
<p><code class="docutils literal"><span class="pre">TokenMecab</span></code> は再現率より適合率に優れています。 <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> では <code class="docutils literal"><span class="pre">京都</span></code> というクエリーで <code class="docutils literal"><span class="pre">東京都</span></code> も <code class="docutils literal"><span class="pre">京都</span></code> も見つかりますが、この場合は <code class="docutils literal"><span class="pre">東京都</span></code> は期待した結果ではありません。 <code class="docutils literal"><span class="pre">TokenMecab</span></code> を使うと <code class="docutils literal"><span class="pre">京都</span></code> というクエリーで <code class="docutils literal"><span class="pre">京都</span></code> だけを見つけられます。</p>
|
|
1214
|
+
<p>新語をサポートしたい場合は、MeCabの辞書を更新し続ける必要があります。これはメンテナンスコストがかかります。（ <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> には辞書のメンテナンスコストはありません。なぜなら、 <a class="reference internal" href="#token-bigram"><span>TokenBigram</span></a> は辞書を使っていないからです。）新語への対応に <a class="reference external" href="https://github.com/neologd/mecab-ipadic-neologd">mecab-ipadic-NEologd : Neologism dictionary for MeCab</a> が役に立つかもしれません。</p>
|
|
1215
|
+
<p>以下は <code class="docutils literal"><span class="pre">TokenMeCab</span></code> の例です。 <code class="docutils literal"><span class="pre">東京都</span></code> は <code class="docutils literal"><span class="pre">東京</span></code> と <code class="docutils literal"><span class="pre">都</span></code> にトークナイズされています。 <code class="docutils literal"><span class="pre">京都</span></code> というトークンはありません。</p>
|
|
1086
1216
|
<p>実行例:</p>
|
|
1087
1217
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenMecab "東京都"
|
|
1088
1218
|
# [
|
|
@@ -1094,10 +1224,12 @@
|
|
|
1094
1224
|
# [
|
|
1095
1225
|
# {
|
|
1096
1226
|
# "position": 0,
|
|
1227
|
+
# "force_prefix": false,
|
|
1097
1228
|
# "value": "東京"
|
|
1098
1229
|
# },
|
|
1099
1230
|
# {
|
|
1100
1231
|
# "position": 1,
|
|
1232
|
+
# "force_prefix": false,
|
|
1101
1233
|
# "value": "都"
|
|
1102
1234
|
# }
|
|
1103
1235
|
# ]
|
|
@@ -1106,7 +1238,7 @@
|
|
|
1106
1238
|
</div>
|
|
1107
1239
|
</div>
|
|
1108
1240
|
<div class="section" id="tokenregexp">
|
|
1109
|
-
<span id="token-regexp"></span><h3>7.8.3.14. <
|
|
1241
|
+
<span id="token-regexp"></span><h3>7.8.3.14. <code class="docutils literal"><span class="pre">TokenRegexp</span></code><a class="headerlink" href="#tokenregexp" title="このヘッドラインへのパーマリンク">¶</a></h3>
|
|
1110
1242
|
<div class="versionadded">
|
|
1111
1243
|
<p><span class="versionmodified">バージョン 5.0.1 で追加.</span></p>
|
|
1112
1244
|
</div>
|
|
@@ -1118,20 +1250,20 @@
|
|
|
1118
1250
|
<p class="first admonition-title">ご用心</p>
|
|
1119
1251
|
<p class="last">このトークナイザーはUTF-8でしか使えません。EUC-JPやShift_JISなどと一緒には使えません。</p>
|
|
1120
1252
|
</div>
|
|
1121
|
-
<p><
|
|
1253
|
+
<p><code class="docutils literal"><span class="pre">TokenRegexp</span></code> はインデックスを使った正規表現検索をサポートするトークナイザーです。</p>
|
|
1122
1254
|
<p>一般的に、正規表現検索は逐次検索で実行します。しかし、次のケースはインデックスを使って検索できます。</p>
|
|
1123
1255
|
<blockquote>
|
|
1124
1256
|
<div><ul class="simple">
|
|
1125
|
-
<li><p class="first"><
|
|
1257
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">hello</span></code> のようにリテラルしかないケース</p>
|
|
1126
1258
|
</li>
|
|
1127
|
-
<li><p class="first"><
|
|
1259
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">\A/home/alice</span></code> のようにテキストの最初でのマッチとリテラルのみのケース</p>
|
|
1128
1260
|
</li>
|
|
1129
|
-
<li><p class="first"><
|
|
1261
|
+
<li><p class="first"><code class="docutils literal"><span class="pre">\.txt\z</span></code> のようにテキストの最後でのマッチとリテラルのみのケース</p>
|
|
1130
1262
|
</li>
|
|
1131
1263
|
</ul>
|
|
1132
1264
|
</div></blockquote>
|
|
1133
1265
|
<p>多くのケースでは、逐次検索よりもインデックスを使った検索の方が高速です。</p>
|
|
1134
|
-
<p><
|
|
1266
|
+
<p><code class="docutils literal"><span class="pre">TokenRegexp</span></code> はベースはバイグラムを使います。 <code class="docutils literal"><span class="pre">TokenRegexp</span></code> は、インデックス時に、テキストの先頭にテキストの先頭であるというマーク（ <code class="docutils literal"><span class="pre">U+FFEF</span></code> ）を入れ、テキストの最後にテキストの最後であるというマーク（ <code class="docutils literal"><span class="pre">U+FFF0</span></code> ）を入れます。</p>
|
|
1135
1267
|
<p>実行例:</p>
|
|
1136
1268
|
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp "/home/alice/test.txt" NormalizerAuto --mode ADD
|
|
1137
1269
|
# [
|
|
@@ -1143,186 +1275,112 @@
|
|
|
1143
1275
|
# [
|
|
1144
1276
|
# {
|
|
1145
1277
|
# "position": 0,
|
|
1278
|
+
# "force_prefix": false,
|
|
1146
1279
|
# "value": ""
|
|
1147
1280
|
# },
|
|
1148
1281
|
# {
|
|
1149
1282
|
# "position": 1,
|
|
1283
|
+
# "force_prefix": false,
|
|
1150
1284
|
# "value": "/h"
|
|
1151
1285
|
# },
|
|
1152
1286
|
# {
|
|
1153
1287
|
# "position": 2,
|
|
1288
|
+
# "force_prefix": false,
|
|
1154
1289
|
# "value": "ho"
|
|
1155
1290
|
# },
|
|
1156
1291
|
# {
|
|
1157
1292
|
# "position": 3,
|
|
1293
|
+
# "force_prefix": false,
|
|
1158
1294
|
# "value": "om"
|
|
1159
1295
|
# },
|
|
1160
1296
|
# {
|
|
1161
1297
|
# "position": 4,
|
|
1298
|
+
# "force_prefix": false,
|
|
1162
1299
|
# "value": "me"
|
|
1163
1300
|
# },
|
|
1164
1301
|
# {
|
|
1165
1302
|
# "position": 5,
|
|
1303
|
+
# "force_prefix": false,
|
|
1166
1304
|
# "value": "e/"
|
|
1167
1305
|
# },
|
|
1168
1306
|
# {
|
|
1169
1307
|
# "position": 6,
|
|
1308
|
+
# "force_prefix": false,
|
|
1170
1309
|
# "value": "/a"
|
|
1171
1310
|
# },
|
|
1172
1311
|
# {
|
|
1173
1312
|
# "position": 7,
|
|
1313
|
+
# "force_prefix": false,
|
|
1174
1314
|
# "value": "al"
|
|
1175
1315
|
# },
|
|
1176
1316
|
# {
|
|
1177
1317
|
# "position": 8,
|
|
1318
|
+
# "force_prefix": false,
|
|
1178
1319
|
# "value": "li"
|
|
1179
1320
|
# },
|
|
1180
1321
|
# {
|
|
1181
1322
|
# "position": 9,
|
|
1323
|
+
# "force_prefix": false,
|
|
1182
1324
|
# "value": "ic"
|
|
1183
1325
|
# },
|
|
1184
1326
|
# {
|
|
1185
1327
|
# "position": 10,
|
|
1328
|
+
# "force_prefix": false,
|
|
1186
1329
|
# "value": "ce"
|
|
1187
1330
|
# },
|
|
1188
1331
|
# {
|
|
1189
1332
|
# "position": 11,
|
|
1333
|
+
# "force_prefix": false,
|
|
1190
1334
|
# "value": "e/"
|
|
1191
1335
|
# },
|
|
1192
1336
|
# {
|
|
1193
1337
|
# "position": 12,
|
|
1338
|
+
# "force_prefix": false,
|
|
1194
1339
|
# "value": "/t"
|
|
1195
1340
|
# },
|
|
1196
1341
|
# {
|
|
1197
1342
|
# "position": 13,
|
|
1343
|
+
# "force_prefix": false,
|
|
1198
1344
|
# "value": "te"
|
|
1199
1345
|
# },
|
|
1200
1346
|
# {
|
|
1201
1347
|
# "position": 14,
|
|
1348
|
+
# "force_prefix": false,
|
|
1202
1349
|
# "value": "es"
|
|
1203
1350
|
# },
|
|
1204
1351
|
# {
|
|
1205
1352
|
# "position": 15,
|
|
1353
|
+
# "force_prefix": false,
|
|
1206
1354
|
# "value": "st"
|
|
1207
1355
|
# },
|
|
1208
1356
|
# {
|
|
1209
1357
|
# "position": 16,
|
|
1358
|
+
# "force_prefix": false,
|
|
1210
1359
|
# "value": "t."
|
|
1211
1360
|
# },
|
|
1212
1361
|
# {
|
|
1213
1362
|
# "position": 17,
|
|
1363
|
+
# "force_prefix": false,
|
|
1214
1364
|
# "value": ".t"
|
|
1215
1365
|
# },
|
|
1216
1366
|
# {
|
|
1217
1367
|
# "position": 18,
|
|
1368
|
+
# "force_prefix": false,
|
|
1218
1369
|
# "value": "tx"
|
|
1219
1370
|
# },
|
|
1220
1371
|
# {
|
|
1221
1372
|
# "position": 19,
|
|
1373
|
+
# "force_prefix": false,
|
|
1222
1374
|
# "value": "xt"
|
|
1223
1375
|
# },
|
|
1224
1376
|
# {
|
|
1225
1377
|
# "position": 20,
|
|
1378
|
+
# "force_prefix": false,
|
|
1226
1379
|
# "value": "t"
|
|
1227
1380
|
# },
|
|
1228
1381
|
# {
|
|
1229
1382
|
# "position": 21,
|
|
1230
|
-
# "
|
|
1231
|
-
# }
|
|
1232
|
-
# ]
|
|
1233
|
-
# ]
|
|
1234
|
-
</pre></div>
|
|
1235
|
-
</div>
|
|
1236
|
-
<p><tt class="docutils literal"><span class="pre">\A</span></tt> で検索したとき、テキストの先頭であるというマークを使います。クエリーをトークナイズするために <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> を使うときは、 <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> は最初のトークンとしてテキストの先頭であるというマーク（ <tt class="docutils literal"><span class="pre">U+FFEF</span></tt> ）を追加します。テキストの先頭であるというマークは先頭にしか存在しないはずなので、テキストの先頭であるという検索結果を得ることができます。</p>
|
|
1237
|
-
<p>実行例:</p>
|
|
1238
|
-
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp "\\A/home/alice/" NormalizerAuto --mode GET
|
|
1239
|
-
# [
|
|
1240
|
-
# [
|
|
1241
|
-
# 0,
|
|
1242
|
-
# 1337566253.89858,
|
|
1243
|
-
# 0.000355720520019531
|
|
1244
|
-
# ],
|
|
1245
|
-
# [
|
|
1246
|
-
# {
|
|
1247
|
-
# "position": 0,
|
|
1248
|
-
# "value": ""
|
|
1249
|
-
# },
|
|
1250
|
-
# {
|
|
1251
|
-
# "position": 1,
|
|
1252
|
-
# "value": "/h"
|
|
1253
|
-
# },
|
|
1254
|
-
# {
|
|
1255
|
-
# "position": 2,
|
|
1256
|
-
# "value": "ho"
|
|
1257
|
-
# },
|
|
1258
|
-
# {
|
|
1259
|
-
# "position": 3,
|
|
1260
|
-
# "value": "om"
|
|
1261
|
-
# },
|
|
1262
|
-
# {
|
|
1263
|
-
# "position": 4,
|
|
1264
|
-
# "value": "me"
|
|
1265
|
-
# },
|
|
1266
|
-
# {
|
|
1267
|
-
# "position": 5,
|
|
1268
|
-
# "value": "e/"
|
|
1269
|
-
# },
|
|
1270
|
-
# {
|
|
1271
|
-
# "position": 6,
|
|
1272
|
-
# "value": "/a"
|
|
1273
|
-
# },
|
|
1274
|
-
# {
|
|
1275
|
-
# "position": 7,
|
|
1276
|
-
# "value": "al"
|
|
1277
|
-
# },
|
|
1278
|
-
# {
|
|
1279
|
-
# "position": 8,
|
|
1280
|
-
# "value": "li"
|
|
1281
|
-
# },
|
|
1282
|
-
# {
|
|
1283
|
-
# "position": 9,
|
|
1284
|
-
# "value": "ic"
|
|
1285
|
-
# },
|
|
1286
|
-
# {
|
|
1287
|
-
# "position": 10,
|
|
1288
|
-
# "value": "ce"
|
|
1289
|
-
# },
|
|
1290
|
-
# {
|
|
1291
|
-
# "position": 11,
|
|
1292
|
-
# "value": "e/"
|
|
1293
|
-
# }
|
|
1294
|
-
# ]
|
|
1295
|
-
# ]
|
|
1296
|
-
</pre></div>
|
|
1297
|
-
</div>
|
|
1298
|
-
<p><tt class="docutils literal"><span class="pre">\z</span></tt> で検索したとき、テキストの最後であるというマークを使います。クエリーをトークナイズするために <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> を使うときは、 <tt class="docutils literal"><span class="pre">TokenRegexp</span></tt> は最後のトークンとしてテキストの最後であるというマーク（ <tt class="docutils literal"><span class="pre">U+FFF0</span></tt> ）を追加します。テキストの最後であるというマークは最後にしか存在しないはずなので、テキストの最後であるという検索結果を得ることができます。</p>
|
|
1299
|
-
<p>実行例:</p>
|
|
1300
|
-
<div class="highlight-none"><div class="highlight"><pre>tokenize TokenRegexp "\\.txt\\z" NormalizerAuto --mode GET
|
|
1301
|
-
# [
|
|
1302
|
-
# [
|
|
1303
|
-
# 0,
|
|
1304
|
-
# 1337566253.89858,
|
|
1305
|
-
# 0.000355720520019531
|
|
1306
|
-
# ],
|
|
1307
|
-
# [
|
|
1308
|
-
# {
|
|
1309
|
-
# "position": 0,
|
|
1310
|
-
# "value": "\\."
|
|
1311
|
-
# },
|
|
1312
|
-
# {
|
|
1313
|
-
# "position": 1,
|
|
1314
|
-
# "value": ".t"
|
|
1315
|
-
# },
|
|
1316
|
-
# {
|
|
1317
|
-
# "position": 2,
|
|
1318
|
-
# "value": "tx"
|
|
1319
|
-
# },
|
|
1320
|
-
# {
|
|
1321
|
-
# "position": 3,
|
|
1322
|
-
# "value": "xt"
|
|
1323
|
-
# },
|
|
1324
|
-
# {
|
|
1325
|
-
# "position": 5,
|
|
1383
|
+
# "force_prefix": false,
|
|
1326
1384
|
# "value": ""
|
|
1327
1385
|
# }
|
|
1328
1386
|
# ]
|
|
@@ -1337,7 +1395,7 @@
|
|
|
1337
1395
|
</div>
|
|
1338
1396
|
</div>
|
|
1339
1397
|
</div>
|
|
1340
|
-
<div class="sphinxsidebar">
|
|
1398
|
+
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
|
|
1341
1399
|
<div class="sphinxsidebarwrapper">
|
|
1342
1400
|
<h3><a href="../index.html">目次</a></h3>
|
|
1343
1401
|
<ul>
|
|
@@ -1345,20 +1403,20 @@
|
|
|
1345
1403
|
<li><a class="reference internal" href="#summary">7.8.1. 概要</a></li>
|
|
1346
1404
|
<li><a class="reference internal" href="#what-is-tokenize">7.8.2. 「トークナイズ」とはなにか</a></li>
|
|
1347
1405
|
<li><a class="reference internal" href="#built-in-tokenizsers">7.8.3. 組み込みトークナイザー</a><ul>
|
|
1348
|
-
<li><a class="reference internal" href="#tokenbigram">7.8.3.1. <
|
|
1349
|
-
<li><a class="reference internal" href="#tokenbigramsplitsymbol">7.8.3.2. <
|
|
1350
|
-
<li><a class="reference internal" href="#tokenbigramsplitsymbolalpha">7.8.3.3. <
|
|
1351
|
-
<li><a class="reference internal" href="#tokenbigramsplitsymbolalphadigit">7.8.3.4. <
|
|
1352
|
-
<li><a class="reference internal" href="#tokenbigramignoreblank">7.8.3.5. <
|
|
1353
|
-
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbol">7.8.3.6. <
|
|
1354
|
-
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalpha">7.8.3.7. <
|
|
1355
|
-
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalphadigit">7.8.3.8. <
|
|
1356
|
-
<li><a class="reference internal" href="#tokenunigram">7.8.3.9. <
|
|
1357
|
-
<li><a class="reference internal" href="#tokentrigram">7.8.3.10. <
|
|
1358
|
-
<li><a class="reference internal" href="#tokendelimit">7.8.3.11. <
|
|
1359
|
-
<li><a class="reference internal" href="#tokendelimitnull">7.8.3.12. <
|
|
1360
|
-
<li><a class="reference internal" href="#tokenmecab">7.8.3.13. <
|
|
1361
|
-
<li><a class="reference internal" href="#tokenregexp">7.8.3.14. <
|
|
1406
|
+
<li><a class="reference internal" href="#tokenbigram">7.8.3.1. <code class="docutils literal"><span class="pre">TokenBigram</span></code></a></li>
|
|
1407
|
+
<li><a class="reference internal" href="#tokenbigramsplitsymbol">7.8.3.2. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbol</span></code></a></li>
|
|
1408
|
+
<li><a class="reference internal" href="#tokenbigramsplitsymbolalpha">7.8.3.3. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code></a></li>
|
|
1409
|
+
<li><a class="reference internal" href="#tokenbigramsplitsymbolalphadigit">7.8.3.4. <code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code></a></li>
|
|
1410
|
+
<li><a class="reference internal" href="#tokenbigramignoreblank">7.8.3.5. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlank</span></code></a></li>
|
|
1411
|
+
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbol">7.8.3.6. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbol</span></code></a></li>
|
|
1412
|
+
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalpha">7.8.3.7. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlpha</span></code></a></li>
|
|
1413
|
+
<li><a class="reference internal" href="#tokenbigramignoreblanksplitsymbolalphadigit">7.8.3.8. <code class="docutils literal"><span class="pre">TokenBigramIgnoreBlankSplitSymbolAlphaDigit</span></code></a></li>
|
|
1414
|
+
<li><a class="reference internal" href="#tokenunigram">7.8.3.9. <code class="docutils literal"><span class="pre">TokenUnigram</span></code></a></li>
|
|
1415
|
+
<li><a class="reference internal" href="#tokentrigram">7.8.3.10. <code class="docutils literal"><span class="pre">TokenTrigram</span></code></a></li>
|
|
1416
|
+
<li><a class="reference internal" href="#tokendelimit">7.8.3.11. <code class="docutils literal"><span class="pre">TokenDelimit</span></code></a></li>
|
|
1417
|
+
<li><a class="reference internal" href="#tokendelimitnull">7.8.3.12. <code class="docutils literal"><span class="pre">TokenDelimitNull</span></code></a></li>
|
|
1418
|
+
<li><a class="reference internal" href="#tokenmecab">7.8.3.13. <code class="docutils literal"><span class="pre">TokenMecab</span></code></a></li>
|
|
1419
|
+
<li><a class="reference internal" href="#tokenregexp">7.8.3.14. <code class="docutils literal"><span class="pre">TokenRegexp</span></code></a></li>
|
|
1362
1420
|
</ul>
|
|
1363
1421
|
</li>
|
|
1364
1422
|
</ul>
|
|
@@ -1371,12 +1429,14 @@
|
|
|
1371
1429
|
<h4>次のトピックへ</h4>
|
|
1372
1430
|
<p class="topless"><a href="token_filters.html"
|
|
1373
1431
|
title="次の章へ">7.9. トークンフィルター</a></p>
|
|
1374
|
-
<
|
|
1375
|
-
|
|
1376
|
-
<
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1432
|
+
<div role="note" aria-label="source link">
|
|
1433
|
+
<h3>このページ</h3>
|
|
1434
|
+
<ul class="this-page-menu">
|
|
1435
|
+
<li><a href="../_sources/reference/tokenizers.txt"
|
|
1436
|
+
rel="nofollow">ソースコードを表示</a></li>
|
|
1437
|
+
</ul>
|
|
1438
|
+
</div>
|
|
1439
|
+
<div id="searchbox" style="display: none" role="search">
|
|
1380
1440
|
<h3>クイック検索</h3>
|
|
1381
1441
|
<form class="search" action="../search.html" method="get">
|
|
1382
1442
|
<input type="text" name="q" />
|
|
@@ -1393,7 +1453,7 @@
|
|
|
1393
1453
|
</div>
|
|
1394
1454
|
<div class="clearer"></div>
|
|
1395
1455
|
</div>
|
|
1396
|
-
<div class="related">
|
|
1456
|
+
<div class="related" role="navigation" aria-label="related navigation">
|
|
1397
1457
|
<h3>ナビゲーション</h3>
|
|
1398
1458
|
<ul>
|
|
1399
1459
|
<li class="right" style="margin-right: 10px">
|
|
@@ -1405,11 +1465,11 @@
|
|
|
1405
1465
|
<li class="right" >
|
|
1406
1466
|
<a href="normalizers.html" title="7.7. ノーマライザー"
|
|
1407
1467
|
>前へ</a> |</li>
|
|
1408
|
-
<li><a href="../index.html">Groonga v5.0.
|
|
1409
|
-
<li><a href="../reference.html" >7. リファレンスマニュアル</a> »</li>
|
|
1468
|
+
<li class="nav-item nav-item-0"><a href="../index.html">Groonga v5.0.6-226-gd7da7e7ドキュメント</a> »</li>
|
|
1469
|
+
<li class="nav-item nav-item-1"><a href="../reference.html" >7. リファレンスマニュアル</a> »</li>
|
|
1410
1470
|
</ul>
|
|
1411
1471
|
</div>
|
|
1412
|
-
<div class="footer">
|
|
1472
|
+
<div class="footer" role="contentinfo">
|
|
1413
1473
|
© Copyright 2009-2015, Brazil, Inc.
|
|
1414
1474
|
</div>
|
|
1415
1475
|
</body>
|