fluent-plugin-droonga 0.9.0 → 0.9.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
@@ -1,106 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga"
19
- require "groonga/command/column-create"
20
-
21
- module Droonga
22
- class GroongaHandler
23
- class ColumnCreate < Command
24
- def process_request(request)
25
- command_class = Groonga::Command.find("column_create")
26
- @command = command_class.new("column_create", request)
27
-
28
- table_name = @command["table"]
29
- if table_name.nil? || @context[table_name].nil?
30
- raise CommandError.new(:status => Status::INVALID_ARGUMENT,
31
- :message => "table doesn't exist: <#{table_name.to_s}>",
32
- :result => false)
33
- end
34
-
35
- if @command.column_index?
36
- define_index
37
- else
38
- define_column
39
- end
40
- end
41
-
42
- private
43
- def define_column
44
- table_name = @command["table"]
45
- column_name = @command["name"]
46
- column_type = @command["type"]
47
-
48
- options = create_column_options
49
- Groonga::Schema.define(:context => @context) do |schema|
50
- schema.change_table(table_name) do |table|
51
- table.column(column_name, column_type, options)
52
- end
53
- end
54
- true
55
- end
56
-
57
- def create_column_options
58
- options = {}
59
- create_column_options_flags(options)
60
- options
61
- end
62
-
63
- def create_column_options_flags(options)
64
- options[:type] = :scalar
65
- if @command.column_scalar?
66
- options[:type] = :scalar
67
- elsif @command.column_vector?
68
- options[:type] = :vector
69
- end
70
- options
71
- end
72
-
73
- def define_index
74
- table_name = @command["table"]
75
- target_table = @command["type"]
76
- target_column = @command["source"]
77
-
78
- options = create_index_options
79
- Groonga::Schema.define(:context => @context) do |schema|
80
- schema.change_table(table_name) do |table|
81
- table.index("#{target_table}.#{target_column}", options)
82
- end
83
- end
84
- true
85
- end
86
-
87
- def create_index_options
88
- options = {}
89
- create_index_options_name(options)
90
- create_index_options_flags(options)
91
- options
92
- end
93
-
94
- def create_index_options_name(options)
95
- options[:name] = @command["name"]
96
- end
97
-
98
- def create_index_options_flags(options)
99
- options[:with_section] = true if @command.with_section?
100
- options[:with_weight] = true if @command.with_weight?
101
- options[:with_position] = true if @command.with_position?
102
- options
103
- end
104
- end
105
- end
106
- end
@@ -1,91 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga"
19
- require "groonga/command/table-create"
20
-
21
- module Droonga
22
- class GroongaHandler
23
- class TableCreate < Command
24
- def process_request(request)
25
- command_class = Groonga::Command.find("table_create")
26
- @command = command_class.new("table_create", request)
27
-
28
- name = @command["name"]
29
- unless name
30
- raise CommandError.new(:status => Status::INVALID_ARGUMENT,
31
- :message => "Should not create anonymous table",
32
- :result => false)
33
- end
34
-
35
- options = parse_command
36
- Groonga::Schema.define(:context => @context) do |schema|
37
- schema.create_table(name, options)
38
- end
39
- true
40
- end
41
-
42
- private
43
- def parse_command
44
- options = {}
45
- parse_flags(options)
46
- parse_key_type(options)
47
- parse_value_type(options)
48
- parse_default_tokenizer(options)
49
- parse_normalizer(options)
50
- options
51
- end
52
-
53
- def parse_flags(options)
54
- options[:type] = :hash
55
- if @command.table_no_key?
56
- options[:type] = :array
57
- elsif @command.table_hash_key?
58
- options[:type] = :hash
59
- elsif @command.table_pat_key?
60
- options[:type] = :patricia_trie
61
- elsif @command.table_dat_key?
62
- options[:type] = :double_array_trie
63
- end
64
- if @command.key_with_sis? and @command.table_pat_key?
65
- options[:key_with_sis] = true
66
- end
67
- options
68
- end
69
-
70
- def parse_key_type(options)
71
- options[:key_type] = @command["key_type"] if @command["key_type"]
72
- options
73
- end
74
-
75
- def parse_value_type(options)
76
- options[:value_type] = @command["value_type"] if @command["value_type"]
77
- options
78
- end
79
-
80
- def parse_default_tokenizer(options)
81
- options[:default_tokenizer] = @command["default_tokenizer"] if @command["default_tokenizer"]
82
- options
83
- end
84
-
85
- def parse_normalizer(options)
86
- options[:normalizer] = @command["normalizer"] if @command["normalizer"]
87
- options
88
- end
89
- end
90
- end
91
- end
@@ -1,108 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "droonga/watcher"
19
- require "droonga/sweeper"
20
- require "droonga/watch_schema"
21
- require "droonga/handler_plugin"
22
-
23
- module Droonga
24
- class WatchHandler < Droonga::HandlerPlugin
25
- repository.register("watch", self)
26
-
27
- def initialize(*args)
28
- super
29
-
30
- # XXX just workaround. This must be re-written.
31
- # When secondary and later processes opens the database,
32
- # creation processes of tables by the first process is
33
- # not finished yet. Then secondary and others tries to
34
- # create tables and raises errors. To avoid such a problem,
35
- # the creation processes of tables is disabled on workers.
36
- if $0 !~ /\AServer/
37
- ensure_schema_created
38
- else
39
- until @context["Keyword"]
40
- sleep 0.1
41
- end
42
- sleep 1
43
- end
44
-
45
- @watcher = Watcher.new(@context)
46
- @sweeper = Sweeper.new(@context)
47
- end
48
-
49
- command "watch.subscribe" => :subscribe
50
- def subscribe(message, messenger)
51
- subscriber, condition, query, route = parse_message(message)
52
- normalized_request = {
53
- :subscriber => subscriber,
54
- :condition => condition,
55
- :query => query,
56
- :route => route,
57
- }
58
- @watcher.subscribe(normalized_request)
59
- messenger.emit([true])
60
- end
61
-
62
- command "watch.unsubscribe" => :unsubscribe
63
- def unsubscribe(message, messenger)
64
- subscriber, condition, query, route = parse_message(message)
65
- normalized_request = {
66
- :subscriber => subscriber,
67
- :condition => condition,
68
- :query => query,
69
- }
70
- @watcher.unsubscribe(normalized_request)
71
- messenger.emit([true])
72
- end
73
-
74
- command "watch.feed" => :feed
75
- def feed(message, messenger)
76
- request = message.request
77
- @watcher.feed(:targets => request["targets"]) do |route, subscribers|
78
- published_message = {
79
- "to" => subscribers,
80
- "body" => request,
81
- }
82
- published_message = message.raw.merge(published_message)
83
- messenger.forward(published_message,
84
- "to" => route, "type" => "watch.publish")
85
- end
86
- end
87
-
88
- command "watch.sweep" => :sweep
89
- def sweep(message, messenger)
90
- @sweeper.sweep_expired_subscribers
91
- end
92
-
93
- private
94
- def parse_message(message)
95
- request = message.request
96
- subscriber = request["subscriber"]
97
- condition = request["condition"]
98
- route = request["route"] || message["from"]
99
- query = condition && condition.to_json
100
- [subscriber, condition, query, route]
101
- end
102
-
103
- def ensure_schema_created
104
- schema = WatchSchema.new(@context)
105
- schema.ensure_created
106
- end
107
- end
108
- end
@@ -1,49 +0,0 @@
1
- # Copyright (C) 2013-2014 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/input_adapter_plugin"
17
-
18
- module Droonga
19
- class GroongaInputAdapter < Droonga::InputAdapterPlugin
20
- repository.register("groonga", self)
21
-
22
- command :select
23
- def select(input_message)
24
- command = Select.new
25
- select_request = input_message.body
26
- search_request = command.convert(select_request)
27
- input_message.add_route("select_response")
28
- input_message.command = "search"
29
- input_message.body = search_request
30
- end
31
-
32
- command :table_create
33
- def table_create(input_message)
34
- input_message.add_route("groonga_generic_response")
35
- end
36
-
37
- command :table_remove
38
- def table_remove(input_message)
39
- input_message.add_route("groonga_generic_response")
40
- end
41
-
42
- command :column_create
43
- def column_create(input_message)
44
- input_message.add_route("groonga_generic_response")
45
- end
46
- end
47
- end
48
-
49
- require "droonga/plugin/input_adapter/groonga/select"
@@ -1,63 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- module Droonga
17
- class GroongaInputAdapter
18
- class Select
19
- def convert(select_request)
20
- table = select_request["table"]
21
- result_name = table + "_result"
22
- match_columns = select_request["match_columns"]
23
- match_to = match_columns ? match_columns.split(/ *\|\| */) : []
24
- query = select_request["query"]
25
- output_columns = select_request["output_columns"] || ""
26
- attributes = output_columns.split(/, */)
27
- offset = (select_request["offset"] || "0").to_i
28
- limit = (select_request["limit"] || "10").to_i
29
-
30
- search_request = {
31
- "queries" => {
32
- result_name => {
33
- "source" => table,
34
- "output" => {
35
- "elements" => [
36
- "startTime",
37
- "elapsedTime",
38
- "count",
39
- "attributes",
40
- "records",
41
- ],
42
- "attributes" => attributes,
43
- "offset" => offset,
44
- "limit" => limit,
45
- },
46
- }
47
- }
48
- }
49
- if query
50
- condition = {
51
- "query" => query,
52
- "matchTo"=> match_to,
53
- "defaultOperator"=> "&&",
54
- "allowPragma"=> false,
55
- "allowColumn"=> true,
56
- }
57
- search_request["queries"][result_name]["condition"] = condition
58
- end
59
- search_request
60
- end
61
- end
62
- end
63
- end
@@ -1,51 +0,0 @@
1
- # Copyright (C) 2013-2014 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/output_adapter_plugin"
17
-
18
- module Droonga
19
- class CRUDOutputAdapter < Droonga::OutputAdapterPlugin
20
- repository.register("crud", self)
21
-
22
- command :crud_generic_response
23
- def crud_generic_response(output_message)
24
- if output_message.body.include?("errors")
25
- errors = output_message.body["errors"]
26
- if errors && !errors.empty?
27
- output_message.errors = errors
28
-
29
- status_codes = []
30
- errors.values.each do |error|
31
- status_codes << error["statusCode"]
32
- end
33
- status_codes = status_codes.uniq
34
- if status_codes.size == 1
35
- output_message.status_code = status_codes.first
36
- else
37
- output_message.status_code = MessageProcessingError::STATUS_CODE
38
- end
39
-
40
- output_message.body = errors.values.first["body"]
41
- end
42
- end
43
- if output_message.body.include?("success")
44
- success = output_message.body["success"]
45
- unless success.nil?
46
- output_message.body = output_message.body["success"]
47
- end
48
- end
49
- end
50
- end
51
- end