fluent-plugin-droonga 0.9.0 → 0.9.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
@@ -1,248 +0,0 @@
1
-
2
- # Copyright (C) 2013 Droonga Project
3
- #
4
- # This library is free software; you can redistribute it and/or
5
- # modify it under the terms of the GNU Lesser General Public
6
- # License version 2.1 as published by the Free Software Foundation.
7
- #
8
- # This library is distributed in the hope that it will be useful,
9
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
- # Lesser General Public License for more details.
12
- #
13
- # You should have received a copy of the GNU Lesser General Public
14
- # License along with this library; if not, write to the Free Software
15
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
-
17
- require "droonga/plugin/input_adapter/groonga/select"
18
-
19
- class InputAdapterGroongaSelectTest < Test::Unit::TestCase
20
- def setup
21
- @select = Droonga::GroongaInputAdapter::Select.new
22
- end
23
-
24
- class RequestTest < self
25
- def test_empty
26
- select_request = {
27
- "table" => "EmptyTable",
28
- "output_columns" => "_id",
29
- }
30
-
31
- expected_search_request = {
32
- "queries" => {
33
- "EmptyTable_result" => {
34
- "source" => "EmptyTable",
35
- "output" => {
36
- "elements" => [
37
- "startTime",
38
- "elapsedTime",
39
- "count",
40
- "attributes",
41
- "records",
42
- ],
43
- "attributes" => ["_id"],
44
- "offset" => 0,
45
- "limit" => 10,
46
- },
47
- },
48
- },
49
- }
50
-
51
- assert_equal(expected_search_request, convert(select_request))
52
- end
53
-
54
- private
55
- def convert(select_request)
56
- @select.convert(select_request)
57
- end
58
-
59
- class OutputColumnsTest < self
60
- def assert_attributes(expected_attributes, output_columns)
61
- select_request = {
62
- "table" => "EmptyTable",
63
- "output_columns" => output_columns,
64
- }
65
-
66
- expected_search_request = {
67
- "queries" => {
68
- "EmptyTable_result" => {
69
- "source" => "EmptyTable",
70
- "output" => {
71
- "elements" => [
72
- "startTime",
73
- "elapsedTime",
74
- "count",
75
- "attributes",
76
- "records",
77
- ],
78
- "attributes" => expected_attributes,
79
- "offset" => 0,
80
- "limit" => 10,
81
- },
82
- },
83
- },
84
- }
85
- assert_equal(expected_search_request, convert(select_request))
86
- end
87
-
88
- def test_multiple_columns
89
- assert_attributes(["_id", "_key"], "_id,_key")
90
- end
91
-
92
- class FunctionTest < self
93
- def test_single_argument
94
- assert_attributes(["snippet_html(content)"], "snippet_html(content)")
95
- end
96
-
97
- def test_with_columns
98
- assert_attributes(["_id","_key","snippet_html(content)"], "_id,_key,snippet_html(content)")
99
- end
100
- end
101
- end
102
-
103
- class MatchColumnsTest < self
104
- def assert_matchTo(expected_matchTo, match_columns)
105
- select_request = {
106
- "table" => "EmptyTable",
107
- "match_columns" => match_columns,
108
- "query" => "QueryTest",
109
- "output_columns" => "_id",
110
- }
111
-
112
- expected_search_request = {
113
- "queries" => {
114
- "EmptyTable_result" => {
115
- "source" => "EmptyTable",
116
- "condition"=> {
117
- "query" => "QueryTest",
118
- "matchTo"=> expected_matchTo,
119
- "defaultOperator"=> "&&",
120
- "allowPragma"=> false,
121
- "allowColumn"=> true,
122
- },
123
- "output" => {
124
- "elements" => [
125
- "startTime",
126
- "elapsedTime",
127
- "count",
128
- "attributes",
129
- "records",
130
- ],
131
- "attributes" => ["_id"],
132
- "offset" => 0,
133
- "limit" => 10,
134
- },
135
- },
136
- },
137
- }
138
- assert_equal(expected_search_request, convert(select_request))
139
- end
140
-
141
- def test_single_column
142
- assert_matchTo(["_key"], "_key")
143
- end
144
-
145
- def test_multiple_columns
146
- assert_matchTo(["_key", "content"], "_key || content")
147
- end
148
- end
149
-
150
- class OffsetTest < self
151
- def assert_offset(expected_offset, offset)
152
- select_request = {
153
- "table" => "EmptyTable",
154
- "output_columns" => "_id",
155
- }
156
- select_request["offset"] = offset unless offset.nil?
157
-
158
- expected_search_request = {
159
- "queries" => {
160
- "EmptyTable_result" => {
161
- "source" => "EmptyTable",
162
- "output" => {
163
- "elements" => [
164
- "startTime",
165
- "elapsedTime",
166
- "count",
167
- "attributes",
168
- "records",
169
- ],
170
- "attributes" => ["_id"],
171
- "offset" => expected_offset,
172
- "limit" => 10,
173
- },
174
- },
175
- },
176
- }
177
- assert_equal(expected_search_request, convert(select_request))
178
- end
179
-
180
- def test_zero
181
- assert_offset(0, "0")
182
- end
183
-
184
- def test_large
185
- assert_offset(100, "100")
186
- end
187
-
188
- def test_integer
189
- assert_offset(100, 100)
190
- end
191
-
192
- def test_default
193
- assert_offset(0, nil)
194
- end
195
- end
196
-
197
- class LimitTest < self
198
- def assert_limit(expected_limit, limit)
199
- select_request = {
200
- "table" => "EmptyTable",
201
- "output_columns" => "_id",
202
- }
203
- select_request["limit"] = limit unless limit.nil?
204
-
205
- expected_search_request = {
206
- "queries" => {
207
- "EmptyTable_result" => {
208
- "source" => "EmptyTable",
209
- "output" => {
210
- "elements" => [
211
- "startTime",
212
- "elapsedTime",
213
- "count",
214
- "attributes",
215
- "records",
216
- ],
217
- "attributes" => ["_id"],
218
- "offset" => 0,
219
- "limit" => expected_limit,
220
- },
221
- },
222
- },
223
- }
224
- assert_equal(expected_search_request, convert(select_request))
225
- end
226
-
227
- def test_zero
228
- assert_limit(0, "0")
229
- end
230
-
231
- def test_large
232
- assert_limit(100, "100")
233
- end
234
-
235
- def test_negative
236
- assert_limit(-1, "-1")
237
- end
238
-
239
- def test_integer
240
- assert_limit(100, 100)
241
- end
242
-
243
- def test_default
244
- assert_limit(10, nil)
245
- end
246
- end
247
- end
248
- end
@@ -1,44 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/command_mapper"
17
-
18
- class CommandMapperTest < Test::Unit::TestCase
19
- class RegisterTest < self
20
- def setup
21
- @command_mapper = Droonga::CommandMapper.new
22
- end
23
-
24
- def test_name
25
- @command_mapper.register(:select)
26
- assert_equal(:select, @command_mapper[:select])
27
- end
28
-
29
- def test_different_method_name
30
- @command_mapper.register(:command_name => :method_name)
31
- assert_equal(:method_name, @command_mapper[:command_name])
32
- end
33
-
34
- def test_multiple_pairs
35
- map = {
36
- :command_name_1 => :command_name_1,
37
- :command_name_2 => :command_name_2,
38
- }
39
- @command_mapper.register(map)
40
- assert_equal(["command_name_1", "command_name_2"],
41
- @command_mapper.commands)
42
- end
43
- end
44
- end
@@ -1,242 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "groonga_command_converter"
17
-
18
- class GroongaCommandConverterTest < Test::Unit::TestCase
19
- def setup
20
- options = {
21
- :id => "test",
22
- :date => date,
23
- :reply_to => reply_to,
24
- :status_code => status_code,
25
- :dataset => dataset,
26
- }
27
- @converter = Droonga::GroongaCommandConverter.new(options)
28
- end
29
-
30
- def test_table_create
31
- droonga_commands = []
32
- command = <<-COMMAND.chomp
33
- table_create Terms TABLE_PAT_KEY ShortText \
34
- --default_tokenizer TokenBigram --normalizer NormalizerAuto
35
- COMMAND
36
- @converter.convert(command) do |droonga_command|
37
- droonga_commands << droonga_command
38
- end
39
- assert_equal([
40
- {
41
- :id => "test:0",
42
- :date => formatted_date,
43
- :replyTo => reply_to,
44
- :statusCode => status_code,
45
- :dataset => dataset,
46
- :type => "table_create",
47
- :body => {
48
- :name => "Terms",
49
- :flags => "TABLE_PAT_KEY",
50
- :key_type => "ShortText",
51
- :default_tokenizer => "TokenBigram",
52
- :normalizer => "NormalizerAuto",
53
- },
54
- },
55
- ],
56
- droonga_commands)
57
- end
58
-
59
- def test_column_create
60
- droonga_commands = []
61
- command = <<-COMMAND.chomp
62
- column_create Terms Users_name COLUMN_INDEX|WITH_POSITION Users name
63
- COMMAND
64
- @converter.convert(command) do |droonga_command|
65
- droonga_commands << droonga_command
66
- end
67
- assert_equal([
68
- {
69
- :id => "test:0",
70
- :date => formatted_date,
71
- :replyTo => reply_to,
72
- :statusCode => status_code,
73
- :dataset => dataset,
74
- :type => "column_create",
75
- :body => {
76
- :table => "Terms",
77
- :name => "Users_name",
78
- :flags => "COLUMN_INDEX|WITH_POSITION",
79
- :type => "Users",
80
- :source => "name",
81
- },
82
- },
83
- ],
84
- droonga_commands)
85
- end
86
-
87
- def test_load
88
- droonga_commands = []
89
- command = <<-COMMAND.chomp
90
- load --table Users
91
- [
92
- ["_key","name"],
93
- ["user0","Abe Shinzo"],
94
- ["user1","Noda Yoshihiko"],
95
- ["user2","Kan Naoto"]
96
- ]
97
- COMMAND
98
- @converter.convert(command) do |droonga_command|
99
- droonga_commands << droonga_command
100
- end
101
- assert_equal([
102
- {
103
- :id => "test:0",
104
- :date => formatted_date,
105
- :replyTo => reply_to,
106
- :statusCode => status_code,
107
- :dataset => dataset,
108
- :type => "add",
109
- :body => {
110
- :table => "Users",
111
- :key => "user0",
112
- :values => {
113
- :name => "Abe Shinzo",
114
- },
115
- },
116
- },
117
- {
118
- :id => "test:1",
119
- :date => formatted_date,
120
- :replyTo => reply_to,
121
- :statusCode => status_code,
122
- :dataset => dataset,
123
- :type => "add",
124
- :body => {
125
- :table => "Users",
126
- :key => "user1",
127
- :values => {
128
- :name => "Noda Yoshihiko",
129
- },
130
- },
131
- },
132
- {
133
- :id => "test:2",
134
- :date => formatted_date,
135
- :replyTo => reply_to,
136
- :statusCode => status_code,
137
- :dataset => dataset,
138
- :type => "add",
139
- :body => {
140
- :table => "Users",
141
- :key => "user2",
142
- :values => {
143
- :name => "Kan Naoto",
144
- },
145
- },
146
- },
147
- ],
148
- droonga_commands)
149
- end
150
-
151
- def test_select
152
- droonga_commands = []
153
- command = <<-COMMAND.chomp
154
- select --filter "age<=30" --output_type "json" --table "Users"
155
- COMMAND
156
- @converter.convert(command) do |droonga_command|
157
- droonga_commands << droonga_command
158
- end
159
- assert_equal([
160
- {
161
- :id => "test:0",
162
- :date => formatted_date,
163
- :replyTo => reply_to,
164
- :statusCode => status_code,
165
- :dataset => dataset,
166
- :type => "select",
167
- :body => {
168
- :table => "Users",
169
- :filter => "age<=30",
170
- :output_type => "json",
171
- },
172
- },
173
- ],
174
- droonga_commands)
175
- end
176
-
177
- def test_multiple_commands
178
- droonga_commands = []
179
- commands = <<-COMMANDS.chomp
180
- table_create Terms TABLE_PAT_KEY ShortText \
181
- --default_tokenizer TokenBigram --normalizer NormalizerAuto
182
- column_create Terms Users_name COLUMN_INDEX|WITH_POSITION Users name
183
- COMMANDS
184
- @converter.convert(commands) do |droonga_command|
185
- droonga_commands << droonga_command
186
- end
187
- assert_equal([
188
- {
189
- :id => "test:0",
190
- :date => formatted_date,
191
- :replyTo => reply_to,
192
- :statusCode => status_code,
193
- :dataset => dataset,
194
- :type => "table_create",
195
- :body => {
196
- :name => "Terms",
197
- :flags => "TABLE_PAT_KEY",
198
- :key_type => "ShortText",
199
- :default_tokenizer => "TokenBigram",
200
- :normalizer => "NormalizerAuto",
201
- },
202
- },
203
- {
204
- :id => "test:1",
205
- :date => formatted_date,
206
- :replyTo => reply_to,
207
- :statusCode => status_code,
208
- :dataset => dataset,
209
- :type => "column_create",
210
- :body => {
211
- :table => "Terms",
212
- :name => "Users_name",
213
- :flags => "COLUMN_INDEX|WITH_POSITION",
214
- :type => "Users",
215
- :source => "name",
216
- },
217
- },
218
- ],
219
- droonga_commands)
220
- end
221
-
222
- private
223
- def date
224
- Time.utc(2013, 11, 29, 0, 0, 0)
225
- end
226
-
227
- def formatted_date
228
- "2013-11-29T00:00:00Z"
229
- end
230
-
231
- def reply_to
232
- "localhost:20033"
233
- end
234
-
235
- def status_code
236
- 200
237
- end
238
-
239
- def dataset
240
- "test-dataset"
241
- end
242
- end