fluent-plugin-droonga 0.9.0 → 0.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
@@ -1,248 +0,0 @@
1
-
2
- # Copyright (C) 2013 Droonga Project
3
- #
4
- # This library is free software; you can redistribute it and/or
5
- # modify it under the terms of the GNU Lesser General Public
6
- # License version 2.1 as published by the Free Software Foundation.
7
- #
8
- # This library is distributed in the hope that it will be useful,
9
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
- # Lesser General Public License for more details.
12
- #
13
- # You should have received a copy of the GNU Lesser General Public
14
- # License along with this library; if not, write to the Free Software
15
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
-
17
- require "droonga/plugin/input_adapter/groonga/select"
18
-
19
- class InputAdapterGroongaSelectTest < Test::Unit::TestCase
20
- def setup
21
- @select = Droonga::GroongaInputAdapter::Select.new
22
- end
23
-
24
- class RequestTest < self
25
- def test_empty
26
- select_request = {
27
- "table" => "EmptyTable",
28
- "output_columns" => "_id",
29
- }
30
-
31
- expected_search_request = {
32
- "queries" => {
33
- "EmptyTable_result" => {
34
- "source" => "EmptyTable",
35
- "output" => {
36
- "elements" => [
37
- "startTime",
38
- "elapsedTime",
39
- "count",
40
- "attributes",
41
- "records",
42
- ],
43
- "attributes" => ["_id"],
44
- "offset" => 0,
45
- "limit" => 10,
46
- },
47
- },
48
- },
49
- }
50
-
51
- assert_equal(expected_search_request, convert(select_request))
52
- end
53
-
54
- private
55
- def convert(select_request)
56
- @select.convert(select_request)
57
- end
58
-
59
- class OutputColumnsTest < self
60
- def assert_attributes(expected_attributes, output_columns)
61
- select_request = {
62
- "table" => "EmptyTable",
63
- "output_columns" => output_columns,
64
- }
65
-
66
- expected_search_request = {
67
- "queries" => {
68
- "EmptyTable_result" => {
69
- "source" => "EmptyTable",
70
- "output" => {
71
- "elements" => [
72
- "startTime",
73
- "elapsedTime",
74
- "count",
75
- "attributes",
76
- "records",
77
- ],
78
- "attributes" => expected_attributes,
79
- "offset" => 0,
80
- "limit" => 10,
81
- },
82
- },
83
- },
84
- }
85
- assert_equal(expected_search_request, convert(select_request))
86
- end
87
-
88
- def test_multiple_columns
89
- assert_attributes(["_id", "_key"], "_id,_key")
90
- end
91
-
92
- class FunctionTest < self
93
- def test_single_argument
94
- assert_attributes(["snippet_html(content)"], "snippet_html(content)")
95
- end
96
-
97
- def test_with_columns
98
- assert_attributes(["_id","_key","snippet_html(content)"], "_id,_key,snippet_html(content)")
99
- end
100
- end
101
- end
102
-
103
- class MatchColumnsTest < self
104
- def assert_matchTo(expected_matchTo, match_columns)
105
- select_request = {
106
- "table" => "EmptyTable",
107
- "match_columns" => match_columns,
108
- "query" => "QueryTest",
109
- "output_columns" => "_id",
110
- }
111
-
112
- expected_search_request = {
113
- "queries" => {
114
- "EmptyTable_result" => {
115
- "source" => "EmptyTable",
116
- "condition"=> {
117
- "query" => "QueryTest",
118
- "matchTo"=> expected_matchTo,
119
- "defaultOperator"=> "&&",
120
- "allowPragma"=> false,
121
- "allowColumn"=> true,
122
- },
123
- "output" => {
124
- "elements" => [
125
- "startTime",
126
- "elapsedTime",
127
- "count",
128
- "attributes",
129
- "records",
130
- ],
131
- "attributes" => ["_id"],
132
- "offset" => 0,
133
- "limit" => 10,
134
- },
135
- },
136
- },
137
- }
138
- assert_equal(expected_search_request, convert(select_request))
139
- end
140
-
141
- def test_single_column
142
- assert_matchTo(["_key"], "_key")
143
- end
144
-
145
- def test_multiple_columns
146
- assert_matchTo(["_key", "content"], "_key || content")
147
- end
148
- end
149
-
150
- class OffsetTest < self
151
- def assert_offset(expected_offset, offset)
152
- select_request = {
153
- "table" => "EmptyTable",
154
- "output_columns" => "_id",
155
- }
156
- select_request["offset"] = offset unless offset.nil?
157
-
158
- expected_search_request = {
159
- "queries" => {
160
- "EmptyTable_result" => {
161
- "source" => "EmptyTable",
162
- "output" => {
163
- "elements" => [
164
- "startTime",
165
- "elapsedTime",
166
- "count",
167
- "attributes",
168
- "records",
169
- ],
170
- "attributes" => ["_id"],
171
- "offset" => expected_offset,
172
- "limit" => 10,
173
- },
174
- },
175
- },
176
- }
177
- assert_equal(expected_search_request, convert(select_request))
178
- end
179
-
180
- def test_zero
181
- assert_offset(0, "0")
182
- end
183
-
184
- def test_large
185
- assert_offset(100, "100")
186
- end
187
-
188
- def test_integer
189
- assert_offset(100, 100)
190
- end
191
-
192
- def test_default
193
- assert_offset(0, nil)
194
- end
195
- end
196
-
197
- class LimitTest < self
198
- def assert_limit(expected_limit, limit)
199
- select_request = {
200
- "table" => "EmptyTable",
201
- "output_columns" => "_id",
202
- }
203
- select_request["limit"] = limit unless limit.nil?
204
-
205
- expected_search_request = {
206
- "queries" => {
207
- "EmptyTable_result" => {
208
- "source" => "EmptyTable",
209
- "output" => {
210
- "elements" => [
211
- "startTime",
212
- "elapsedTime",
213
- "count",
214
- "attributes",
215
- "records",
216
- ],
217
- "attributes" => ["_id"],
218
- "offset" => 0,
219
- "limit" => expected_limit,
220
- },
221
- },
222
- },
223
- }
224
- assert_equal(expected_search_request, convert(select_request))
225
- end
226
-
227
- def test_zero
228
- assert_limit(0, "0")
229
- end
230
-
231
- def test_large
232
- assert_limit(100, "100")
233
- end
234
-
235
- def test_negative
236
- assert_limit(-1, "-1")
237
- end
238
-
239
- def test_integer
240
- assert_limit(100, 100)
241
- end
242
-
243
- def test_default
244
- assert_limit(10, nil)
245
- end
246
- end
247
- end
248
- end
@@ -1,44 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/command_mapper"
17
-
18
- class CommandMapperTest < Test::Unit::TestCase
19
- class RegisterTest < self
20
- def setup
21
- @command_mapper = Droonga::CommandMapper.new
22
- end
23
-
24
- def test_name
25
- @command_mapper.register(:select)
26
- assert_equal(:select, @command_mapper[:select])
27
- end
28
-
29
- def test_different_method_name
30
- @command_mapper.register(:command_name => :method_name)
31
- assert_equal(:method_name, @command_mapper[:command_name])
32
- end
33
-
34
- def test_multiple_pairs
35
- map = {
36
- :command_name_1 => :command_name_1,
37
- :command_name_2 => :command_name_2,
38
- }
39
- @command_mapper.register(map)
40
- assert_equal(["command_name_1", "command_name_2"],
41
- @command_mapper.commands)
42
- end
43
- end
44
- end
@@ -1,242 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "groonga_command_converter"
17
-
18
- class GroongaCommandConverterTest < Test::Unit::TestCase
19
- def setup
20
- options = {
21
- :id => "test",
22
- :date => date,
23
- :reply_to => reply_to,
24
- :status_code => status_code,
25
- :dataset => dataset,
26
- }
27
- @converter = Droonga::GroongaCommandConverter.new(options)
28
- end
29
-
30
- def test_table_create
31
- droonga_commands = []
32
- command = <<-COMMAND.chomp
33
- table_create Terms TABLE_PAT_KEY ShortText \
34
- --default_tokenizer TokenBigram --normalizer NormalizerAuto
35
- COMMAND
36
- @converter.convert(command) do |droonga_command|
37
- droonga_commands << droonga_command
38
- end
39
- assert_equal([
40
- {
41
- :id => "test:0",
42
- :date => formatted_date,
43
- :replyTo => reply_to,
44
- :statusCode => status_code,
45
- :dataset => dataset,
46
- :type => "table_create",
47
- :body => {
48
- :name => "Terms",
49
- :flags => "TABLE_PAT_KEY",
50
- :key_type => "ShortText",
51
- :default_tokenizer => "TokenBigram",
52
- :normalizer => "NormalizerAuto",
53
- },
54
- },
55
- ],
56
- droonga_commands)
57
- end
58
-
59
- def test_column_create
60
- droonga_commands = []
61
- command = <<-COMMAND.chomp
62
- column_create Terms Users_name COLUMN_INDEX|WITH_POSITION Users name
63
- COMMAND
64
- @converter.convert(command) do |droonga_command|
65
- droonga_commands << droonga_command
66
- end
67
- assert_equal([
68
- {
69
- :id => "test:0",
70
- :date => formatted_date,
71
- :replyTo => reply_to,
72
- :statusCode => status_code,
73
- :dataset => dataset,
74
- :type => "column_create",
75
- :body => {
76
- :table => "Terms",
77
- :name => "Users_name",
78
- :flags => "COLUMN_INDEX|WITH_POSITION",
79
- :type => "Users",
80
- :source => "name",
81
- },
82
- },
83
- ],
84
- droonga_commands)
85
- end
86
-
87
- def test_load
88
- droonga_commands = []
89
- command = <<-COMMAND.chomp
90
- load --table Users
91
- [
92
- ["_key","name"],
93
- ["user0","Abe Shinzo"],
94
- ["user1","Noda Yoshihiko"],
95
- ["user2","Kan Naoto"]
96
- ]
97
- COMMAND
98
- @converter.convert(command) do |droonga_command|
99
- droonga_commands << droonga_command
100
- end
101
- assert_equal([
102
- {
103
- :id => "test:0",
104
- :date => formatted_date,
105
- :replyTo => reply_to,
106
- :statusCode => status_code,
107
- :dataset => dataset,
108
- :type => "add",
109
- :body => {
110
- :table => "Users",
111
- :key => "user0",
112
- :values => {
113
- :name => "Abe Shinzo",
114
- },
115
- },
116
- },
117
- {
118
- :id => "test:1",
119
- :date => formatted_date,
120
- :replyTo => reply_to,
121
- :statusCode => status_code,
122
- :dataset => dataset,
123
- :type => "add",
124
- :body => {
125
- :table => "Users",
126
- :key => "user1",
127
- :values => {
128
- :name => "Noda Yoshihiko",
129
- },
130
- },
131
- },
132
- {
133
- :id => "test:2",
134
- :date => formatted_date,
135
- :replyTo => reply_to,
136
- :statusCode => status_code,
137
- :dataset => dataset,
138
- :type => "add",
139
- :body => {
140
- :table => "Users",
141
- :key => "user2",
142
- :values => {
143
- :name => "Kan Naoto",
144
- },
145
- },
146
- },
147
- ],
148
- droonga_commands)
149
- end
150
-
151
- def test_select
152
- droonga_commands = []
153
- command = <<-COMMAND.chomp
154
- select --filter "age<=30" --output_type "json" --table "Users"
155
- COMMAND
156
- @converter.convert(command) do |droonga_command|
157
- droonga_commands << droonga_command
158
- end
159
- assert_equal([
160
- {
161
- :id => "test:0",
162
- :date => formatted_date,
163
- :replyTo => reply_to,
164
- :statusCode => status_code,
165
- :dataset => dataset,
166
- :type => "select",
167
- :body => {
168
- :table => "Users",
169
- :filter => "age<=30",
170
- :output_type => "json",
171
- },
172
- },
173
- ],
174
- droonga_commands)
175
- end
176
-
177
- def test_multiple_commands
178
- droonga_commands = []
179
- commands = <<-COMMANDS.chomp
180
- table_create Terms TABLE_PAT_KEY ShortText \
181
- --default_tokenizer TokenBigram --normalizer NormalizerAuto
182
- column_create Terms Users_name COLUMN_INDEX|WITH_POSITION Users name
183
- COMMANDS
184
- @converter.convert(commands) do |droonga_command|
185
- droonga_commands << droonga_command
186
- end
187
- assert_equal([
188
- {
189
- :id => "test:0",
190
- :date => formatted_date,
191
- :replyTo => reply_to,
192
- :statusCode => status_code,
193
- :dataset => dataset,
194
- :type => "table_create",
195
- :body => {
196
- :name => "Terms",
197
- :flags => "TABLE_PAT_KEY",
198
- :key_type => "ShortText",
199
- :default_tokenizer => "TokenBigram",
200
- :normalizer => "NormalizerAuto",
201
- },
202
- },
203
- {
204
- :id => "test:1",
205
- :date => formatted_date,
206
- :replyTo => reply_to,
207
- :statusCode => status_code,
208
- :dataset => dataset,
209
- :type => "column_create",
210
- :body => {
211
- :table => "Terms",
212
- :name => "Users_name",
213
- :flags => "COLUMN_INDEX|WITH_POSITION",
214
- :type => "Users",
215
- :source => "name",
216
- },
217
- },
218
- ],
219
- droonga_commands)
220
- end
221
-
222
- private
223
- def date
224
- Time.utc(2013, 11, 29, 0, 0, 0)
225
- end
226
-
227
- def formatted_date
228
- "2013-11-29T00:00:00Z"
229
- end
230
-
231
- def reply_to
232
- "localhost:20033"
233
- end
234
-
235
- def status_code
236
- 200
237
- end
238
-
239
- def dataset
240
- "test-dataset"
241
- end
242
- end