fluent-plugin-droonga 0.9.0 → 0.9.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
data/lib/droonga/plugin/output_adapter/groonga/select.rb DELETED
@@ -1,54 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- module Droonga
17
- class GroongaOutputAdapter
18
- class Select
19
- def convert(search_response)
20
- select_responses = search_response.collect do |key, value|
21
- status_code = 0
22
-
23
- start_time = value["startTime"]
24
- start_time_in_unix_time = if start_time
25
- Time.parse(start_time).to_f
26
- else
27
- Time.now.to_f
28
- end
29
- elapsed_time = value["elapsedTime"] || 0
30
- count = value["count"]
31
-
32
- attributes = value["attributes"] || []
33
- converted_attributes = attributes.collect do |attribute|
34
- name = attribute["name"]
35
- type = attribute["type"]
36
- [name, type]
37
- end
38
-
39
- header = [status_code, start_time_in_unix_time, elapsed_time]
40
- records = value["records"]
41
- if records.empty?
42
- results = [[count], converted_attributes]
43
- else
44
- results = [[count], converted_attributes, records]
45
- end
46
- body = [results]
47
-
48
- [header, body]
49
- end
50
- select_responses.first
51
- end
52
- end
53
- end
54
- end
data/lib/groonga_command_converter.rb DELETED
@@ -1,143 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga/command/parser"
19
- require "digest/sha1"
20
- require "time"
21
-
22
- module Droonga
23
- class GroongaCommandConverter
24
- STATUS_OK = 200
25
-
26
- def initialize(options={})
27
- @options = options
28
- @count = 0
29
-
30
- @command_parser = Groonga::Command::Parser.new
31
- end
32
-
33
- def convert(input, &block)
34
- @command_parser.on_command do |command|
35
- case command.name
36
- when "table_create"
37
- yield create_table_create_command(command)
38
- when "table_remove"
39
- yield create_table_remove_command(command)
40
- when "column_create"
41
- yield create_column_create_command(command)
42
- when "select"
43
- yield create_select_command(command)
44
- end
45
- end
46
-
47
- parsed_values = nil
48
- parsed_columns = nil
49
- @command_parser.on_load_start do |command|
50
- parsed_values = []
51
- parsed_columns = nil
52
- end
53
- @command_parser.on_load_columns do |command, columns|
54
- parsed_columns = columns
55
- end
56
- @command_parser.on_load_value do |command, value|
57
- parsed_values << value
58
- end
59
- @command_parser.on_load_complete do |command|
60
- command[:columns] = parsed_columns.join(",")
61
- command[:values] = parsed_values.to_json
62
- split_load_command_to_add_commands(command, &block)
63
- end
64
-
65
- input.each_line do |line|
66
- @command_parser << line
67
- end
68
- @command_parser.finish
69
- end
70
-
71
- private
72
- def create_message(type, body)
73
- id = @options[:id]
74
- if id.nil?
75
- id = new_unique_id
76
- else
77
- id = "#{id}:#{@count}"
78
- @count += 1
79
- end
80
-
81
- {
82
- :id => id,
83
- :date => format_date(@options[:date] || Time.now),
84
- :replyTo => @options[:reply_to],
85
- :statusCode => @options[:status_code] || STATUS_OK,
86
- :dataset => @options[:dataset],
87
- :type => type,
88
- :body => body,
89
- }
90
- end
91
-
92
- def new_unique_id
93
- now = Time.now
94
- now_msec = now.to_i * 1000 + now.usec
95
- random_string = rand(36 ** 16).to_s(36) # Base36
96
- Digest::SHA1.hexdigest("#{now_msec}:#{random_string}")
97
- end
98
-
99
- def format_date(time)
100
- time.iso8601
101
- end
102
-
103
- def create_table_create_command(command)
104
- create_message("table_create", command.arguments)
105
- end
106
-
107
- def create_table_remove_command(command)
108
- create_message("table_remove", command.arguments)
109
- end
110
-
111
- def create_column_create_command(command)
112
- create_message("column_create", command.arguments)
113
- end
114
-
115
- def split_load_command_to_add_commands(command, &block)
116
- columns = command[:columns].split(",")
117
- values = command[:values]
118
- values = JSON.parse(values)
119
- values.each do |record|
120
- body = {
121
- :table => command[:table],
122
- }
123
-
124
- record_values = {}
125
- record.each_with_index do |value, column_index|
126
- column = columns[column_index]
127
- if column == "_key"
128
- body[:key] = value
129
- else
130
- record_values[column.to_sym] = value
131
- end
132
- end
133
- body[:values] = record_values unless record_values.empty?
134
-
135
- yield create_message("add", body)
136
- end
137
- end
138
-
139
- def create_select_command(command)
140
- create_message("select", command.arguments)
141
- end
142
- end
143
- end
data/sample/fluentd.conf DELETED
@@ -1,8 +0,0 @@
1
- <source>
2
- type forward
3
- </source>
4
-
5
- <match droonga.message>
6
- type droonga
7
- handlers search,groonga
8
- </match>
data/test/unit/plugin/distributor/test_search_planner.rb DELETED
@@ -1,1102 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/plugin/distributor/distributed_search_planner"
17
-
18
- class DistributedSearchPlannerTest < Test::Unit::TestCase
19
- def assert_planned(expected, search_request)
20
- plan = Droonga::DistributedSearchPlanner.new(search_request)
21
- actual = plan.messages
22
- assert_equal(expected, actual)
23
- end
24
-
25
- class MultipleQueriesTest < self
26
- def test_distribute
27
- request = {
28
- "type" => "search",
29
- "dataset" => "Droonga",
30
- "body" => {
31
- "queries" => {
32
- "query1" => {
33
- "source" => "User",
34
- "output" => {
35
- "format" => "complex",
36
- "elements" => ["count", "records"],
37
- "attributes" => [],
38
- "offset" => 0,
39
- "limit" => 10,
40
- },
41
- },
42
- "query2" => {
43
- "source" => "User",
44
- "output" => {
45
- "format" => "complex",
46
- "elements" => ["count", "records"],
47
- "attributes" => [],
48
- "offset" => 0,
49
- "limit" => 20,
50
- },
51
- },
52
- "query3" => {
53
- "source" => "User",
54
- "output" => {
55
- "format" => "complex",
56
- "elements" => ["count", "records"],
57
- "attributes" => [],
58
- "offset" => 0,
59
- "limit" => 30,
60
- },
61
- },
62
- },
63
- },
64
- }
65
-
66
- expected_plan = []
67
-
68
- expected_plan << {
69
- "type" => "search_reduce",
70
- "body" => {
71
- "query1" => {
72
- "query1_reduced" => {
73
- "count" => {
74
- "type" => "sum",
75
- },
76
- "records" => {
77
- "type" => "sort",
78
- "operators" => [],
79
- "limit" => 10,
80
- },
81
- },
82
- },
83
- },
84
- "inputs" => ["query1"],
85
- "outputs" => ["query1_reduced"],
86
- }
87
- expected_plan << {
88
- "type" => "search_reduce",
89
- "body" => {
90
- "query2" => {
91
- "query2_reduced" => {
92
- "count" => {
93
- "type" => "sum",
94
- },
95
- "records" => {
96
- "type" => "sort",
97
- "operators" => [],
98
- "limit" => 20,
99
- },
100
- },
101
- },
102
- },
103
- "inputs" => ["query2"],
104
- "outputs" => ["query2_reduced"],
105
- }
106
- expected_plan << {
107
- "type" => "search_reduce",
108
- "body" => {
109
- "query3" => {
110
- "query3_reduced" => {
111
- "count" => {
112
- "type" => "sum",
113
- },
114
- "records" => {
115
- "type" => "sort",
116
- "operators" => [],
117
- "limit" => 30,
118
- },
119
- },
120
- },
121
- },
122
- "inputs" => ["query3"],
123
- "outputs" => ["query3_reduced"],
124
- }
125
-
126
- gatherer = {
127
- "type" => "search_gather",
128
- "body" => {
129
- "query1_reduced" => {
130
- "output" => "query1",
131
- "elements" => {
132
- "records" => {
133
- "type" => "sort",
134
- "offset" => 0,
135
- "limit" => 10,
136
- "format" => "complex",
137
- "attributes" => [],
138
- },
139
- },
140
- },
141
- "query2_reduced" => {
142
- "output" => "query2",
143
- "elements" => {
144
- "records" => {
145
- "type" => "sort",
146
- "offset" => 0,
147
- "limit" => 20,
148
- "format" => "complex",
149
- "attributes" => [],
150
- },
151
- },
152
- },
153
- "query3_reduced" => {
154
- "output" => "query3",
155
- "elements" => {
156
- "records" => {
157
- "type" => "sort",
158
- "offset" => 0,
159
- "limit" => 30,
160
- "format" => "complex",
161
- "attributes" => [],
162
- },
163
- },
164
- },
165
- },
166
- "inputs" => [
167
- "query1_reduced",
168
- "query2_reduced",
169
- "query3_reduced",
170
- ],
171
- "post" => true,
172
- }
173
- expected_plan << gatherer
174
-
175
- searcher = {
176
- "type" => "broadcast",
177
- "command" => "search",
178
- "dataset" => "Droonga",
179
- "body" => {
180
- "queries" => {
181
- "query1" => {
182
- "source" => "User",
183
- "output" => {
184
- "format" => "simple",
185
- "elements" => ["count", "records"],
186
- "attributes" => [],
187
- "offset" => 0,
188
- "limit" => 10,
189
- },
190
- },
191
- "query2" => {
192
- "source" => "User",
193
- "output" => {
194
- "format" => "simple",
195
- "elements" => ["count", "records"],
196
- "attributes" => [],
197
- "offset" => 0,
198
- "limit" => 20,
199
- },
200
- },
201
- "query3" => {
202
- "source" => "User",
203
- "output" => {
204
- "format" => "simple",
205
- "elements" => ["count", "records"],
206
- "attributes" => [],
207
- "offset" => 0,
208
- "limit" => 30,
209
- },
210
- },
211
- },
212
- },
213
- "outputs" => [
214
- "query1",
215
- "query2",
216
- "query3",
217
- ],
218
- "replica" => "random",
219
- }
220
- expected_plan << searcher
221
-
222
- assert_planned(expected_plan, request)
223
- end
224
- end
225
-
226
- class SingleQueryTest < self
227
- def test_no_output
228
- request = {
229
- "type" => "search",
230
- "dataset" => "Droonga",
231
- "body" => {
232
- "queries" => {
233
- "no_output" => {
234
- "source" => "User",
235
- "sortBy" => {
236
- "keys" => ["name"],
237
- "offset" => 0,
238
- "limit" => 1,
239
- },
240
- },
241
- },
242
- },
243
- }
244
-
245
- expected_plan = []
246
- expected_plan << gatherer(request, :no_output => true)
247
- expected_plan << searcher(request, :no_output => true)
248
- assert_planned(expected_plan, request)
249
- end
250
-
251
- def test_no_records_element
252
- request = {
253
- "type" => "search",
254
- "dataset" => "Droonga",
255
- "body" => {
256
- "queries" => {
257
- "no_records" => {
258
- "source" => "User",
259
- "sortBy" => {
260
- "keys" => ["name"],
261
- "offset" => 0,
262
- "limit" => 1,
263
- },
264
- "output" => {
265
- "elements" => ["count"],
266
- },
267
- },
268
- },
269
- },
270
- }
271
-
272
- expected_plan = []
273
- expected_plan << reducer(request, {
274
- "count" => {
275
- "type" => "sum",
276
- },
277
- })
278
- expected_plan << gatherer(request)
279
- expected_plan << searcher(request, :sort_limit => 1,
280
- :output_limit => 0)
281
- assert_planned(expected_plan, request)
282
- end
283
-
284
- def test_no_output_limit
285
- request = {
286
- "type" => "search",
287
- "dataset" => "Droonga",
288
- "body" => {
289
- "queries" => {
290
- "no_limit" => {
291
- "source" => "User",
292
- "output" => {
293
- "format" => "complex",
294
- "elements" => ["count", "records"],
295
- },
296
- },
297
- },
298
- },
299
- }
300
-
301
- expected_plan = []
302
- expected_plan << reducer(request, {
303
- "count" => {
304
- "type" => "sum",
305
- },
306
- })
307
- expected_plan << gatherer(request)
308
- expected_plan << searcher(request, :output_offset => 0,
309
- :output_limit => 0)
310
- assert_planned(expected_plan, request)
311
- end
312
-
313
- def test_have_records
314
- request = {
315
- "type" => "search",
316
- "dataset" => "Droonga",
317
- "body" => {
318
- "queries" => {
319
- "have_records" => {
320
- "source" => "User",
321
- "output" => {
322
- "format" => "complex",
323
- "elements" => ["records"],
324
- "attributes" => ["_key", "name", "age"],
325
- "offset" => 0,
326
- "limit" => 1,
327
- },
328
- },
329
- },
330
- },
331
- }
332
-
333
- expected_plan = []
334
- expected_plan << reducer(request, {
335
- "records" => {
336
- "type" => "sort",
337
- "operators" => [],
338
- "limit" => 1,
339
- },
340
- })
341
- expected_plan << gatherer(request, :elements => {
342
- "records" => records_mapper(
343
- :offset => 0,
344
- :limit => 1,
345
- :format => "complex",
346
- :attributes => ["_key", "name", "age"],
347
- ),
348
- })
349
- expected_plan << searcher(request, :output_offset => 0,
350
- :output_limit => 1)
351
- assert_planned(expected_plan, request)
352
- end
353
-
354
- def test_have_output_offset
355
- request = {
356
- "type" => "search",
357
- "dataset" => "Droonga",
358
- "body" => {
359
- "queries" => {
360
- "have_records" => {
361
- "source" => "User",
362
- "output" => {
363
- "format" => "complex",
364
- "elements" => ["records"],
365
- "attributes" => ["_key", "name", "age"],
366
- "offset" => 1,
367
- "limit" => 1,
368
- },
369
- },
370
- },
371
- },
372
- }
373
-
374
- expected_plan = []
375
- expected_plan << reducer(request, {
376
- "records" => {
377
- "type" => "sort",
378
- "operators" => [],
379
- "limit" => 2,
380
- },
381
- })
382
- expected_plan << gatherer(request, :elements => {
383
- "records" => records_mapper(
384
- :offset => 1,
385
- :limit => 1,
386
- :format => "complex",
387
- :attributes => ["_key", "name", "age"],
388
- ),
389
- })
390
- expected_plan << searcher(request, :output_offset => 0,
391
- :output_limit => 2)
392
- assert_planned(expected_plan, request)
393
- end
394
-
395
- def test_have_simple_sortBy
396
- request = {
397
- "type" => "search",
398
- "dataset" => "Droonga",
399
- "body" => {
400
- "queries" => {
401
- "have_records" => {
402
- "source" => "User",
403
- "sortBy" => ["name"],
404
- "output" => {
405
- "format" => "complex",
406
- "elements" => ["records"],
407
- "attributes" => ["_key", "name", "age"],
408
- "offset" => 0,
409
- "limit" => 1,
410
- },
411
- },
412
- },
413
- },
414
- }
415
-
416
- expected_plan = []
417
- expected_plan << reducer(request, {
418
- "records" => {
419
- "type" => "sort",
420
- "operators" => [
421
- { "column" => 1, "operator" => "<" },
422
- ],
423
- "limit" => 1,
424
- },
425
- })
426
- expected_plan << gatherer(request, :elements => {
427
- "records" => records_mapper(
428
- :offset => 0,
429
- :limit => 1,
430
- :format => "complex",
431
- :attributes => ["_key", "name", "age"],
432
- ),
433
- })
434
- expected_plan << searcher(request, :output_offset => 0,
435
- :output_limit => 1)
436
- assert_planned(expected_plan, request)
437
- end
438
-
439
- def test_have_sortBy
440
- request = {
441
- "type" => "search",
442
- "dataset" => "Droonga",
443
- "body" => {
444
- "queries" => {
445
- "have_records" => {
446
- "source" => "User",
447
- "sortBy" => {
448
- "keys" => ["name"],
449
- },
450
- "output" => {
451
- "format" => "complex",
452
- "elements" => ["records"],
453
- "attributes" => ["_key", "name", "age"],
454
- "offset" => 0,
455
- "limit" => 1,
456
- },
457
- },
458
- },
459
- },
460
- }
461
-
462
- expected_plan = []
463
- expected_plan << reducer(request, {
464
- "records" => {
465
- "type" => "sort",
466
- "operators" => [
467
- { "column" => 1, "operator" => "<" },
468
- ],
469
- "limit" => 1,
470
- },
471
- })
472
- expected_plan << gatherer(request, :elements => {
473
- "records" => records_mapper(
474
- :offset => 0,
475
- :limit => 1,
476
- :format => "complex",
477
- :attributes => ["_key", "name", "age"],
478
- ),
479
- })
480
- expected_plan << searcher(request, :sort_offset => 0,
481
- :sort_limit => 1,
482
- :output_offset => 0,
483
- :output_limit => 1)
484
- assert_planned(expected_plan, request)
485
- end
486
-
487
- def test_have_sortBy_offset_limit
488
- request = {
489
- "type" => "search",
490
- "dataset" => "Droonga",
491
- "body" => {
492
- "queries" => {
493
- "have_records" => {
494
- "source" => "User",
495
- "sortBy" => {
496
- "keys" => ["name"],
497
- "offset" => 1,
498
- "limit" => 2,
499
- },
500
- "output" => {
501
- "format" => "complex",
502
- "elements" => ["records"],
503
- "attributes" => ["_key", "name", "age"],
504
- "offset" => 4,
505
- "limit" => 8,
506
- },
507
- },
508
- },
509
- },
510
- }
511
-
512
- sort_limit = 1 + 4 + [2, 8].max
513
- output_limit = 1 + 4 + [2, 8].min
514
- expected_plan = []
515
- expected_plan << reducer(request, {
516
- "records" => {
517
- "type" => "sort",
518
- "operators" => [
519
- { "column" => 1, "operator" => "<" },
520
- ],
521
- "limit" => output_limit,
522
- },
523
- })
524
- expected_plan << gatherer(request, :elements => {
525
- "records" => records_mapper(
526
- :offset => 5,
527
- :limit => 2,
528
- :format => "complex",
529
- :attributes => ["_key", "name", "age"],
530
- ),
531
- })
532
- expected_plan << searcher(request, :sort_offset => 0,
533
- :sort_limit => sort_limit,
534
- :output_offset => 0,
535
- :output_limit => output_limit)
536
- assert_planned(expected_plan, request)
537
- end
538
-
539
- def test_have_sortBy_with_infinity_output_limit
540
- request = {
541
- "type" => "search",
542
- "dataset" => "Droonga",
543
- "body" => {
544
- "queries" => {
545
- "have_records" => {
546
- "source" => "User",
547
- "sortBy" => {
548
- "keys" => ["name"],
549
- "offset" => 1,
550
- "limit" => 2,
551
- },
552
- "output" => {
553
- "format" => "complex",
554
- "elements" => ["records"],
555
- "attributes" => ["_key", "name", "age"],
556
- "offset" => 4,
557
- "limit" => -1,
558
- },
559
- },
560
- },
561
- },
562
- }
563
-
564
- limit = 1 + 4 + 2
565
- expected_plan = []
566
- expected_plan << reducer(request, {
567
- "records" => {
568
- "type" => "sort",
569
- "operators" => [
570
- { "column" => 1, "operator" => "<" },
571
- ],
572
- "limit" => limit,
573
- },
574
- })
575
- expected_plan << gatherer(request, :elements => {
576
- "records" => records_mapper(
577
- :offset => 5,
578
- :limit => 2,
579
- :format => "complex",
580
- :attributes => ["_key", "name", "age"],
581
- ),
582
- })
583
- expected_plan << searcher(request, :sort_offset => 0,
584
- :sort_limit => limit,
585
- :output_offset => 0,
586
- :output_limit => limit)
587
- assert_planned(expected_plan, request)
588
- end
589
-
590
- def test_have_sortBy_with_infinity_sort_limit
591
- request = {
592
- "type" => "search",
593
- "dataset" => "Droonga",
594
- "body" => {
595
- "queries" => {
596
- "have_records" => {
597
- "source" => "User",
598
- "sortBy" => {
599
- "keys" => ["name"],
600
- "offset" => 1,
601
- "limit" => -1,
602
- },
603
- "output" => {
604
- "format" => "complex",
605
- "elements" => ["records"],
606
- "attributes" => ["_key", "name", "age"],
607
- "offset" => 4,
608
- "limit" => 8,
609
- },
610
- },
611
- },
612
- },
613
- }
614
-
615
- limit = 1 + 4 + 8
616
- expected_plan = []
617
- expected_plan << reducer(request, {
618
- "records" => {
619
- "type" => "sort",
620
- "operators" => [
621
- { "column" => 1, "operator" => "<" },
622
- ],
623
- "limit" => limit,
624
- },
625
- })
626
- expected_plan << gatherer(request, :elements => {
627
- "records" => records_mapper(
628
- :offset => 5,
629
- :limit => 8,
630
- :format => "complex",
631
- :attributes => ["_key", "name", "age"],
632
- ),
633
- })
634
- expected_plan << searcher(request, :sort_offset => 0,
635
- :sort_limit => limit,
636
- :output_offset => 0,
637
- :output_limit => limit)
638
- assert_planned(expected_plan, request)
639
- end
640
-
641
- def test_have_sortBy_with_infinity_limit
642
- request = {
643
- "type" => "search",
644
- "dataset" => "Droonga",
645
- "body" => {
646
- "queries" => {
647
- "have_records" => {
648
- "source" => "User",
649
- "sortBy" => {
650
- "keys" => ["name"],
651
- "offset" => 1,
652
- "limit" => -1,
653
- },
654
- "output" => {
655
- "format" => "complex",
656
- "elements" => ["records"],
657
- "attributes" => ["_key", "name", "age"],
658
- "offset" => 4,
659
- "limit" => -1,
660
- },
661
- },
662
- },
663
- },
664
- }
665
-
666
- expected_plan = []
667
- expected_plan << reducer(request, {
668
- "records" => {
669
- "type" => "sort",
670
- "operators" => [
671
- { "column" => 1, "operator" => "<" },
672
- ],
673
- "limit" => -1,
674
- },
675
- })
676
- expected_plan << gatherer(request, :elements => {
677
- "records" => records_mapper(
678
- :offset => 5,
679
- :limit => -1,
680
- :format => "complex",
681
- :attributes => ["_key", "name", "age"],
682
- ),
683
- })
684
- expected_plan << searcher(request, :sort_offset => 0,
685
- :sort_limit => -1,
686
- :output_offset => 0,
687
- :output_limit => -1)
688
- assert_planned(expected_plan, request)
689
- end
690
-
691
- def test_have_sortBy_with_multiple_sort_keys
692
- request = {
693
- "type" => "search",
694
- "dataset" => "Droonga",
695
- "body" => {
696
- "queries" => {
697
- "have_records" => {
698
- "source" => "User",
699
- "sortBy" => {
700
- "keys" => ["-age", "name"],
701
- "limit" => -1,
702
- },
703
- "output" => {
704
- "format" => "complex",
705
- "elements" => ["records"],
706
- "attributes" => ["_key", "name", "age"],
707
- "limit" => -1,
708
- },
709
- },
710
- },
711
- },
712
- }
713
-
714
- expected_plan = []
715
- expected_plan << reducer(request, {
716
- "records" => {
717
- "type" => "sort",
718
- "operators" => [
719
- { "column" => 2, "operator" => ">" },
720
- { "column" => 1, "operator" => "<" },
721
- ],
722
- "limit" => -1,
723
- },
724
- })
725
- expected_plan << gatherer(request, :elements => {
726
- "records" => records_mapper(
727
- :offset => 0,
728
- :limit => -1,
729
- :format => "complex",
730
- :attributes => ["_key", "name", "age"],
731
- ),
732
- })
733
- expected_plan << searcher(request, :sort_offset => 0,
734
- :sort_limit => -1,
735
- :output_offset => 0,
736
- :output_limit => -1)
737
- assert_planned(expected_plan, request)
738
- end
739
-
740
- def test_have_sortBy_with_missing_sort_attributes
741
- request = {
742
- "type" => "search",
743
- "dataset" => "Droonga",
744
- "body" => {
745
- "queries" => {
746
- "have_records" => {
747
- "source" => "User",
748
- "sortBy" => {
749
- "keys" => ["-public_age", "public_name"],
750
- "limit" => -1,
751
- },
752
- "output" => {
753
- "format" => "complex",
754
- "elements" => ["records"],
755
- "attributes" => ["_key", "name", "age"],
756
- "limit" => -1,
757
- },
758
- },
759
- },
760
- },
761
- }
762
-
763
- expected_plan = []
764
- expected_plan << reducer(request, {
765
- "records" => {
766
- "type" => "sort",
767
- "operators" => [
768
- { "column" => 3, "operator" => ">" },
769
- { "column" => 4, "operator" => "<" },
770
- ],
771
- "limit" => -1,
772
- },
773
- })
774
- expected_plan << gatherer(request, :elements => {
775
- "records" => records_mapper(
776
- :offset => 0,
777
- :limit => -1,
778
- :format => "complex",
779
- :attributes => ["_key", "name", "age"],
780
- ),
781
- })
782
- expected_plan << searcher(request, :sort_offset => 0,
783
- :sort_limit => -1,
784
- :output_offset => 0,
785
- :output_limit => -1,
786
- :extra_attributes => ["public_age", "public_name"])
787
- assert_planned(expected_plan, request)
788
- end
789
-
790
- def test_hash_attributes
791
- request = {
792
- "type" => "search",
793
- "dataset" => "Droonga",
794
- "body" => {
795
- "queries" => {
796
- "have_records" => {
797
- "source" => "User",
798
- "sortBy" => {
799
- "keys" => ["-public_age", "public_name"],
800
- "limit" => -1,
801
- },
802
- "output" => {
803
- "format" => "complex",
804
- "elements" => ["records"],
805
- "attributes" => {
806
- "id" => "_key",
807
- "name" => { "source" => "name" },
808
- "age" => { "source" => "age" },
809
- },
810
- "limit" => -1,
811
- },
812
- },
813
- },
814
- },
815
- }
816
-
817
- expected_plan = []
818
- expected_plan << reducer(request, {
819
- "records" => {
820
- "type" => "sort",
821
- "operators" => [
822
- { "column" => 3, "operator" => ">" },
823
- { "column" => 4, "operator" => "<" },
824
- ],
825
- "limit" => -1,
826
- },
827
- })
828
- expected_plan << gatherer(request, :elements => {
829
- "records" => records_mapper(
830
- :offset => 0,
831
- :limit => -1,
832
- :format => "complex",
833
- :attributes => ["id", "name", "age"],
834
- ),
835
- })
836
- expected_plan << searcher(request, :sort_offset => 0,
837
- :sort_limit => -1,
838
- :output_offset => 0,
839
- :output_limit => -1,
840
- :extra_attributes => ["public_age", "public_name"])
841
- assert_planned(expected_plan, request)
842
- end
843
-
844
- def test_groupBy
845
- request = {
846
- "type" => "search",
847
- "dataset" => "Droonga",
848
- "body" => {
849
- "queries" => {
850
- "grouped_records" => {
851
- "source" => "User",
852
- "groupBy" => "family_name",
853
- "output" => {
854
- "format" => "complex",
855
- "elements" => ["records"],
856
- "attributes" => ["_key", "_nsubrecs"],
857
- "limit" => -1,
858
- },
859
- },
860
- },
861
- },
862
- }
863
-
864
- expected_plan = []
865
- expected_plan << reducer(request, {
866
- "records" => {
867
- "type" => "sort",
868
- "operators" => [],
869
- "key_column" => 0,
870
- "limit" => -1,
871
- },
872
- })
873
- expected_plan << gatherer(request, :elements => {
874
- "records" => records_mapper(
875
- :offset => 0,
876
- :limit => -1,
877
- :format => "complex",
878
- :attributes => ["_key", "_nsubrecs"],
879
- ),
880
- })
881
- expected_plan << searcher(request, :output_offset => 0,
882
- :output_limit => -1,
883
- :unifiable => true)
884
- assert_planned(expected_plan, request)
885
- end
886
-
887
- def test_groupBy_count
888
- request = {
889
- "type" => "search",
890
- "dataset" => "Droonga",
891
- "body" => {
892
- "queries" => {
893
- "grouped_records" => {
894
- "source" => "User",
895
- "groupBy" => "family_name",
896
- "output" => {
897
- "elements" => ["count"],
898
- },
899
- },
900
- },
901
- },
902
- }
903
-
904
- expected_plan = []
905
- expected_plan << reducer(request, {
906
- "count" => {
907
- "type" => "sum",
908
- },
909
- "records" => {
910
- "type" => "sort",
911
- "operators" => [],
912
- "key_column" => 0,
913
- "limit" => -1,
914
- },
915
- })
916
- expected_plan << gatherer(request, :elements => {
917
- "count" => count_mapper,
918
- "records" => records_mapper(
919
- :limit => -1,
920
- :attributes => ["_key"],
921
- :no_output => true,
922
- ),
923
- })
924
- expected_plan << searcher(request, :output_limit => -1,
925
- :extra_attributes => ["_key"],
926
- :extra_elements => ["records"],
927
- :unifiable => true)
928
- assert_planned(expected_plan, request)
929
- end
930
-
931
- def test_groupBy_hash
932
- request = {
933
- "type" => "search",
934
- "dataset" => "Droonga",
935
- "body" => {
936
- "queries" => {
937
- "grouped_records" => {
938
- "source" => "User",
939
- "groupBy" => {
940
- "key" => "family_name",
941
- "maxNSubRecords" => 3,
942
- },
943
- "output" => {
944
- "format" => "complex",
945
- "elements" => ["records"],
946
- "attributes" => [
947
- { "label" => "family_name", "source" => "_key" },
948
- { "label" => "count", "source" => "_nsubrecs" },
949
- { "label" => "users",
950
- "source" => "_subrecs",
951
- "attributes" => ["name", "age"] },
952
- ],
953
- "limit" => -1,
954
- },
955
- },
956
- },
957
- },
958
- }
959
-
960
- expected_plan = []
961
- expected_plan << reducer(request, {
962
- "records" => {
963
- "type" => "sort",
964
- "operators" => [],
965
- "key_column" => 3, # 0=family_name, 1=_nsubrecs, 2=_subrecs, 3=_keys
966
- "limit" => -1,
967
- },
968
- })
969
- expected_plan << gatherer(request, :elements => {
970
- "records" => records_mapper(
971
- :offset => 0,
972
- :limit => -1,
973
- :format => "complex",
974
- :attributes => ["family_name", "count", "users"],
975
- ),
976
- })
977
- expected_plan << searcher(request, :output_offset => 0,
978
- :output_limit => -1,
979
- :extra_attributes => ["_key"],
980
- :unifiable => true)
981
- assert_planned(expected_plan, request)
982
- end
983
-
984
- private
985
- def reducer(search_request, reducer_body)
986
- queries = search_request["body"]["queries"]
987
- query_name = queries.keys.first
988
-
989
- reducer = {
990
- "type" => "search_reduce",
991
- "body" => {
992
- query_name => {
993
- "#{query_name}_reduced" => reducer_body,
994
- },
995
- },
996
- "inputs" => [query_name],
997
- "outputs" => ["#{query_name}_reduced"],
998
- }
999
-
1000
- reducer
1001
- end
1002
-
1003
- def count_mapper(options={})
1004
- mapper = {
1005
- "type" => "count",
1006
- "target" => "records",
1007
- }
1008
- mapper
1009
- end
1010
-
1011
- def records_mapper(options={})
1012
- mapper = {
1013
- "type" => "sort",
1014
- "offset" => options[:offset] || 0,
1015
- "limit" => options[:limit] || 0,
1016
- "format" => options[:format] || "simple",
1017
- "attributes" => options[:attributes] || [],
1018
- }
1019
- unless options[:no_output].nil?
1020
- mapper["no_output"] = options[:no_output]
1021
- end
1022
- mapper
1023
- end
1024
-
1025
- def gatherer(search_request, options={})
1026
- queries = search_request["body"]["queries"]
1027
- query_name = queries.keys.first
1028
-
1029
- gatherer = {
1030
- "type" => "search_gather",
1031
- "body" => {
1032
- },
1033
- "inputs" => [
1034
- ],
1035
- "post" => true,
1036
- }
1037
-
1038
- unless options[:no_output]
1039
- output = {
1040
- "output" => query_name,
1041
- "elements" => options[:elements] || {},
1042
- }
1043
- gatherer["body"]["#{query_name}_reduced"] = output
1044
- gatherer["inputs"] << "#{query_name}_reduced"
1045
- end
1046
-
1047
- gatherer
1048
- end
1049
-
1050
- def searcher(search_request, options={})
1051
- # dup and clone don't copy it deeply...
1052
- searcher = Marshal.load(Marshal.dump(search_request))
1053
-
1054
- queries = searcher["body"]["queries"]
1055
- query_name = queries.keys.first
1056
- query = queries.values.first
1057
- if options[:extra_attributes]
1058
- attributes = query["output"]["attributes"] || []
1059
- if attributes.is_a?(Hash)
1060
- array_attributes = attributes.collect do |label, attribute|
1061
- case attribute
1062
- when Hash
1063
- attribute["label"] = label
1064
- when String
1065
- attribute = { "label" => label, "source" => attribute }
1066
- end
1067
- attribute
1068
- end
1069
- attributes = array_attributes
1070
- end
1071
- query["output"]["attributes"] = attributes + options[:extra_attributes]
1072
- end
1073
- if options[:extra_elements]
1074
- query["output"]["elements"] += options[:extra_elements]
1075
- end
1076
- if options[:sort_offset]
1077
- query["sortBy"]["offset"] = options[:sort_offset]
1078
- end
1079
- if options[:sort_limit]
1080
- query["sortBy"]["limit"] = options[:sort_limit]
1081
- end
1082
- if options[:output_offset]
1083
- query["output"]["offset"] = options[:output_offset]
1084
- end
1085
- if options[:output_limit]
1086
- query["output"]["limit"] = options[:output_limit]
1087
- end
1088
-
1089
- query["output"]["format"] = "simple" if query["output"]
1090
- query["output"]["unifiable"] = true if options[:unifiable]
1091
-
1092
- outputs = []
1093
- outputs << query_name unless options[:no_output]
1094
-
1095
- searcher["type"] = "broadcast"
1096
- searcher["command"] = "search"
1097
- searcher["outputs"] = outputs
1098
- searcher["replica"] = "random"
1099
- searcher
1100
- end
1101
- end
1102
- end