fluent-plugin-droonga 0.9.0 → 0.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -0
  3. data/Gemfile +8 -1
  4. data/fluent-plugin-droonga.gemspec +2 -2
  5. data/lib/droonga/adapter.rb +39 -0
  6. data/lib/droonga/adapter_runner.rb +99 -0
  7. data/lib/droonga/catalog/base.rb +11 -11
  8. data/lib/droonga/catalog/dataset.rb +54 -0
  9. data/lib/droonga/catalog/version1.rb +1 -1
  10. data/lib/droonga/collector.rb +5 -7
  11. data/lib/droonga/collector_plugin.rb +7 -7
  12. data/lib/droonga/command.rb +36 -0
  13. data/lib/droonga/{plugin/input_adapter/crud.rb → command_repository.rb} +14 -8
  14. data/lib/droonga/dispatcher.rb +86 -54
  15. data/lib/droonga/distributed_command_planner.rb +183 -0
  16. data/lib/droonga/distributor.rb +43 -17
  17. data/lib/droonga/handler.rb +13 -72
  18. data/lib/droonga/handler_message.rb +5 -5
  19. data/lib/droonga/handler_messenger.rb +4 -1
  20. data/lib/droonga/handler_plugin.rb +2 -2
  21. data/lib/droonga/handler_runner.rb +104 -0
  22. data/lib/droonga/input_message.rb +4 -4
  23. data/lib/droonga/legacy_pluggable.rb +66 -0
  24. data/lib/droonga/{input_adapter.rb → legacy_plugin.rb} +27 -22
  25. data/lib/droonga/{plugin_repository.rb → legacy_plugin_repository.rb} +2 -4
  26. data/lib/droonga/message_matcher.rb +101 -0
  27. data/lib/droonga/{input_adapter_plugin.rb → planner.rb} +14 -10
  28. data/lib/droonga/planner_plugin.rb +54 -0
  29. data/lib/droonga/pluggable.rb +9 -45
  30. data/lib/droonga/plugin.rb +9 -33
  31. data/lib/droonga/plugin/collector/basic.rb +2 -0
  32. data/lib/droonga/plugin/collector/search.rb +31 -37
  33. data/lib/droonga/plugin/{handler/groonga/table_remove.rb → metadata/adapter_message.rb} +23 -18
  34. data/lib/droonga/plugin/{handler/search.rb → metadata/handler_action.rb} +19 -15
  35. data/lib/droonga/plugin/metadata/input_message.rb +39 -0
  36. data/lib/droonga/plugin/planner/crud.rb +49 -0
  37. data/lib/droonga/plugin/{distributor → planner}/distributed_search_planner.rb +62 -70
  38. data/lib/droonga/plugin/{distributor → planner}/groonga.rb +11 -32
  39. data/lib/droonga/plugin/{distributor → planner}/search.rb +5 -5
  40. data/lib/droonga/plugin/{distributor → planner}/watch.rb +15 -6
  41. data/lib/droonga/plugin_loader.rb +10 -0
  42. data/lib/droonga/plugin_registerable.rb +34 -10
  43. data/lib/droonga/plugin_registry.rb +58 -0
  44. data/lib/droonga/plugins/crud.rb +124 -0
  45. data/lib/droonga/plugins/error.rb +50 -0
  46. data/lib/droonga/{output_adapter_plugin.rb → plugins/groonga.rb} +9 -13
  47. data/lib/droonga/plugins/groonga/column_create.rb +123 -0
  48. data/lib/droonga/plugins/groonga/generic_command.rb +65 -0
  49. data/lib/droonga/{plugin/output_adapter/groonga.rb → plugins/groonga/generic_response.rb} +16 -15
  50. data/lib/droonga/plugins/groonga/select.rb +124 -0
  51. data/lib/droonga/plugins/groonga/table_create.rb +106 -0
  52. data/lib/droonga/plugins/groonga/table_remove.rb +57 -0
  53. data/lib/droonga/plugins/search.rb +40 -0
  54. data/lib/droonga/plugins/watch.rb +156 -0
  55. data/lib/droonga/processor.rb +8 -10
  56. data/lib/droonga/searcher.rb +14 -4
  57. data/lib/droonga/searcher/mecab_filter.rb +67 -0
  58. data/lib/droonga/session.rb +5 -5
  59. data/lib/droonga/test.rb +1 -1
  60. data/lib/droonga/test/stub_handler_message.rb +1 -1
  61. data/lib/droonga/test/{stub_distributor.rb → stub_planner.rb} +1 -1
  62. data/lib/droonga/worker.rb +7 -8
  63. data/lib/fluent/plugin/out_droonga.rb +0 -1
  64. data/sample/cluster/catalog.json +2 -4
  65. data/sample/mecab_filter/data.grn +7 -0
  66. data/sample/mecab_filter/ddl.grn +7 -0
  67. data/sample/mecab_filter/search_with_mecab_filter.json +21 -0
  68. data/sample/mecab_filter/search_without_mecab_filter.json +21 -0
  69. data/test/command/config/default/catalog.json +2 -5
  70. data/test/command/suite/search/error/no-query.expected +13 -0
  71. data/test/command/suite/search/error/no-query.test +7 -0
  72. data/test/command/suite/search/error/unknown-source.expected +26 -0
  73. data/test/command/suite/watch/subscribe.expected +3 -3
  74. data/test/command/suite/watch/unsubscribe.expected +3 -3
  75. data/test/unit/catalog/test_dataset.rb +385 -0
  76. data/test/unit/catalog/test_version1.rb +111 -45
  77. data/test/unit/fixtures/catalog/version1.json +0 -3
  78. data/test/unit/helper.rb +2 -1
  79. data/test/unit/helper/distributed_search_planner_helper.rb +83 -0
  80. data/test/unit/plugin/collector/test_basic.rb +233 -376
  81. data/test/unit/plugin/collector/test_search.rb +8 -17
  82. data/test/unit/plugin/planner/search_planner/test_basic.rb +120 -0
  83. data/test/unit/plugin/planner/search_planner/test_group_by.rb +573 -0
  84. data/test/unit/plugin/planner/search_planner/test_output.rb +388 -0
  85. data/test/unit/plugin/planner/search_planner/test_sort_by.rb +938 -0
  86. data/test/unit/plugin/{distributor → planner}/test_search.rb +20 -75
  87. data/test/unit/{plugin/handler → plugins/crud}/test_add.rb +11 -11
  88. data/test/unit/plugins/groonga/select/test_adapter_input.rb +213 -0
  89. data/test/unit/{plugin/output_adapter/groonga/test_select.rb → plugins/groonga/select/test_adapter_output.rb} +12 -13
  90. data/test/unit/{plugin/handler → plugins}/groonga/test_column_create.rb +20 -5
  91. data/test/unit/{plugin/handler → plugins}/groonga/test_table_create.rb +5 -0
  92. data/test/unit/{plugin/handler → plugins}/groonga/test_table_remove.rb +8 -1
  93. data/test/unit/{plugin/handler → plugins}/test_groonga.rb +5 -5
  94. data/test/unit/{plugin/handler → plugins}/test_search.rb +21 -5
  95. data/test/unit/{plugin/handler → plugins}/test_watch.rb +29 -10
  96. data/{lib/droonga/command_mapper.rb → test/unit/test_command_repository.rb} +16 -22
  97. data/test/unit/{test_plugin.rb → test_legacy_plugin.rb} +3 -3
  98. data/test/unit/{test_plugin_repository.rb → test_legacy_plugin_repository.rb} +3 -3
  99. data/test/unit/test_message_matcher.rb +137 -0
  100. metadata +86 -66
  101. data/bin/grn2jsons +0 -82
  102. data/lib/droonga/distribution_planner.rb +0 -76
  103. data/lib/droonga/distributor_plugin.rb +0 -95
  104. data/lib/droonga/output_adapter.rb +0 -53
  105. data/lib/droonga/plugin/collector/groonga.rb +0 -83
  106. data/lib/droonga/plugin/distributor/crud.rb +0 -84
  107. data/lib/droonga/plugin/handler/add.rb +0 -109
  108. data/lib/droonga/plugin/handler/forward.rb +0 -75
  109. data/lib/droonga/plugin/handler/groonga.rb +0 -99
  110. data/lib/droonga/plugin/handler/groonga/column_create.rb +0 -106
  111. data/lib/droonga/plugin/handler/groonga/table_create.rb +0 -91
  112. data/lib/droonga/plugin/handler/watch.rb +0 -108
  113. data/lib/droonga/plugin/input_adapter/groonga.rb +0 -49
  114. data/lib/droonga/plugin/input_adapter/groonga/select.rb +0 -63
  115. data/lib/droonga/plugin/output_adapter/crud.rb +0 -51
  116. data/lib/droonga/plugin/output_adapter/groonga/select.rb +0 -54
  117. data/lib/groonga_command_converter.rb +0 -143
  118. data/sample/fluentd.conf +0 -8
  119. data/test/unit/plugin/distributor/test_search_planner.rb +0 -1102
  120. data/test/unit/plugin/input_adapter/groonga/test_select.rb +0 -248
  121. data/test/unit/test_command_mapper.rb +0 -44
  122. data/test/unit/test_groonga_command_converter.rb +0 -242
data/lib/droonga/plugin/output_adapter/groonga/select.rb DELETED
@@ -1,54 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- module Droonga
17
- class GroongaOutputAdapter
18
- class Select
19
- def convert(search_response)
20
- select_responses = search_response.collect do |key, value|
21
- status_code = 0
22
-
23
- start_time = value["startTime"]
24
- start_time_in_unix_time = if start_time
25
- Time.parse(start_time).to_f
26
- else
27
- Time.now.to_f
28
- end
29
- elapsed_time = value["elapsedTime"] || 0
30
- count = value["count"]
31
-
32
- attributes = value["attributes"] || []
33
- converted_attributes = attributes.collect do |attribute|
34
- name = attribute["name"]
35
- type = attribute["type"]
36
- [name, type]
37
- end
38
-
39
- header = [status_code, start_time_in_unix_time, elapsed_time]
40
- records = value["records"]
41
- if records.empty?
42
- results = [[count], converted_attributes]
43
- else
44
- results = [[count], converted_attributes, records]
45
- end
46
- body = [results]
47
-
48
- [header, body]
49
- end
50
- select_responses.first
51
- end
52
- end
53
- end
54
- end
data/lib/groonga_command_converter.rb DELETED
@@ -1,143 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- #
3
- # Copyright (C) 2013 Droonga Project
4
- #
5
- # This library is free software; you can redistribute it and/or
6
- # modify it under the terms of the GNU Lesser General Public
7
- # License version 2.1 as published by the Free Software Foundation.
8
- #
9
- # This library is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
- # Lesser General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU Lesser General Public
15
- # License along with this library; if not, write to the Free Software
16
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
-
18
- require "groonga/command/parser"
19
- require "digest/sha1"
20
- require "time"
21
-
22
- module Droonga
23
- class GroongaCommandConverter
24
- STATUS_OK = 200
25
-
26
- def initialize(options={})
27
- @options = options
28
- @count = 0
29
-
30
- @command_parser = Groonga::Command::Parser.new
31
- end
32
-
33
- def convert(input, &block)
34
- @command_parser.on_command do |command|
35
- case command.name
36
- when "table_create"
37
- yield create_table_create_command(command)
38
- when "table_remove"
39
- yield create_table_remove_command(command)
40
- when "column_create"
41
- yield create_column_create_command(command)
42
- when "select"
43
- yield create_select_command(command)
44
- end
45
- end
46
-
47
- parsed_values = nil
48
- parsed_columns = nil
49
- @command_parser.on_load_start do |command|
50
- parsed_values = []
51
- parsed_columns = nil
52
- end
53
- @command_parser.on_load_columns do |command, columns|
54
- parsed_columns = columns
55
- end
56
- @command_parser.on_load_value do |command, value|
57
- parsed_values << value
58
- end
59
- @command_parser.on_load_complete do |command|
60
- command[:columns] = parsed_columns.join(",")
61
- command[:values] = parsed_values.to_json
62
- split_load_command_to_add_commands(command, &block)
63
- end
64
-
65
- input.each_line do |line|
66
- @command_parser << line
67
- end
68
- @command_parser.finish
69
- end
70
-
71
- private
72
- def create_message(type, body)
73
- id = @options[:id]
74
- if id.nil?
75
- id = new_unique_id
76
- else
77
- id = "#{id}:#{@count}"
78
- @count += 1
79
- end
80
-
81
- {
82
- :id => id,
83
- :date => format_date(@options[:date] || Time.now),
84
- :replyTo => @options[:reply_to],
85
- :statusCode => @options[:status_code] || STATUS_OK,
86
- :dataset => @options[:dataset],
87
- :type => type,
88
- :body => body,
89
- }
90
- end
91
-
92
- def new_unique_id
93
- now = Time.now
94
- now_msec = now.to_i * 1000 + now.usec
95
- random_string = rand(36 ** 16).to_s(36) # Base36
96
- Digest::SHA1.hexdigest("#{now_msec}:#{random_string}")
97
- end
98
-
99
- def format_date(time)
100
- time.iso8601
101
- end
102
-
103
- def create_table_create_command(command)
104
- create_message("table_create", command.arguments)
105
- end
106
-
107
- def create_table_remove_command(command)
108
- create_message("table_remove", command.arguments)
109
- end
110
-
111
- def create_column_create_command(command)
112
- create_message("column_create", command.arguments)
113
- end
114
-
115
- def split_load_command_to_add_commands(command, &block)
116
- columns = command[:columns].split(",")
117
- values = command[:values]
118
- values = JSON.parse(values)
119
- values.each do |record|
120
- body = {
121
- :table => command[:table],
122
- }
123
-
124
- record_values = {}
125
- record.each_with_index do |value, column_index|
126
- column = columns[column_index]
127
- if column == "_key"
128
- body[:key] = value
129
- else
130
- record_values[column.to_sym] = value
131
- end
132
- end
133
- body[:values] = record_values unless record_values.empty?
134
-
135
- yield create_message("add", body)
136
- end
137
- end
138
-
139
- def create_select_command(command)
140
- create_message("select", command.arguments)
141
- end
142
- end
143
- end
data/sample/fluentd.conf DELETED
@@ -1,8 +0,0 @@
1
- <source>
2
- type forward
3
- </source>
4
-
5
- <match droonga.message>
6
- type droonga
7
- handlers search,groonga
8
- </match>
data/test/unit/plugin/distributor/test_search_planner.rb DELETED
@@ -1,1102 +0,0 @@
1
- # Copyright (C) 2013 Droonga Project
2
- #
3
- # This library is free software; you can redistribute it and/or
4
- # modify it under the terms of the GNU Lesser General Public
5
- # License version 2.1 as published by the Free Software Foundation.
6
- #
7
- # This library is distributed in the hope that it will be useful,
8
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
- # Lesser General Public License for more details.
11
- #
12
- # You should have received a copy of the GNU Lesser General Public
13
- # License along with this library; if not, write to the Free Software
14
- # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
-
16
- require "droonga/plugin/distributor/distributed_search_planner"
17
-
18
- class DistributedSearchPlannerTest < Test::Unit::TestCase
19
- def assert_planned(expected, search_request)
20
- plan = Droonga::DistributedSearchPlanner.new(search_request)
21
- actual = plan.messages
22
- assert_equal(expected, actual)
23
- end
24
-
25
- class MultipleQueriesTest < self
26
- def test_distribute
27
- request = {
28
- "type" => "search",
29
- "dataset" => "Droonga",
30
- "body" => {
31
- "queries" => {
32
- "query1" => {
33
- "source" => "User",
34
- "output" => {
35
- "format" => "complex",
36
- "elements" => ["count", "records"],
37
- "attributes" => [],
38
- "offset" => 0,
39
- "limit" => 10,
40
- },
41
- },
42
- "query2" => {
43
- "source" => "User",
44
- "output" => {
45
- "format" => "complex",
46
- "elements" => ["count", "records"],
47
- "attributes" => [],
48
- "offset" => 0,
49
- "limit" => 20,
50
- },
51
- },
52
- "query3" => {
53
- "source" => "User",
54
- "output" => {
55
- "format" => "complex",
56
- "elements" => ["count", "records"],
57
- "attributes" => [],
58
- "offset" => 0,
59
- "limit" => 30,
60
- },
61
- },
62
- },
63
- },
64
- }
65
-
66
- expected_plan = []
67
-
68
- expected_plan << {
69
- "type" => "search_reduce",
70
- "body" => {
71
- "query1" => {
72
- "query1_reduced" => {
73
- "count" => {
74
- "type" => "sum",
75
- },
76
- "records" => {
77
- "type" => "sort",
78
- "operators" => [],
79
- "limit" => 10,
80
- },
81
- },
82
- },
83
- },
84
- "inputs" => ["query1"],
85
- "outputs" => ["query1_reduced"],
86
- }
87
- expected_plan << {
88
- "type" => "search_reduce",
89
- "body" => {
90
- "query2" => {
91
- "query2_reduced" => {
92
- "count" => {
93
- "type" => "sum",
94
- },
95
- "records" => {
96
- "type" => "sort",
97
- "operators" => [],
98
- "limit" => 20,
99
- },
100
- },
101
- },
102
- },
103
- "inputs" => ["query2"],
104
- "outputs" => ["query2_reduced"],
105
- }
106
- expected_plan << {
107
- "type" => "search_reduce",
108
- "body" => {
109
- "query3" => {
110
- "query3_reduced" => {
111
- "count" => {
112
- "type" => "sum",
113
- },
114
- "records" => {
115
- "type" => "sort",
116
- "operators" => [],
117
- "limit" => 30,
118
- },
119
- },
120
- },
121
- },
122
- "inputs" => ["query3"],
123
- "outputs" => ["query3_reduced"],
124
- }
125
-
126
- gatherer = {
127
- "type" => "search_gather",
128
- "body" => {
129
- "query1_reduced" => {
130
- "output" => "query1",
131
- "elements" => {
132
- "records" => {
133
- "type" => "sort",
134
- "offset" => 0,
135
- "limit" => 10,
136
- "format" => "complex",
137
- "attributes" => [],
138
- },
139
- },
140
- },
141
- "query2_reduced" => {
142
- "output" => "query2",
143
- "elements" => {
144
- "records" => {
145
- "type" => "sort",
146
- "offset" => 0,
147
- "limit" => 20,
148
- "format" => "complex",
149
- "attributes" => [],
150
- },
151
- },
152
- },
153
- "query3_reduced" => {
154
- "output" => "query3",
155
- "elements" => {
156
- "records" => {
157
- "type" => "sort",
158
- "offset" => 0,
159
- "limit" => 30,
160
- "format" => "complex",
161
- "attributes" => [],
162
- },
163
- },
164
- },
165
- },
166
- "inputs" => [
167
- "query1_reduced",
168
- "query2_reduced",
169
- "query3_reduced",
170
- ],
171
- "post" => true,
172
- }
173
- expected_plan << gatherer
174
-
175
- searcher = {
176
- "type" => "broadcast",
177
- "command" => "search",
178
- "dataset" => "Droonga",
179
- "body" => {
180
- "queries" => {
181
- "query1" => {
182
- "source" => "User",
183
- "output" => {
184
- "format" => "simple",
185
- "elements" => ["count", "records"],
186
- "attributes" => [],
187
- "offset" => 0,
188
- "limit" => 10,
189
- },
190
- },
191
- "query2" => {
192
- "source" => "User",
193
- "output" => {
194
- "format" => "simple",
195
- "elements" => ["count", "records"],
196
- "attributes" => [],
197
- "offset" => 0,
198
- "limit" => 20,
199
- },
200
- },
201
- "query3" => {
202
- "source" => "User",
203
- "output" => {
204
- "format" => "simple",
205
- "elements" => ["count", "records"],
206
- "attributes" => [],
207
- "offset" => 0,
208
- "limit" => 30,
209
- },
210
- },
211
- },
212
- },
213
- "outputs" => [
214
- "query1",
215
- "query2",
216
- "query3",
217
- ],
218
- "replica" => "random",
219
- }
220
- expected_plan << searcher
221
-
222
- assert_planned(expected_plan, request)
223
- end
224
- end
225
-
226
- class SingleQueryTest < self
227
- def test_no_output
228
- request = {
229
- "type" => "search",
230
- "dataset" => "Droonga",
231
- "body" => {
232
- "queries" => {
233
- "no_output" => {
234
- "source" => "User",
235
- "sortBy" => {
236
- "keys" => ["name"],
237
- "offset" => 0,
238
- "limit" => 1,
239
- },
240
- },
241
- },
242
- },
243
- }
244
-
245
- expected_plan = []
246
- expected_plan << gatherer(request, :no_output => true)
247
- expected_plan << searcher(request, :no_output => true)
248
- assert_planned(expected_plan, request)
249
- end
250
-
251
- def test_no_records_element
252
- request = {
253
- "type" => "search",
254
- "dataset" => "Droonga",
255
- "body" => {
256
- "queries" => {
257
- "no_records" => {
258
- "source" => "User",
259
- "sortBy" => {
260
- "keys" => ["name"],
261
- "offset" => 0,
262
- "limit" => 1,
263
- },
264
- "output" => {
265
- "elements" => ["count"],
266
- },
267
- },
268
- },
269
- },
270
- }
271
-
272
- expected_plan = []
273
- expected_plan << reducer(request, {
274
- "count" => {
275
- "type" => "sum",
276
- },
277
- })
278
- expected_plan << gatherer(request)
279
- expected_plan << searcher(request, :sort_limit => 1,
280
- :output_limit => 0)
281
- assert_planned(expected_plan, request)
282
- end
283
-
284
- def test_no_output_limit
285
- request = {
286
- "type" => "search",
287
- "dataset" => "Droonga",
288
- "body" => {
289
- "queries" => {
290
- "no_limit" => {
291
- "source" => "User",
292
- "output" => {
293
- "format" => "complex",
294
- "elements" => ["count", "records"],
295
- },
296
- },
297
- },
298
- },
299
- }
300
-
301
- expected_plan = []
302
- expected_plan << reducer(request, {
303
- "count" => {
304
- "type" => "sum",
305
- },
306
- })
307
- expected_plan << gatherer(request)
308
- expected_plan << searcher(request, :output_offset => 0,
309
- :output_limit => 0)
310
- assert_planned(expected_plan, request)
311
- end
312
-
313
- def test_have_records
314
- request = {
315
- "type" => "search",
316
- "dataset" => "Droonga",
317
- "body" => {
318
- "queries" => {
319
- "have_records" => {
320
- "source" => "User",
321
- "output" => {
322
- "format" => "complex",
323
- "elements" => ["records"],
324
- "attributes" => ["_key", "name", "age"],
325
- "offset" => 0,
326
- "limit" => 1,
327
- },
328
- },
329
- },
330
- },
331
- }
332
-
333
- expected_plan = []
334
- expected_plan << reducer(request, {
335
- "records" => {
336
- "type" => "sort",
337
- "operators" => [],
338
- "limit" => 1,
339
- },
340
- })
341
- expected_plan << gatherer(request, :elements => {
342
- "records" => records_mapper(
343
- :offset => 0,
344
- :limit => 1,
345
- :format => "complex",
346
- :attributes => ["_key", "name", "age"],
347
- ),
348
- })
349
- expected_plan << searcher(request, :output_offset => 0,
350
- :output_limit => 1)
351
- assert_planned(expected_plan, request)
352
- end
353
-
354
- def test_have_output_offset
355
- request = {
356
- "type" => "search",
357
- "dataset" => "Droonga",
358
- "body" => {
359
- "queries" => {
360
- "have_records" => {
361
- "source" => "User",
362
- "output" => {
363
- "format" => "complex",
364
- "elements" => ["records"],
365
- "attributes" => ["_key", "name", "age"],
366
- "offset" => 1,
367
- "limit" => 1,
368
- },
369
- },
370
- },
371
- },
372
- }
373
-
374
- expected_plan = []
375
- expected_plan << reducer(request, {
376
- "records" => {
377
- "type" => "sort",
378
- "operators" => [],
379
- "limit" => 2,
380
- },
381
- })
382
- expected_plan << gatherer(request, :elements => {
383
- "records" => records_mapper(
384
- :offset => 1,
385
- :limit => 1,
386
- :format => "complex",
387
- :attributes => ["_key", "name", "age"],
388
- ),
389
- })
390
- expected_plan << searcher(request, :output_offset => 0,
391
- :output_limit => 2)
392
- assert_planned(expected_plan, request)
393
- end
394
-
395
- def test_have_simple_sortBy
396
- request = {
397
- "type" => "search",
398
- "dataset" => "Droonga",
399
- "body" => {
400
- "queries" => {
401
- "have_records" => {
402
- "source" => "User",
403
- "sortBy" => ["name"],
404
- "output" => {
405
- "format" => "complex",
406
- "elements" => ["records"],
407
- "attributes" => ["_key", "name", "age"],
408
- "offset" => 0,
409
- "limit" => 1,
410
- },
411
- },
412
- },
413
- },
414
- }
415
-
416
- expected_plan = []
417
- expected_plan << reducer(request, {
418
- "records" => {
419
- "type" => "sort",
420
- "operators" => [
421
- { "column" => 1, "operator" => "<" },
422
- ],
423
- "limit" => 1,
424
- },
425
- })
426
- expected_plan << gatherer(request, :elements => {
427
- "records" => records_mapper(
428
- :offset => 0,
429
- :limit => 1,
430
- :format => "complex",
431
- :attributes => ["_key", "name", "age"],
432
- ),
433
- })
434
- expected_plan << searcher(request, :output_offset => 0,
435
- :output_limit => 1)
436
- assert_planned(expected_plan, request)
437
- end
438
-
439
- def test_have_sortBy
440
- request = {
441
- "type" => "search",
442
- "dataset" => "Droonga",
443
- "body" => {
444
- "queries" => {
445
- "have_records" => {
446
- "source" => "User",
447
- "sortBy" => {
448
- "keys" => ["name"],
449
- },
450
- "output" => {
451
- "format" => "complex",
452
- "elements" => ["records"],
453
- "attributes" => ["_key", "name", "age"],
454
- "offset" => 0,
455
- "limit" => 1,
456
- },
457
- },
458
- },
459
- },
460
- }
461
-
462
- expected_plan = []
463
- expected_plan << reducer(request, {
464
- "records" => {
465
- "type" => "sort",
466
- "operators" => [
467
- { "column" => 1, "operator" => "<" },
468
- ],
469
- "limit" => 1,
470
- },
471
- })
472
- expected_plan << gatherer(request, :elements => {
473
- "records" => records_mapper(
474
- :offset => 0,
475
- :limit => 1,
476
- :format => "complex",
477
- :attributes => ["_key", "name", "age"],
478
- ),
479
- })
480
- expected_plan << searcher(request, :sort_offset => 0,
481
- :sort_limit => 1,
482
- :output_offset => 0,
483
- :output_limit => 1)
484
- assert_planned(expected_plan, request)
485
- end
486
-
487
- def test_have_sortBy_offset_limit
488
- request = {
489
- "type" => "search",
490
- "dataset" => "Droonga",
491
- "body" => {
492
- "queries" => {
493
- "have_records" => {
494
- "source" => "User",
495
- "sortBy" => {
496
- "keys" => ["name"],
497
- "offset" => 1,
498
- "limit" => 2,
499
- },
500
- "output" => {
501
- "format" => "complex",
502
- "elements" => ["records"],
503
- "attributes" => ["_key", "name", "age"],
504
- "offset" => 4,
505
- "limit" => 8,
506
- },
507
- },
508
- },
509
- },
510
- }
511
-
512
- sort_limit = 1 + 4 + [2, 8].max
513
- output_limit = 1 + 4 + [2, 8].min
514
- expected_plan = []
515
- expected_plan << reducer(request, {
516
- "records" => {
517
- "type" => "sort",
518
- "operators" => [
519
- { "column" => 1, "operator" => "<" },
520
- ],
521
- "limit" => output_limit,
522
- },
523
- })
524
- expected_plan << gatherer(request, :elements => {
525
- "records" => records_mapper(
526
- :offset => 5,
527
- :limit => 2,
528
- :format => "complex",
529
- :attributes => ["_key", "name", "age"],
530
- ),
531
- })
532
- expected_plan << searcher(request, :sort_offset => 0,
533
- :sort_limit => sort_limit,
534
- :output_offset => 0,
535
- :output_limit => output_limit)
536
- assert_planned(expected_plan, request)
537
- end
538
-
539
- def test_have_sortBy_with_infinity_output_limit
540
- request = {
541
- "type" => "search",
542
- "dataset" => "Droonga",
543
- "body" => {
544
- "queries" => {
545
- "have_records" => {
546
- "source" => "User",
547
- "sortBy" => {
548
- "keys" => ["name"],
549
- "offset" => 1,
550
- "limit" => 2,
551
- },
552
- "output" => {
553
- "format" => "complex",
554
- "elements" => ["records"],
555
- "attributes" => ["_key", "name", "age"],
556
- "offset" => 4,
557
- "limit" => -1,
558
- },
559
- },
560
- },
561
- },
562
- }
563
-
564
- limit = 1 + 4 + 2
565
- expected_plan = []
566
- expected_plan << reducer(request, {
567
- "records" => {
568
- "type" => "sort",
569
- "operators" => [
570
- { "column" => 1, "operator" => "<" },
571
- ],
572
- "limit" => limit,
573
- },
574
- })
575
- expected_plan << gatherer(request, :elements => {
576
- "records" => records_mapper(
577
- :offset => 5,
578
- :limit => 2,
579
- :format => "complex",
580
- :attributes => ["_key", "name", "age"],
581
- ),
582
- })
583
- expected_plan << searcher(request, :sort_offset => 0,
584
- :sort_limit => limit,
585
- :output_offset => 0,
586
- :output_limit => limit)
587
- assert_planned(expected_plan, request)
588
- end
589
-
590
- def test_have_sortBy_with_infinity_sort_limit
591
- request = {
592
- "type" => "search",
593
- "dataset" => "Droonga",
594
- "body" => {
595
- "queries" => {
596
- "have_records" => {
597
- "source" => "User",
598
- "sortBy" => {
599
- "keys" => ["name"],
600
- "offset" => 1,
601
- "limit" => -1,
602
- },
603
- "output" => {
604
- "format" => "complex",
605
- "elements" => ["records"],
606
- "attributes" => ["_key", "name", "age"],
607
- "offset" => 4,
608
- "limit" => 8,
609
- },
610
- },
611
- },
612
- },
613
- }
614
-
615
- limit = 1 + 4 + 8
616
- expected_plan = []
617
- expected_plan << reducer(request, {
618
- "records" => {
619
- "type" => "sort",
620
- "operators" => [
621
- { "column" => 1, "operator" => "<" },
622
- ],
623
- "limit" => limit,
624
- },
625
- })
626
- expected_plan << gatherer(request, :elements => {
627
- "records" => records_mapper(
628
- :offset => 5,
629
- :limit => 8,
630
- :format => "complex",
631
- :attributes => ["_key", "name", "age"],
632
- ),
633
- })
634
- expected_plan << searcher(request, :sort_offset => 0,
635
- :sort_limit => limit,
636
- :output_offset => 0,
637
- :output_limit => limit)
638
- assert_planned(expected_plan, request)
639
- end
640
-
641
- def test_have_sortBy_with_infinity_limit
642
- request = {
643
- "type" => "search",
644
- "dataset" => "Droonga",
645
- "body" => {
646
- "queries" => {
647
- "have_records" => {
648
- "source" => "User",
649
- "sortBy" => {
650
- "keys" => ["name"],
651
- "offset" => 1,
652
- "limit" => -1,
653
- },
654
- "output" => {
655
- "format" => "complex",
656
- "elements" => ["records"],
657
- "attributes" => ["_key", "name", "age"],
658
- "offset" => 4,
659
- "limit" => -1,
660
- },
661
- },
662
- },
663
- },
664
- }
665
-
666
- expected_plan = []
667
- expected_plan << reducer(request, {
668
- "records" => {
669
- "type" => "sort",
670
- "operators" => [
671
- { "column" => 1, "operator" => "<" },
672
- ],
673
- "limit" => -1,
674
- },
675
- })
676
- expected_plan << gatherer(request, :elements => {
677
- "records" => records_mapper(
678
- :offset => 5,
679
- :limit => -1,
680
- :format => "complex",
681
- :attributes => ["_key", "name", "age"],
682
- ),
683
- })
684
- expected_plan << searcher(request, :sort_offset => 0,
685
- :sort_limit => -1,
686
- :output_offset => 0,
687
- :output_limit => -1)
688
- assert_planned(expected_plan, request)
689
- end
690
-
691
- def test_have_sortBy_with_multiple_sort_keys
692
- request = {
693
- "type" => "search",
694
- "dataset" => "Droonga",
695
- "body" => {
696
- "queries" => {
697
- "have_records" => {
698
- "source" => "User",
699
- "sortBy" => {
700
- "keys" => ["-age", "name"],
701
- "limit" => -1,
702
- },
703
- "output" => {
704
- "format" => "complex",
705
- "elements" => ["records"],
706
- "attributes" => ["_key", "name", "age"],
707
- "limit" => -1,
708
- },
709
- },
710
- },
711
- },
712
- }
713
-
714
- expected_plan = []
715
- expected_plan << reducer(request, {
716
- "records" => {
717
- "type" => "sort",
718
- "operators" => [
719
- { "column" => 2, "operator" => ">" },
720
- { "column" => 1, "operator" => "<" },
721
- ],
722
- "limit" => -1,
723
- },
724
- })
725
- expected_plan << gatherer(request, :elements => {
726
- "records" => records_mapper(
727
- :offset => 0,
728
- :limit => -1,
729
- :format => "complex",
730
- :attributes => ["_key", "name", "age"],
731
- ),
732
- })
733
- expected_plan << searcher(request, :sort_offset => 0,
734
- :sort_limit => -1,
735
- :output_offset => 0,
736
- :output_limit => -1)
737
- assert_planned(expected_plan, request)
738
- end
739
-
740
- def test_have_sortBy_with_missing_sort_attributes
741
- request = {
742
- "type" => "search",
743
- "dataset" => "Droonga",
744
- "body" => {
745
- "queries" => {
746
- "have_records" => {
747
- "source" => "User",
748
- "sortBy" => {
749
- "keys" => ["-public_age", "public_name"],
750
- "limit" => -1,
751
- },
752
- "output" => {
753
- "format" => "complex",
754
- "elements" => ["records"],
755
- "attributes" => ["_key", "name", "age"],
756
- "limit" => -1,
757
- },
758
- },
759
- },
760
- },
761
- }
762
-
763
- expected_plan = []
764
- expected_plan << reducer(request, {
765
- "records" => {
766
- "type" => "sort",
767
- "operators" => [
768
- { "column" => 3, "operator" => ">" },
769
- { "column" => 4, "operator" => "<" },
770
- ],
771
- "limit" => -1,
772
- },
773
- })
774
- expected_plan << gatherer(request, :elements => {
775
- "records" => records_mapper(
776
- :offset => 0,
777
- :limit => -1,
778
- :format => "complex",
779
- :attributes => ["_key", "name", "age"],
780
- ),
781
- })
782
- expected_plan << searcher(request, :sort_offset => 0,
783
- :sort_limit => -1,
784
- :output_offset => 0,
785
- :output_limit => -1,
786
- :extra_attributes => ["public_age", "public_name"])
787
- assert_planned(expected_plan, request)
788
- end
789
-
790
- def test_hash_attributes
791
- request = {
792
- "type" => "search",
793
- "dataset" => "Droonga",
794
- "body" => {
795
- "queries" => {
796
- "have_records" => {
797
- "source" => "User",
798
- "sortBy" => {
799
- "keys" => ["-public_age", "public_name"],
800
- "limit" => -1,
801
- },
802
- "output" => {
803
- "format" => "complex",
804
- "elements" => ["records"],
805
- "attributes" => {
806
- "id" => "_key",
807
- "name" => { "source" => "name" },
808
- "age" => { "source" => "age" },
809
- },
810
- "limit" => -1,
811
- },
812
- },
813
- },
814
- },
815
- }
816
-
817
- expected_plan = []
818
- expected_plan << reducer(request, {
819
- "records" => {
820
- "type" => "sort",
821
- "operators" => [
822
- { "column" => 3, "operator" => ">" },
823
- { "column" => 4, "operator" => "<" },
824
- ],
825
- "limit" => -1,
826
- },
827
- })
828
- expected_plan << gatherer(request, :elements => {
829
- "records" => records_mapper(
830
- :offset => 0,
831
- :limit => -1,
832
- :format => "complex",
833
- :attributes => ["id", "name", "age"],
834
- ),
835
- })
836
- expected_plan << searcher(request, :sort_offset => 0,
837
- :sort_limit => -1,
838
- :output_offset => 0,
839
- :output_limit => -1,
840
- :extra_attributes => ["public_age", "public_name"])
841
- assert_planned(expected_plan, request)
842
- end
843
-
844
- def test_groupBy
845
- request = {
846
- "type" => "search",
847
- "dataset" => "Droonga",
848
- "body" => {
849
- "queries" => {
850
- "grouped_records" => {
851
- "source" => "User",
852
- "groupBy" => "family_name",
853
- "output" => {
854
- "format" => "complex",
855
- "elements" => ["records"],
856
- "attributes" => ["_key", "_nsubrecs"],
857
- "limit" => -1,
858
- },
859
- },
860
- },
861
- },
862
- }
863
-
864
- expected_plan = []
865
- expected_plan << reducer(request, {
866
- "records" => {
867
- "type" => "sort",
868
- "operators" => [],
869
- "key_column" => 0,
870
- "limit" => -1,
871
- },
872
- })
873
- expected_plan << gatherer(request, :elements => {
874
- "records" => records_mapper(
875
- :offset => 0,
876
- :limit => -1,
877
- :format => "complex",
878
- :attributes => ["_key", "_nsubrecs"],
879
- ),
880
- })
881
- expected_plan << searcher(request, :output_offset => 0,
882
- :output_limit => -1,
883
- :unifiable => true)
884
- assert_planned(expected_plan, request)
885
- end
886
-
887
- def test_groupBy_count
888
- request = {
889
- "type" => "search",
890
- "dataset" => "Droonga",
891
- "body" => {
892
- "queries" => {
893
- "grouped_records" => {
894
- "source" => "User",
895
- "groupBy" => "family_name",
896
- "output" => {
897
- "elements" => ["count"],
898
- },
899
- },
900
- },
901
- },
902
- }
903
-
904
- expected_plan = []
905
- expected_plan << reducer(request, {
906
- "count" => {
907
- "type" => "sum",
908
- },
909
- "records" => {
910
- "type" => "sort",
911
- "operators" => [],
912
- "key_column" => 0,
913
- "limit" => -1,
914
- },
915
- })
916
- expected_plan << gatherer(request, :elements => {
917
- "count" => count_mapper,
918
- "records" => records_mapper(
919
- :limit => -1,
920
- :attributes => ["_key"],
921
- :no_output => true,
922
- ),
923
- })
924
- expected_plan << searcher(request, :output_limit => -1,
925
- :extra_attributes => ["_key"],
926
- :extra_elements => ["records"],
927
- :unifiable => true)
928
- assert_planned(expected_plan, request)
929
- end
930
-
931
- def test_groupBy_hash
932
- request = {
933
- "type" => "search",
934
- "dataset" => "Droonga",
935
- "body" => {
936
- "queries" => {
937
- "grouped_records" => {
938
- "source" => "User",
939
- "groupBy" => {
940
- "key" => "family_name",
941
- "maxNSubRecords" => 3,
942
- },
943
- "output" => {
944
- "format" => "complex",
945
- "elements" => ["records"],
946
- "attributes" => [
947
- { "label" => "family_name", "source" => "_key" },
948
- { "label" => "count", "source" => "_nsubrecs" },
949
- { "label" => "users",
950
- "source" => "_subrecs",
951
- "attributes" => ["name", "age"] },
952
- ],
953
- "limit" => -1,
954
- },
955
- },
956
- },
957
- },
958
- }
959
-
960
- expected_plan = []
961
- expected_plan << reducer(request, {
962
- "records" => {
963
- "type" => "sort",
964
- "operators" => [],
965
- "key_column" => 3, # 0=family_name, 1=_nsubrecs, 2=_subrecs, 3=_keys
966
- "limit" => -1,
967
- },
968
- })
969
- expected_plan << gatherer(request, :elements => {
970
- "records" => records_mapper(
971
- :offset => 0,
972
- :limit => -1,
973
- :format => "complex",
974
- :attributes => ["family_name", "count", "users"],
975
- ),
976
- })
977
- expected_plan << searcher(request, :output_offset => 0,
978
- :output_limit => -1,
979
- :extra_attributes => ["_key"],
980
- :unifiable => true)
981
- assert_planned(expected_plan, request)
982
- end
983
-
984
- private
985
- def reducer(search_request, reducer_body)
986
- queries = search_request["body"]["queries"]
987
- query_name = queries.keys.first
988
-
989
- reducer = {
990
- "type" => "search_reduce",
991
- "body" => {
992
- query_name => {
993
- "#{query_name}_reduced" => reducer_body,
994
- },
995
- },
996
- "inputs" => [query_name],
997
- "outputs" => ["#{query_name}_reduced"],
998
- }
999
-
1000
- reducer
1001
- end
1002
-
1003
- def count_mapper(options={})
1004
- mapper = {
1005
- "type" => "count",
1006
- "target" => "records",
1007
- }
1008
- mapper
1009
- end
1010
-
1011
- def records_mapper(options={})
1012
- mapper = {
1013
- "type" => "sort",
1014
- "offset" => options[:offset] || 0,
1015
- "limit" => options[:limit] || 0,
1016
- "format" => options[:format] || "simple",
1017
- "attributes" => options[:attributes] || [],
1018
- }
1019
- unless options[:no_output].nil?
1020
- mapper["no_output"] = options[:no_output]
1021
- end
1022
- mapper
1023
- end
1024
-
1025
- def gatherer(search_request, options={})
1026
- queries = search_request["body"]["queries"]
1027
- query_name = queries.keys.first
1028
-
1029
- gatherer = {
1030
- "type" => "search_gather",
1031
- "body" => {
1032
- },
1033
- "inputs" => [
1034
- ],
1035
- "post" => true,
1036
- }
1037
-
1038
- unless options[:no_output]
1039
- output = {
1040
- "output" => query_name,
1041
- "elements" => options[:elements] || {},
1042
- }
1043
- gatherer["body"]["#{query_name}_reduced"] = output
1044
- gatherer["inputs"] << "#{query_name}_reduced"
1045
- end
1046
-
1047
- gatherer
1048
- end
1049
-
1050
- def searcher(search_request, options={})
1051
- # dup and clone don't copy it deeply...
1052
- searcher = Marshal.load(Marshal.dump(search_request))
1053
-
1054
- queries = searcher["body"]["queries"]
1055
- query_name = queries.keys.first
1056
- query = queries.values.first
1057
- if options[:extra_attributes]
1058
- attributes = query["output"]["attributes"] || []
1059
- if attributes.is_a?(Hash)
1060
- array_attributes = attributes.collect do |label, attribute|
1061
- case attribute
1062
- when Hash
1063
- attribute["label"] = label
1064
- when String
1065
- attribute = { "label" => label, "source" => attribute }
1066
- end
1067
- attribute
1068
- end
1069
- attributes = array_attributes
1070
- end
1071
- query["output"]["attributes"] = attributes + options[:extra_attributes]
1072
- end
1073
- if options[:extra_elements]
1074
- query["output"]["elements"] += options[:extra_elements]
1075
- end
1076
- if options[:sort_offset]
1077
- query["sortBy"]["offset"] = options[:sort_offset]
1078
- end
1079
- if options[:sort_limit]
1080
- query["sortBy"]["limit"] = options[:sort_limit]
1081
- end
1082
- if options[:output_offset]
1083
- query["output"]["offset"] = options[:output_offset]
1084
- end
1085
- if options[:output_limit]
1086
- query["output"]["limit"] = options[:output_limit]
1087
- end
1088
-
1089
- query["output"]["format"] = "simple" if query["output"]
1090
- query["output"]["unifiable"] = true if options[:unifiable]
1091
-
1092
- outputs = []
1093
- outputs << query_name unless options[:no_output]
1094
-
1095
- searcher["type"] = "broadcast"
1096
- searcher["command"] = "search"
1097
- searcher["outputs"] = outputs
1098
- searcher["replica"] = "random"
1099
- searcher
1100
- end
1101
- end
1102
- end