fluent-plugin-droonga 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +40 -0
  5. data/LICENSE.txt +14 -0
  6. data/README.md +18 -0
  7. data/Rakefile +25 -0
  8. data/benchmark/benchmark.rb +123 -0
  9. data/benchmark/utils.rb +243 -0
  10. data/benchmark/watch/benchmark-notify.rb +143 -0
  11. data/benchmark/watch/benchmark-notify.sh +19 -0
  12. data/benchmark/watch/benchmark-publish.rb +120 -0
  13. data/benchmark/watch/benchmark-scan.rb +210 -0
  14. data/benchmark/watch/catalog.json +32 -0
  15. data/benchmark/watch/fluentd.conf +12 -0
  16. data/bin/grn2jsons +85 -0
  17. data/fluent-plugin-droonga.gemspec +41 -0
  18. data/lib/droonga/adapter.rb +156 -0
  19. data/lib/droonga/catalog.rb +153 -0
  20. data/lib/droonga/command_mapper.rb +45 -0
  21. data/lib/droonga/engine.rb +83 -0
  22. data/lib/droonga/executor.rb +289 -0
  23. data/lib/droonga/handler.rb +140 -0
  24. data/lib/droonga/handler_plugin.rb +35 -0
  25. data/lib/droonga/job_queue.rb +83 -0
  26. data/lib/droonga/job_queue_schema.rb +65 -0
  27. data/lib/droonga/logger.rb +34 -0
  28. data/lib/droonga/plugin.rb +41 -0
  29. data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
  30. data/lib/droonga/plugin/adapter_groonga.rb +40 -0
  31. data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
  32. data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
  33. data/lib/droonga/plugin/handler_add.rb +44 -0
  34. data/lib/droonga/plugin/handler_forward.rb +70 -0
  35. data/lib/droonga/plugin/handler_groonga.rb +52 -0
  36. data/lib/droonga/plugin/handler_proxy.rb +82 -0
  37. data/lib/droonga/plugin/handler_search.rb +33 -0
  38. data/lib/droonga/plugin/handler_watch.rb +102 -0
  39. data/lib/droonga/proxy.rb +371 -0
  40. data/lib/droonga/searcher.rb +415 -0
  41. data/lib/droonga/server.rb +112 -0
  42. data/lib/droonga/sweeper.rb +42 -0
  43. data/lib/droonga/watch_schema.rb +88 -0
  44. data/lib/droonga/watcher.rb +256 -0
  45. data/lib/droonga/worker.rb +51 -0
  46. data/lib/fluent/plugin/out_droonga.rb +56 -0
  47. data/lib/groonga_command_converter.rb +137 -0
  48. data/sample/cluster/catalog.json +43 -0
  49. data/sample/cluster/fluentd.conf +12 -0
  50. data/sample/fluentd.conf +8 -0
  51. data/test/fixtures/catalog.json +43 -0
  52. data/test/fixtures/document.grn +23 -0
  53. data/test/helper.rb +24 -0
  54. data/test/helper/fixture.rb +28 -0
  55. data/test/helper/sandbox.rb +73 -0
  56. data/test/helper/stub_worker.rb +27 -0
  57. data/test/helper/watch_helper.rb +35 -0
  58. data/test/plugin/adapter/groonga/test_select.rb +176 -0
  59. data/test/plugin/handler/groonga/test_column_create.rb +127 -0
  60. data/test/plugin/handler/groonga/test_table_create.rb +140 -0
  61. data/test/plugin/handler/test_handler_add.rb +135 -0
  62. data/test/plugin/handler/test_handler_groonga.rb +64 -0
  63. data/test/plugin/handler/test_handler_search.rb +512 -0
  64. data/test/plugin/handler/test_handler_watch.rb +168 -0
  65. data/test/run-test.rb +55 -0
  66. data/test/test_adapter.rb +48 -0
  67. data/test/test_catalog.rb +59 -0
  68. data/test/test_command_mapper.rb +44 -0
  69. data/test/test_groonga_command_converter.rb +242 -0
  70. data/test/test_handler.rb +53 -0
  71. data/test/test_job_queue_schema.rb +45 -0
  72. data/test/test_output.rb +99 -0
  73. data/test/test_sweeper.rb +95 -0
  74. data/test/test_watch_schema.rb +57 -0
  75. data/test/test_watcher.rb +336 -0
  76. data/test/test_worker.rb +144 -0
  77. metadata +299 -0
@@ -0,0 +1,371 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require 'tsort'
19
+ require "droonga/handler"
20
+ require "droonga/adapter"
21
+ require "droonga/catalog"
22
+
23
+ module Droonga
24
+ class Proxy
25
+ attr_reader :collectors
26
+ def initialize(worker, name)
27
+ @engines = {}
28
+ Droonga::catalog.get_engines(name).each do |name, options|
29
+ engine = Droonga::Engine.new(options.merge(:proxy => false,
30
+ :with_server => false))
31
+ engine.start
32
+ @engines[name] = engine
33
+ end
34
+ @worker = worker
35
+ @name = name
36
+ @collectors = {}
37
+ @current_id = 0
38
+ @local = Regexp.new("^#{@name}")
39
+ plugins = ["proxy"] + (Droonga::catalog.option("plugins")||[]) + ["adapter"]
40
+ plugins.each do |plugin|
41
+ @worker.add_handler(plugin)
42
+ end
43
+ end
44
+
45
+ def shutdown
46
+ @engines.each do |name, engine|
47
+ engine.shutdown
48
+ end
49
+ end
50
+
51
+ def handle(message, arguments)
52
+ case message
53
+ when Array
54
+ handle_incoming_message(message)
55
+ when Hash
56
+ handle_internal_message(message)
57
+ end
58
+ end
59
+
60
+ def handle_incoming_message(message)
61
+ id = generate_id
62
+ planner = Planner.new(self, message)
63
+ destinations = planner.resolve(id)
64
+ components = planner.components
65
+ message = { "id" => id, "components" => components }
66
+ destinations.each do |destination, frequency|
67
+ dispatch(message, destination)
68
+ end
69
+ end
70
+
71
+ def handle_internal_message(message)
72
+ id = message["id"]
73
+ collector = @collectors[id]
74
+ unless collector
75
+ components = message["components"]
76
+ if components
77
+ planner = Planner.new(self, components)
78
+ collector = planner.get_collector(id)
79
+ else
80
+ #todo: take cases receiving result before its query into account
81
+ end
82
+ end
83
+ collector.handle(message["input"], message["value"])
84
+ end
85
+
86
+ def dispatch(message, destination)
87
+ if local?(destination)
88
+ handle_internal_message(message)
89
+ else
90
+ post(message, "to"=>farm_path(destination), "type"=>"proxy")
91
+ end
92
+ end
93
+
94
+ def deliver(id, route, message, type, synchronous)
95
+ if id == route
96
+ post(message, "type" => type, "synchronous"=> synchronous)
97
+ else
98
+ envelope = @worker.envelope.merge("body" => message, "type" => type)
99
+ @engines[route].emit('', Time.now.to_f, envelope, synchronous)
100
+ end
101
+ end
102
+
103
+ def post(message, destination)
104
+ @worker.post(message, destination)
105
+ end
106
+
107
+ def generate_id
108
+ id = @current_id
109
+ @current_id = id.succ
110
+ return [@name, id].join('.#')
111
+ end
112
+
113
+ def farm_path(route)
114
+ if route =~ /\A.*:\d+\/[^\.]+/
115
+ $&
116
+ else
117
+ route
118
+ end
119
+ end
120
+
121
+ def local?(route)
122
+ route =~ @local
123
+ end
124
+
125
+ class Planner
126
+ attr_reader :components
127
+ class UndefinedInputError < StandardError
128
+ attr_reader :input
129
+ def initialize(input)
130
+ @input = input
131
+ super("undefined input assigned: <#{input}>")
132
+ end
133
+ end
134
+
135
+ class CyclicComponentsError < StandardError
136
+ attr_reader :components
137
+ def initialize(components)
138
+ @components = components
139
+ super("cyclic components found: <#{components}>")
140
+ end
141
+ end
142
+
143
+ include TSort
144
+ def initialize(proxy, components)
145
+ @proxy = proxy
146
+ @components = components
147
+ end
148
+
149
+ def resolve(id)
150
+ @dependency = {}
151
+ @components.each do |component|
152
+ @dependency[component] = component["inputs"]
153
+ next unless component["outputs"]
154
+ component["outputs"].each do |output|
155
+ @dependency[output] = [component]
156
+ end
157
+ end
158
+ @components = []
159
+ each_strongly_connected_component do |cs|
160
+ raise CyclicComponentsError.new(cs) if cs.size > 1
161
+ @components.concat(cs) unless cs.first.is_a? String
162
+ end
163
+ resolve_routes(id)
164
+ end
165
+
166
+ def resolve_routes(id)
167
+ local = [id]
168
+ destinations = Hash.new(0)
169
+ @components.each do |component|
170
+ dataset = component["dataset"]
171
+ routes =
172
+ if dataset
173
+ Droonga::catalog.get_routes(dataset, component)
174
+ else
175
+ local
176
+ end
177
+ routes.each do |route|
178
+ destinations[@proxy.farm_path(route)] += 1
179
+ end
180
+ component["routes"] = routes
181
+ end
182
+ return destinations
183
+ end
184
+
185
+ def get_collector(id)
186
+ resolve_descendants
187
+ tasks = []
188
+ inputs = {}
189
+ @components.each do |component|
190
+ component["routes"].each do |route|
191
+ next unless @proxy.local?(route)
192
+ task = {
193
+ "route" => route,
194
+ "component" => component,
195
+ "n_of_inputs" => 0,
196
+ "values" => {}
197
+ }
198
+ tasks << task
199
+ (component["inputs"] || [nil]).each do |input|
200
+ inputs[input] ||= []
201
+ inputs[input] << task
202
+ end
203
+ end
204
+ end
205
+ collector = Collector.new(id, @proxy, @components, tasks, inputs)
206
+ @proxy.collectors[id] = collector
207
+ return collector
208
+ end
209
+
210
+ def resolve_descendants
211
+ @descendants = {}
212
+ @components.size.times do |index|
213
+ component = @components[index]
214
+ (component["inputs"] || []).each do |input|
215
+ @descendants[input] ||= []
216
+ @descendants[input] << index
217
+ end
218
+ component["n_of_expects"] = 0
219
+ end
220
+ @components.each do |component|
221
+ descendants = get_descendants(component)
222
+ component["descendants"] = descendants
223
+ descendants.each do |key, indices|
224
+ indices.each do |index|
225
+ @components[index]["n_of_expects"] += component["routes"].size
226
+ end
227
+ end
228
+ end
229
+ end
230
+
231
+ def get_descendants(component)
232
+ return {} unless component["outputs"]
233
+ descendants = {}
234
+ component["outputs"].each do |output|
235
+ descendants[output] = @descendants[output]
236
+ end
237
+ descendants
238
+ end
239
+
240
+ def tsort_each_node(&block)
241
+ @dependency.each_key(&block)
242
+ end
243
+
244
+ def tsort_each_child(node, &block)
245
+ if node.is_a? String and @dependency[node].nil?
246
+ raise UndefinedInputError.new(node)
247
+ end
248
+ if @dependency[node]
249
+ @dependency[node].each(&block)
250
+ end
251
+ end
252
+ end
253
+
254
+ class Collector
255
+ def initialize(id, proxy, components, tasks, inputs)
256
+ @id = id
257
+ @proxy = proxy
258
+ @components = components
259
+ @tasks = tasks
260
+ @n_dones = 0
261
+ @inputs = inputs
262
+ end
263
+
264
+ def handle(name, value)
265
+ tasks = @inputs[name]
266
+ unless tasks
267
+ #TODO: result arrived before its query
268
+ return
269
+ end
270
+ tasks.each do |task|
271
+ task["n_of_inputs"] += 1 if name
272
+ component = task["component"]
273
+ type = component["type"]
274
+ command = component["command"] || ("proxy_" + type)
275
+ n_of_expects = component["n_of_expects"]
276
+ synchronous = nil
277
+ if command
278
+ # TODO: should be controllable for each command respectively.
279
+ synchronous = !n_of_expects.zero?
280
+ # TODO: check if asynchronous execution is available.
281
+ message = {
282
+ "task"=>task,
283
+ "name"=>name,
284
+ "value"=>value
285
+ }
286
+ unless synchronous
287
+ descendants = {}
288
+ component["descendants"].each do |name, indices|
289
+ descendants[name] = indices.collect do |index|
290
+ @components[index]["routes"].map do |route|
291
+ @proxy.farm_path(route)
292
+ end
293
+ end
294
+ end
295
+ message["descendants"] = descendants
296
+ message["id"] = @id
297
+ end
298
+ @proxy.deliver(@id, task["route"], message, command, synchronous)
299
+ end
300
+ return if task["n_of_inputs"] < n_of_expects
301
+ #the task is done
302
+ if synchronous
303
+ result = task["values"]
304
+ post = component["post"]
305
+ @proxy.post(result, post) if post
306
+ component["descendants"].each do |name, indices|
307
+ message = {
308
+ "id" => @id,
309
+ "input" => name,
310
+ "value" => result[name]
311
+ }
312
+ indices.each do |index|
313
+ @components[index]["routes"].each do |route|
314
+ @proxy.dispatch(message, route)
315
+ end
316
+ end
317
+ end
318
+ end
319
+ @n_dones += 1
320
+ @proxy.collectors.delete(@id) if @n_dones == @tasks.size
321
+ end
322
+ end
323
+ end
324
+ end
325
+
326
+ class ProxyMessageHandler < Droonga::Handler
327
+ Droonga::HandlerPlugin.register("proxy_message", self)
328
+ def initialize(*arguments)
329
+ super
330
+ @proxy = Droonga::Proxy.new(@worker, @worker.name)
331
+ end
332
+
333
+ def shutdown
334
+ @proxy.shutdown
335
+ end
336
+
337
+ command :proxy
338
+ def proxy(request, *arguments)
339
+ @proxy.handle(request, arguments)
340
+ end
341
+
342
+ def prefer_synchronous?(command)
343
+ return true
344
+ end
345
+ end
346
+
347
+ class ProxyHandler < Droonga::Handler
348
+ attr_reader :task, :input_name, :component, :output_values, :body, :output_names
349
+ def handle(command, request, *arguments)
350
+ return false unless request.is_a? Hash
351
+ @task = request["task"]
352
+ return false unless @task.is_a? Hash
353
+ @component = @task["component"]
354
+ return false unless @component.is_a? Hash
355
+ @output_values = @task["values"]
356
+ @body = @component["body"]
357
+ @output_names = @component["outputs"]
358
+ @id = request["id"]
359
+ @value = request["value"]
360
+ @input_name = request["name"]
361
+ @descendants = request["descendants"]
362
+ invoke(command, @value, *arguments)
363
+ output if @descendants
364
+ true
365
+ end
366
+
367
+ def prefer_synchronous?(command)
368
+ return true
369
+ end
370
+ end
371
+ end
@@ -0,0 +1,415 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2013 droonga project
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License version 2.1 as published by the Free Software Foundation.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
+
18
+ require "English"
19
+ require "tsort"
20
+ require "groonga"
21
+
22
+ module Droonga
23
+ class Searcher
24
+ class Error < StandardError
25
+ end
26
+
27
+ class UndefinedSourceError < Error
28
+ attr_reader :name
29
+ def initialize(name)
30
+ @name = name
31
+ super("undefined source was used: <#{name}>")
32
+ end
33
+ end
34
+
35
+ def initialize(context)
36
+ @context = context
37
+ end
38
+
39
+ def search(queries)
40
+ outputs = nil
41
+ $log.trace("#{log_tag}: search: start", :queries => queries)
42
+ @context.push_memory_pool do
43
+ outputs = process_queries(queries)
44
+ end
45
+ $log.trace("#{log_tag}: search: done")
46
+ return outputs
47
+ end
48
+
49
+ private
50
+ def process_queries(queries)
51
+ $log.trace("#{log_tag}: process_queries: start")
52
+ unless queries
53
+ $log.trace("#{log_tag}: process_queries: done")
54
+ return {}
55
+ end
56
+ $log.trace("#{log_tag}: process_queries: sort: start")
57
+ query_sorter = QuerySorter.new
58
+ queries.each do |name, query|
59
+ query_sorter.add(name, [query["source"]])
60
+ end
61
+ sorted_queries = query_sorter.tsort
62
+ $log.trace("#{log_tag}: process_queries: sort: done")
63
+ outputs = {}
64
+ results = {}
65
+ sorted_queries.each do |name|
66
+ if queries[name]
67
+ $log.trace("#{log_tag}: process_queries: search: start",
68
+ :name => name)
69
+ searcher = QuerySearcher.new(@context, queries[name])
70
+ results[name] = searcher.search(results)
71
+ $log.trace("#{log_tag}: process_queries: search: done",
72
+ :name => name)
73
+ if searcher.need_output?
74
+ $log.trace("#{log_tag}: process_queries: format: start",
75
+ :name => name)
76
+ outputs[name] = searcher.format
77
+ $log.trace("#{log_tag}: process_queries: format: done",
78
+ :name => name)
79
+ end
80
+ elsif @context[name]
81
+ results[name] = @context[name]
82
+ else
83
+ raise UndefinedSourceError.new(name)
84
+ end
85
+ end
86
+ $log.trace("#{log_tag}: process_queries: done")
87
+ return outputs
88
+ end
89
+
90
+ def log_tag
91
+ "[#{Process.ppid}][#{Process.pid}] searcher"
92
+ end
93
+
94
+ class QuerySorter
95
+ include TSort
96
+ def initialize()
97
+ @queries = {}
98
+ end
99
+
100
+ def add(name, sources=[])
101
+ @queries[name] = sources
102
+ end
103
+
104
+ def tsort_each_node(&block)
105
+ @queries.each_key(&block)
106
+ end
107
+
108
+ def tsort_each_child(node, &block)
109
+ if @queries[node]
110
+ @queries[node].each(&block)
111
+ end
112
+ end
113
+ end
114
+
115
+ class QuerySearcher
116
+ def initialize(context, query)
117
+ @context = context
118
+ @query = query
119
+ @result = nil
120
+ @condition = nil
121
+ @start_time = nil
122
+ end
123
+
124
+ def search(results)
125
+ search_query(results)
126
+ end
127
+
128
+ def need_output?
129
+ @result and @query.has_key?("output")
130
+ end
131
+
132
+ def format
133
+ formatted_result = {}
134
+ format_count(formatted_result)
135
+ format_attributes(formatted_result)
136
+ format_records(formatted_result)
137
+ if need_element_output?("startTime")
138
+ formatted_result["startTime"] = @start_time.iso8601
139
+ end
140
+ if need_element_output?("elapsedTime")
141
+ formatted_result["elapsedTime"] = Time.now.to_f - @start_time.to_f
142
+ end
143
+ formatted_result
144
+ end
145
+
146
+ private
147
+ def parseCondition(source, expression, condition)
148
+ if condition.is_a? String
149
+ expression.parse(condition, :syntax => :script)
150
+ elsif condition.is_a? Hash
151
+ options = {}
152
+ if condition["matchTo"]
153
+ matchTo = Groonga::Expression.new(context: @context)
154
+ matchTo.define_variable(:domain => source)
155
+ match_columns = condition["matchTo"]
156
+ match_columns = match_columns.join(",") if match_columns.is_a?(Array)
157
+ matchTo.parse(match_columns, :syntax => :script)
158
+ options[:default_column] = matchTo
159
+ end
160
+ if condition["query"]
161
+ options[:syntax] = :query
162
+ if condition["default_operator"]
163
+ case condition["default_operator"]
164
+ when "||"
165
+ options[:default_operator] = Groonga::Operator::OR
166
+ when "&&"
167
+ options[:default_operator] = Groonga::Operator::AND
168
+ when "-"
169
+ options[:default_operator] = Groonga::Operator::BUT
170
+ else
171
+ raise "undefined operator assigned #{condition["default_operator"]}"
172
+ end
173
+ end
174
+ if condition["allow_pragma"]
175
+ options[:allow_pragma] = true
176
+ end
177
+ if condition["allow_column"]
178
+ options[:allow_column] = true
179
+ end
180
+ expression.parse(condition["query"], options)
181
+ elsif condition["script"]
182
+ # "script" is ignored when "query" is also assigned.
183
+ options[:syntax] = :script
184
+ if condition["allow_update"]
185
+ options[:allow_update] = true
186
+ end
187
+ expression.parse(condition["script"], options)
188
+ else
189
+ raise "neither 'query' nor 'script' assigned in #{condition.inspect}"
190
+ end
191
+ elsif condition.is_a? Array
192
+ case condition[0]
193
+ when "||"
194
+ operator = Groonga::Operator::OR
195
+ when "&&"
196
+ operator = Groonga::Operator::AND
197
+ when "-"
198
+ operator = Groonga::Operator::BUT
199
+ else
200
+ raise "undefined operator assigned #{condition[0]}"
201
+ end
202
+ if condition[1]
203
+ parseCondition(source, expression, condition[1])
204
+ end
205
+ condition[2..-1].each do |element|
206
+ parseCondition(source, expression, element)
207
+ expression.append_operation(operator, 2)
208
+ end
209
+ else
210
+ raise "unacceptable object #{condition.inspect} assigned"
211
+ end
212
+ end
213
+
214
+ def parse_order_keys(keys)
215
+ keys.collect do |key|
216
+ if key =~ /^-/
217
+ [$POSTMATCH, :descending]
218
+ else
219
+ [key, :ascending]
220
+ end
221
+ end
222
+ end
223
+
224
+ def search_query(results)
225
+ $log.trace("#{log_tag}: search_query: start")
226
+ @start_time = Time.now
227
+ @result = source = results[@query["source"]]
228
+ condition = @query["condition"]
229
+ if condition
230
+ expression = Groonga::Expression.new(context: @context)
231
+ expression.define_variable(:domain => source)
232
+ parseCondition(source, expression, condition)
233
+ $log.trace("#{log_tag}: search_query: select: start",
234
+ :condition => condition)
235
+ @result = source.select(expression)
236
+ $log.trace("#{log_tag}: search_query: select: done")
237
+ @condition = expression
238
+ end
239
+ group_by = @query["groupBy"]
240
+ if group_by
241
+ $log.trace("#{log_tag}: search_query: group: start",
242
+ :by => group_by)
243
+ if group_by.is_a? String
244
+ @result = @result.group(group_by)
245
+ elsif group_by.is_a? Hash
246
+ key = group_by["key"]
247
+ max_n_sub_records = group_by["maxNSubRecords"]
248
+ @result = @result.group(key, :max_n_sub_records => max_n_sub_records)
249
+ else
250
+ raise '"groupBy" parameter must be a Hash or a String'
251
+ end
252
+ $log.trace("#{log_tag}: search_query: group: done",
253
+ :by => group_by)
254
+ end
255
+ @count = @result.size
256
+ sort_by = @query["sortBy"]
257
+ if sort_by
258
+ $log.trace("#{log_tag}: search_query: sort: start",
259
+ :by => sort_by)
260
+ if sort_by.is_a? Array
261
+ keys = parse_order_keys(sort_by)
262
+ offset = 0
263
+ limit = -1
264
+ elsif sort_by.is_a? Hash
265
+ keys = parse_order_keys(sort_by["keys"])
266
+ offset = sort_by["offset"]
267
+ limit = sort_by["limit"]
268
+ else
269
+ raise '"sortBy" parameter must be a Hash or an Array'
270
+ end
271
+ @result = @result.sort(keys, :offset => offset, :limit => limit)
272
+ $log.trace("#{log_tag}: search_query: sort: done",
273
+ :by => sort_by)
274
+ end
275
+ $log.trace("#{log_tag}: search_query: done")
276
+ @result
277
+ end
278
+
279
+ def need_element_output?(element)
280
+ params = @query["output"]
281
+
282
+ elements = params["elements"]
283
+ return false if elements.nil?
284
+
285
+ elements.include?(element)
286
+ end
287
+
288
+ def format_count(formatted_result)
289
+ return unless need_element_output?("count")
290
+ formatted_result["count"] = @count
291
+ end
292
+
293
+ def format_attributes(formatted_result)
294
+ return unless need_element_output?("attributes")
295
+
296
+ # XXX IMPLEMENT ME!!!
297
+ attributes = nil
298
+ if @query["output"]["format"] == "complex"
299
+ attributes = {}
300
+ else
301
+ attributes = []
302
+ end
303
+
304
+ formatted_result["attributes"] = attributes
305
+ end
306
+
307
+ def format_records(formatted_result)
308
+ return unless need_element_output?("records")
309
+
310
+ params = @query["output"]
311
+
312
+ attributes = params["attributes"]
313
+ target_attributes = normalize_target_attributes(attributes)
314
+ offset = params["offset"] || 0
315
+ limit = params["limit"] || 10
316
+ @result.open_cursor(:offset => offset, :limit => limit) do |cursor|
317
+ if params["format"] == "complex"
318
+ formatted_result["records"] = cursor.collect do |record|
319
+ complex_record(target_attributes, record)
320
+ end
321
+ else
322
+ formatted_result["records"] = cursor.collect do |record|
323
+ simple_record(target_attributes, record)
324
+ end
325
+ end
326
+ end
327
+ end
328
+
329
+ def complex_record(attributes, record)
330
+ values = {}
331
+ attributes.collect do |attribute|
332
+ values[attribute[:label]] = record_value(record, attribute)
333
+ end
334
+ values
335
+ end
336
+
337
+ def simple_record(attributes, record)
338
+ attributes.collect do |attribute|
339
+ record_value(record, attribute)
340
+ end
341
+ end
342
+
343
+ def record_value(record, attribute)
344
+ if attribute[:source] == "_subrecs"
345
+ if @query["output"]["format"] == "complex"
346
+ record.sub_records.collect do |sub_record|
347
+ target_attributes = resolve_attributes(attribute, sub_record)
348
+ complex_record(target_attributes, sub_record)
349
+ end
350
+ else
351
+ record.sub_records.collect do |sub_record|
352
+ target_attributes = resolve_attributes(attribute, sub_record)
353
+ simple_record(target_attributes, sub_record)
354
+ end
355
+ end
356
+ else
357
+ expression = attribute[:expression]
358
+ if expression
359
+ variable = attribute[:variable]
360
+ variable.value = record
361
+ expression.execute
362
+ else
363
+ record[attribute[:source]]
364
+ end
365
+ end
366
+ end
367
+
368
+ def resolve_attributes(attribute, record)
369
+ unless attribute[:target_attributes]
370
+ attribute[:target_attributes] =
371
+ normalize_target_attributes(attribute[:attributes], record.table)
372
+ end
373
+ return attribute[:target_attributes]
374
+ end
375
+
376
+ def accessor_name?(source)
377
+ /\A[a-zA-Z\#@$_][a-zA-Z\d\#@$_\-.]*\z/ === source
378
+ end
379
+
380
+ def normalize_target_attributes(attributes, domain = @result)
381
+ attributes.collect do |attribute|
382
+ if attribute.is_a?(String)
383
+ attribute = {
384
+ "source" => attribute,
385
+ }
386
+ end
387
+ source = attribute["source"]
388
+ if accessor_name?(source)
389
+ expression = nil
390
+ variable = nil
391
+ else
392
+ expression = Groonga::Expression.new(context: @context)
393
+ variable = expression.define_variable(domain: domain)
394
+ expression.parse(source, syntax: :script)
395
+ condition = expression.define_variable(name: "$condition",
396
+ reference: true)
397
+ condition.value = @condition
398
+ source = nil
399
+ end
400
+ {
401
+ label: attribute["label"] || attribute["source"],
402
+ source: source,
403
+ expression: expression,
404
+ variable: variable,
405
+ attributes: attribute["attributes"]
406
+ }
407
+ end
408
+ end
409
+
410
+ def log_tag
411
+ "[#{Process.ppid}][#{Process.pid}] query_searcher"
412
+ end
413
+ end
414
+ end
415
+ end