fluent-plugin-droonga 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.travis.yml +7 -0
- data/Gemfile +40 -0
- data/LICENSE.txt +14 -0
- data/README.md +18 -0
- data/Rakefile +25 -0
- data/benchmark/benchmark.rb +123 -0
- data/benchmark/utils.rb +243 -0
- data/benchmark/watch/benchmark-notify.rb +143 -0
- data/benchmark/watch/benchmark-notify.sh +19 -0
- data/benchmark/watch/benchmark-publish.rb +120 -0
- data/benchmark/watch/benchmark-scan.rb +210 -0
- data/benchmark/watch/catalog.json +32 -0
- data/benchmark/watch/fluentd.conf +12 -0
- data/bin/grn2jsons +85 -0
- data/fluent-plugin-droonga.gemspec +41 -0
- data/lib/droonga/adapter.rb +156 -0
- data/lib/droonga/catalog.rb +153 -0
- data/lib/droonga/command_mapper.rb +45 -0
- data/lib/droonga/engine.rb +83 -0
- data/lib/droonga/executor.rb +289 -0
- data/lib/droonga/handler.rb +140 -0
- data/lib/droonga/handler_plugin.rb +35 -0
- data/lib/droonga/job_queue.rb +83 -0
- data/lib/droonga/job_queue_schema.rb +65 -0
- data/lib/droonga/logger.rb +34 -0
- data/lib/droonga/plugin.rb +41 -0
- data/lib/droonga/plugin/adapter/groonga/select.rb +88 -0
- data/lib/droonga/plugin/adapter_groonga.rb +40 -0
- data/lib/droonga/plugin/handler/groonga/column_create.rb +103 -0
- data/lib/droonga/plugin/handler/groonga/table_create.rb +100 -0
- data/lib/droonga/plugin/handler_add.rb +44 -0
- data/lib/droonga/plugin/handler_forward.rb +70 -0
- data/lib/droonga/plugin/handler_groonga.rb +52 -0
- data/lib/droonga/plugin/handler_proxy.rb +82 -0
- data/lib/droonga/plugin/handler_search.rb +33 -0
- data/lib/droonga/plugin/handler_watch.rb +102 -0
- data/lib/droonga/proxy.rb +371 -0
- data/lib/droonga/searcher.rb +415 -0
- data/lib/droonga/server.rb +112 -0
- data/lib/droonga/sweeper.rb +42 -0
- data/lib/droonga/watch_schema.rb +88 -0
- data/lib/droonga/watcher.rb +256 -0
- data/lib/droonga/worker.rb +51 -0
- data/lib/fluent/plugin/out_droonga.rb +56 -0
- data/lib/groonga_command_converter.rb +137 -0
- data/sample/cluster/catalog.json +43 -0
- data/sample/cluster/fluentd.conf +12 -0
- data/sample/fluentd.conf +8 -0
- data/test/fixtures/catalog.json +43 -0
- data/test/fixtures/document.grn +23 -0
- data/test/helper.rb +24 -0
- data/test/helper/fixture.rb +28 -0
- data/test/helper/sandbox.rb +73 -0
- data/test/helper/stub_worker.rb +27 -0
- data/test/helper/watch_helper.rb +35 -0
- data/test/plugin/adapter/groonga/test_select.rb +176 -0
- data/test/plugin/handler/groonga/test_column_create.rb +127 -0
- data/test/plugin/handler/groonga/test_table_create.rb +140 -0
- data/test/plugin/handler/test_handler_add.rb +135 -0
- data/test/plugin/handler/test_handler_groonga.rb +64 -0
- data/test/plugin/handler/test_handler_search.rb +512 -0
- data/test/plugin/handler/test_handler_watch.rb +168 -0
- data/test/run-test.rb +55 -0
- data/test/test_adapter.rb +48 -0
- data/test/test_catalog.rb +59 -0
- data/test/test_command_mapper.rb +44 -0
- data/test/test_groonga_command_converter.rb +242 -0
- data/test/test_handler.rb +53 -0
- data/test/test_job_queue_schema.rb +45 -0
- data/test/test_output.rb +99 -0
- data/test/test_sweeper.rb +95 -0
- data/test/test_watch_schema.rb +57 -0
- data/test/test_watcher.rb +336 -0
- data/test/test_worker.rb +144 -0
- metadata +299 -0
@@ -0,0 +1,371 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2013 droonga project
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
+
|
18
|
+
require 'tsort'
|
19
|
+
require "droonga/handler"
|
20
|
+
require "droonga/adapter"
|
21
|
+
require "droonga/catalog"
|
22
|
+
|
23
|
+
module Droonga
|
24
|
+
class Proxy
|
25
|
+
attr_reader :collectors
|
26
|
+
def initialize(worker, name)
|
27
|
+
@engines = {}
|
28
|
+
Droonga::catalog.get_engines(name).each do |name, options|
|
29
|
+
engine = Droonga::Engine.new(options.merge(:proxy => false,
|
30
|
+
:with_server => false))
|
31
|
+
engine.start
|
32
|
+
@engines[name] = engine
|
33
|
+
end
|
34
|
+
@worker = worker
|
35
|
+
@name = name
|
36
|
+
@collectors = {}
|
37
|
+
@current_id = 0
|
38
|
+
@local = Regexp.new("^#{@name}")
|
39
|
+
plugins = ["proxy"] + (Droonga::catalog.option("plugins")||[]) + ["adapter"]
|
40
|
+
plugins.each do |plugin|
|
41
|
+
@worker.add_handler(plugin)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def shutdown
|
46
|
+
@engines.each do |name, engine|
|
47
|
+
engine.shutdown
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def handle(message, arguments)
|
52
|
+
case message
|
53
|
+
when Array
|
54
|
+
handle_incoming_message(message)
|
55
|
+
when Hash
|
56
|
+
handle_internal_message(message)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def handle_incoming_message(message)
|
61
|
+
id = generate_id
|
62
|
+
planner = Planner.new(self, message)
|
63
|
+
destinations = planner.resolve(id)
|
64
|
+
components = planner.components
|
65
|
+
message = { "id" => id, "components" => components }
|
66
|
+
destinations.each do |destination, frequency|
|
67
|
+
dispatch(message, destination)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def handle_internal_message(message)
|
72
|
+
id = message["id"]
|
73
|
+
collector = @collectors[id]
|
74
|
+
unless collector
|
75
|
+
components = message["components"]
|
76
|
+
if components
|
77
|
+
planner = Planner.new(self, components)
|
78
|
+
collector = planner.get_collector(id)
|
79
|
+
else
|
80
|
+
#todo: take cases receiving result before its query into account
|
81
|
+
end
|
82
|
+
end
|
83
|
+
collector.handle(message["input"], message["value"])
|
84
|
+
end
|
85
|
+
|
86
|
+
def dispatch(message, destination)
|
87
|
+
if local?(destination)
|
88
|
+
handle_internal_message(message)
|
89
|
+
else
|
90
|
+
post(message, "to"=>farm_path(destination), "type"=>"proxy")
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def deliver(id, route, message, type, synchronous)
|
95
|
+
if id == route
|
96
|
+
post(message, "type" => type, "synchronous"=> synchronous)
|
97
|
+
else
|
98
|
+
envelope = @worker.envelope.merge("body" => message, "type" => type)
|
99
|
+
@engines[route].emit('', Time.now.to_f, envelope, synchronous)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def post(message, destination)
|
104
|
+
@worker.post(message, destination)
|
105
|
+
end
|
106
|
+
|
107
|
+
def generate_id
|
108
|
+
id = @current_id
|
109
|
+
@current_id = id.succ
|
110
|
+
return [@name, id].join('.#')
|
111
|
+
end
|
112
|
+
|
113
|
+
def farm_path(route)
|
114
|
+
if route =~ /\A.*:\d+\/[^\.]+/
|
115
|
+
$&
|
116
|
+
else
|
117
|
+
route
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def local?(route)
|
122
|
+
route =~ @local
|
123
|
+
end
|
124
|
+
|
125
|
+
class Planner
|
126
|
+
attr_reader :components
|
127
|
+
class UndefinedInputError < StandardError
|
128
|
+
attr_reader :input
|
129
|
+
def initialize(input)
|
130
|
+
@input = input
|
131
|
+
super("undefined input assigned: <#{input}>")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
class CyclicComponentsError < StandardError
|
136
|
+
attr_reader :components
|
137
|
+
def initialize(components)
|
138
|
+
@components = components
|
139
|
+
super("cyclic components found: <#{components}>")
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
include TSort
|
144
|
+
def initialize(proxy, components)
|
145
|
+
@proxy = proxy
|
146
|
+
@components = components
|
147
|
+
end
|
148
|
+
|
149
|
+
def resolve(id)
|
150
|
+
@dependency = {}
|
151
|
+
@components.each do |component|
|
152
|
+
@dependency[component] = component["inputs"]
|
153
|
+
next unless component["outputs"]
|
154
|
+
component["outputs"].each do |output|
|
155
|
+
@dependency[output] = [component]
|
156
|
+
end
|
157
|
+
end
|
158
|
+
@components = []
|
159
|
+
each_strongly_connected_component do |cs|
|
160
|
+
raise CyclicComponentsError.new(cs) if cs.size > 1
|
161
|
+
@components.concat(cs) unless cs.first.is_a? String
|
162
|
+
end
|
163
|
+
resolve_routes(id)
|
164
|
+
end
|
165
|
+
|
166
|
+
def resolve_routes(id)
|
167
|
+
local = [id]
|
168
|
+
destinations = Hash.new(0)
|
169
|
+
@components.each do |component|
|
170
|
+
dataset = component["dataset"]
|
171
|
+
routes =
|
172
|
+
if dataset
|
173
|
+
Droonga::catalog.get_routes(dataset, component)
|
174
|
+
else
|
175
|
+
local
|
176
|
+
end
|
177
|
+
routes.each do |route|
|
178
|
+
destinations[@proxy.farm_path(route)] += 1
|
179
|
+
end
|
180
|
+
component["routes"] = routes
|
181
|
+
end
|
182
|
+
return destinations
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_collector(id)
|
186
|
+
resolve_descendants
|
187
|
+
tasks = []
|
188
|
+
inputs = {}
|
189
|
+
@components.each do |component|
|
190
|
+
component["routes"].each do |route|
|
191
|
+
next unless @proxy.local?(route)
|
192
|
+
task = {
|
193
|
+
"route" => route,
|
194
|
+
"component" => component,
|
195
|
+
"n_of_inputs" => 0,
|
196
|
+
"values" => {}
|
197
|
+
}
|
198
|
+
tasks << task
|
199
|
+
(component["inputs"] || [nil]).each do |input|
|
200
|
+
inputs[input] ||= []
|
201
|
+
inputs[input] << task
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
collector = Collector.new(id, @proxy, @components, tasks, inputs)
|
206
|
+
@proxy.collectors[id] = collector
|
207
|
+
return collector
|
208
|
+
end
|
209
|
+
|
210
|
+
def resolve_descendants
|
211
|
+
@descendants = {}
|
212
|
+
@components.size.times do |index|
|
213
|
+
component = @components[index]
|
214
|
+
(component["inputs"] || []).each do |input|
|
215
|
+
@descendants[input] ||= []
|
216
|
+
@descendants[input] << index
|
217
|
+
end
|
218
|
+
component["n_of_expects"] = 0
|
219
|
+
end
|
220
|
+
@components.each do |component|
|
221
|
+
descendants = get_descendants(component)
|
222
|
+
component["descendants"] = descendants
|
223
|
+
descendants.each do |key, indices|
|
224
|
+
indices.each do |index|
|
225
|
+
@components[index]["n_of_expects"] += component["routes"].size
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def get_descendants(component)
|
232
|
+
return {} unless component["outputs"]
|
233
|
+
descendants = {}
|
234
|
+
component["outputs"].each do |output|
|
235
|
+
descendants[output] = @descendants[output]
|
236
|
+
end
|
237
|
+
descendants
|
238
|
+
end
|
239
|
+
|
240
|
+
def tsort_each_node(&block)
|
241
|
+
@dependency.each_key(&block)
|
242
|
+
end
|
243
|
+
|
244
|
+
def tsort_each_child(node, &block)
|
245
|
+
if node.is_a? String and @dependency[node].nil?
|
246
|
+
raise UndefinedInputError.new(node)
|
247
|
+
end
|
248
|
+
if @dependency[node]
|
249
|
+
@dependency[node].each(&block)
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
class Collector
|
255
|
+
def initialize(id, proxy, components, tasks, inputs)
|
256
|
+
@id = id
|
257
|
+
@proxy = proxy
|
258
|
+
@components = components
|
259
|
+
@tasks = tasks
|
260
|
+
@n_dones = 0
|
261
|
+
@inputs = inputs
|
262
|
+
end
|
263
|
+
|
264
|
+
def handle(name, value)
|
265
|
+
tasks = @inputs[name]
|
266
|
+
unless tasks
|
267
|
+
#TODO: result arrived before its query
|
268
|
+
return
|
269
|
+
end
|
270
|
+
tasks.each do |task|
|
271
|
+
task["n_of_inputs"] += 1 if name
|
272
|
+
component = task["component"]
|
273
|
+
type = component["type"]
|
274
|
+
command = component["command"] || ("proxy_" + type)
|
275
|
+
n_of_expects = component["n_of_expects"]
|
276
|
+
synchronous = nil
|
277
|
+
if command
|
278
|
+
# TODO: should be controllable for each command respectively.
|
279
|
+
synchronous = !n_of_expects.zero?
|
280
|
+
# TODO: check if asynchronous execution is available.
|
281
|
+
message = {
|
282
|
+
"task"=>task,
|
283
|
+
"name"=>name,
|
284
|
+
"value"=>value
|
285
|
+
}
|
286
|
+
unless synchronous
|
287
|
+
descendants = {}
|
288
|
+
component["descendants"].each do |name, indices|
|
289
|
+
descendants[name] = indices.collect do |index|
|
290
|
+
@components[index]["routes"].map do |route|
|
291
|
+
@proxy.farm_path(route)
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
message["descendants"] = descendants
|
296
|
+
message["id"] = @id
|
297
|
+
end
|
298
|
+
@proxy.deliver(@id, task["route"], message, command, synchronous)
|
299
|
+
end
|
300
|
+
return if task["n_of_inputs"] < n_of_expects
|
301
|
+
#the task is done
|
302
|
+
if synchronous
|
303
|
+
result = task["values"]
|
304
|
+
post = component["post"]
|
305
|
+
@proxy.post(result, post) if post
|
306
|
+
component["descendants"].each do |name, indices|
|
307
|
+
message = {
|
308
|
+
"id" => @id,
|
309
|
+
"input" => name,
|
310
|
+
"value" => result[name]
|
311
|
+
}
|
312
|
+
indices.each do |index|
|
313
|
+
@components[index]["routes"].each do |route|
|
314
|
+
@proxy.dispatch(message, route)
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
@n_dones += 1
|
320
|
+
@proxy.collectors.delete(@id) if @n_dones == @tasks.size
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
class ProxyMessageHandler < Droonga::Handler
|
327
|
+
Droonga::HandlerPlugin.register("proxy_message", self)
|
328
|
+
def initialize(*arguments)
|
329
|
+
super
|
330
|
+
@proxy = Droonga::Proxy.new(@worker, @worker.name)
|
331
|
+
end
|
332
|
+
|
333
|
+
def shutdown
|
334
|
+
@proxy.shutdown
|
335
|
+
end
|
336
|
+
|
337
|
+
command :proxy
|
338
|
+
def proxy(request, *arguments)
|
339
|
+
@proxy.handle(request, arguments)
|
340
|
+
end
|
341
|
+
|
342
|
+
def prefer_synchronous?(command)
|
343
|
+
return true
|
344
|
+
end
|
345
|
+
end
|
346
|
+
|
347
|
+
class ProxyHandler < Droonga::Handler
|
348
|
+
attr_reader :task, :input_name, :component, :output_values, :body, :output_names
|
349
|
+
def handle(command, request, *arguments)
|
350
|
+
return false unless request.is_a? Hash
|
351
|
+
@task = request["task"]
|
352
|
+
return false unless @task.is_a? Hash
|
353
|
+
@component = @task["component"]
|
354
|
+
return false unless @component.is_a? Hash
|
355
|
+
@output_values = @task["values"]
|
356
|
+
@body = @component["body"]
|
357
|
+
@output_names = @component["outputs"]
|
358
|
+
@id = request["id"]
|
359
|
+
@value = request["value"]
|
360
|
+
@input_name = request["name"]
|
361
|
+
@descendants = request["descendants"]
|
362
|
+
invoke(command, @value, *arguments)
|
363
|
+
output if @descendants
|
364
|
+
true
|
365
|
+
end
|
366
|
+
|
367
|
+
def prefer_synchronous?(command)
|
368
|
+
return true
|
369
|
+
end
|
370
|
+
end
|
371
|
+
end
|
@@ -0,0 +1,415 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2013 droonga project
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
+
|
18
|
+
require "English"
|
19
|
+
require "tsort"
|
20
|
+
require "groonga"
|
21
|
+
|
22
|
+
module Droonga
|
23
|
+
class Searcher
|
24
|
+
class Error < StandardError
|
25
|
+
end
|
26
|
+
|
27
|
+
class UndefinedSourceError < Error
|
28
|
+
attr_reader :name
|
29
|
+
def initialize(name)
|
30
|
+
@name = name
|
31
|
+
super("undefined source was used: <#{name}>")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(context)
|
36
|
+
@context = context
|
37
|
+
end
|
38
|
+
|
39
|
+
def search(queries)
|
40
|
+
outputs = nil
|
41
|
+
$log.trace("#{log_tag}: search: start", :queries => queries)
|
42
|
+
@context.push_memory_pool do
|
43
|
+
outputs = process_queries(queries)
|
44
|
+
end
|
45
|
+
$log.trace("#{log_tag}: search: done")
|
46
|
+
return outputs
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
def process_queries(queries)
|
51
|
+
$log.trace("#{log_tag}: process_queries: start")
|
52
|
+
unless queries
|
53
|
+
$log.trace("#{log_tag}: process_queries: done")
|
54
|
+
return {}
|
55
|
+
end
|
56
|
+
$log.trace("#{log_tag}: process_queries: sort: start")
|
57
|
+
query_sorter = QuerySorter.new
|
58
|
+
queries.each do |name, query|
|
59
|
+
query_sorter.add(name, [query["source"]])
|
60
|
+
end
|
61
|
+
sorted_queries = query_sorter.tsort
|
62
|
+
$log.trace("#{log_tag}: process_queries: sort: done")
|
63
|
+
outputs = {}
|
64
|
+
results = {}
|
65
|
+
sorted_queries.each do |name|
|
66
|
+
if queries[name]
|
67
|
+
$log.trace("#{log_tag}: process_queries: search: start",
|
68
|
+
:name => name)
|
69
|
+
searcher = QuerySearcher.new(@context, queries[name])
|
70
|
+
results[name] = searcher.search(results)
|
71
|
+
$log.trace("#{log_tag}: process_queries: search: done",
|
72
|
+
:name => name)
|
73
|
+
if searcher.need_output?
|
74
|
+
$log.trace("#{log_tag}: process_queries: format: start",
|
75
|
+
:name => name)
|
76
|
+
outputs[name] = searcher.format
|
77
|
+
$log.trace("#{log_tag}: process_queries: format: done",
|
78
|
+
:name => name)
|
79
|
+
end
|
80
|
+
elsif @context[name]
|
81
|
+
results[name] = @context[name]
|
82
|
+
else
|
83
|
+
raise UndefinedSourceError.new(name)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
$log.trace("#{log_tag}: process_queries: done")
|
87
|
+
return outputs
|
88
|
+
end
|
89
|
+
|
90
|
+
def log_tag
|
91
|
+
"[#{Process.ppid}][#{Process.pid}] searcher"
|
92
|
+
end
|
93
|
+
|
94
|
+
class QuerySorter
|
95
|
+
include TSort
|
96
|
+
def initialize()
|
97
|
+
@queries = {}
|
98
|
+
end
|
99
|
+
|
100
|
+
def add(name, sources=[])
|
101
|
+
@queries[name] = sources
|
102
|
+
end
|
103
|
+
|
104
|
+
def tsort_each_node(&block)
|
105
|
+
@queries.each_key(&block)
|
106
|
+
end
|
107
|
+
|
108
|
+
def tsort_each_child(node, &block)
|
109
|
+
if @queries[node]
|
110
|
+
@queries[node].each(&block)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
class QuerySearcher
|
116
|
+
def initialize(context, query)
|
117
|
+
@context = context
|
118
|
+
@query = query
|
119
|
+
@result = nil
|
120
|
+
@condition = nil
|
121
|
+
@start_time = nil
|
122
|
+
end
|
123
|
+
|
124
|
+
def search(results)
|
125
|
+
search_query(results)
|
126
|
+
end
|
127
|
+
|
128
|
+
def need_output?
|
129
|
+
@result and @query.has_key?("output")
|
130
|
+
end
|
131
|
+
|
132
|
+
def format
|
133
|
+
formatted_result = {}
|
134
|
+
format_count(formatted_result)
|
135
|
+
format_attributes(formatted_result)
|
136
|
+
format_records(formatted_result)
|
137
|
+
if need_element_output?("startTime")
|
138
|
+
formatted_result["startTime"] = @start_time.iso8601
|
139
|
+
end
|
140
|
+
if need_element_output?("elapsedTime")
|
141
|
+
formatted_result["elapsedTime"] = Time.now.to_f - @start_time.to_f
|
142
|
+
end
|
143
|
+
formatted_result
|
144
|
+
end
|
145
|
+
|
146
|
+
private
|
147
|
+
def parseCondition(source, expression, condition)
|
148
|
+
if condition.is_a? String
|
149
|
+
expression.parse(condition, :syntax => :script)
|
150
|
+
elsif condition.is_a? Hash
|
151
|
+
options = {}
|
152
|
+
if condition["matchTo"]
|
153
|
+
matchTo = Groonga::Expression.new(context: @context)
|
154
|
+
matchTo.define_variable(:domain => source)
|
155
|
+
match_columns = condition["matchTo"]
|
156
|
+
match_columns = match_columns.join(",") if match_columns.is_a?(Array)
|
157
|
+
matchTo.parse(match_columns, :syntax => :script)
|
158
|
+
options[:default_column] = matchTo
|
159
|
+
end
|
160
|
+
if condition["query"]
|
161
|
+
options[:syntax] = :query
|
162
|
+
if condition["default_operator"]
|
163
|
+
case condition["default_operator"]
|
164
|
+
when "||"
|
165
|
+
options[:default_operator] = Groonga::Operator::OR
|
166
|
+
when "&&"
|
167
|
+
options[:default_operator] = Groonga::Operator::AND
|
168
|
+
when "-"
|
169
|
+
options[:default_operator] = Groonga::Operator::BUT
|
170
|
+
else
|
171
|
+
raise "undefined operator assigned #{condition["default_operator"]}"
|
172
|
+
end
|
173
|
+
end
|
174
|
+
if condition["allow_pragma"]
|
175
|
+
options[:allow_pragma] = true
|
176
|
+
end
|
177
|
+
if condition["allow_column"]
|
178
|
+
options[:allow_column] = true
|
179
|
+
end
|
180
|
+
expression.parse(condition["query"], options)
|
181
|
+
elsif condition["script"]
|
182
|
+
# "script" is ignored when "query" is also assigned.
|
183
|
+
options[:syntax] = :script
|
184
|
+
if condition["allow_update"]
|
185
|
+
options[:allow_update] = true
|
186
|
+
end
|
187
|
+
expression.parse(condition["script"], options)
|
188
|
+
else
|
189
|
+
raise "neither 'query' nor 'script' assigned in #{condition.inspect}"
|
190
|
+
end
|
191
|
+
elsif condition.is_a? Array
|
192
|
+
case condition[0]
|
193
|
+
when "||"
|
194
|
+
operator = Groonga::Operator::OR
|
195
|
+
when "&&"
|
196
|
+
operator = Groonga::Operator::AND
|
197
|
+
when "-"
|
198
|
+
operator = Groonga::Operator::BUT
|
199
|
+
else
|
200
|
+
raise "undefined operator assigned #{condition[0]}"
|
201
|
+
end
|
202
|
+
if condition[1]
|
203
|
+
parseCondition(source, expression, condition[1])
|
204
|
+
end
|
205
|
+
condition[2..-1].each do |element|
|
206
|
+
parseCondition(source, expression, element)
|
207
|
+
expression.append_operation(operator, 2)
|
208
|
+
end
|
209
|
+
else
|
210
|
+
raise "unacceptable object #{condition.inspect} assigned"
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def parse_order_keys(keys)
|
215
|
+
keys.collect do |key|
|
216
|
+
if key =~ /^-/
|
217
|
+
[$POSTMATCH, :descending]
|
218
|
+
else
|
219
|
+
[key, :ascending]
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def search_query(results)
|
225
|
+
$log.trace("#{log_tag}: search_query: start")
|
226
|
+
@start_time = Time.now
|
227
|
+
@result = source = results[@query["source"]]
|
228
|
+
condition = @query["condition"]
|
229
|
+
if condition
|
230
|
+
expression = Groonga::Expression.new(context: @context)
|
231
|
+
expression.define_variable(:domain => source)
|
232
|
+
parseCondition(source, expression, condition)
|
233
|
+
$log.trace("#{log_tag}: search_query: select: start",
|
234
|
+
:condition => condition)
|
235
|
+
@result = source.select(expression)
|
236
|
+
$log.trace("#{log_tag}: search_query: select: done")
|
237
|
+
@condition = expression
|
238
|
+
end
|
239
|
+
group_by = @query["groupBy"]
|
240
|
+
if group_by
|
241
|
+
$log.trace("#{log_tag}: search_query: group: start",
|
242
|
+
:by => group_by)
|
243
|
+
if group_by.is_a? String
|
244
|
+
@result = @result.group(group_by)
|
245
|
+
elsif group_by.is_a? Hash
|
246
|
+
key = group_by["key"]
|
247
|
+
max_n_sub_records = group_by["maxNSubRecords"]
|
248
|
+
@result = @result.group(key, :max_n_sub_records => max_n_sub_records)
|
249
|
+
else
|
250
|
+
raise '"groupBy" parameter must be a Hash or a String'
|
251
|
+
end
|
252
|
+
$log.trace("#{log_tag}: search_query: group: done",
|
253
|
+
:by => group_by)
|
254
|
+
end
|
255
|
+
@count = @result.size
|
256
|
+
sort_by = @query["sortBy"]
|
257
|
+
if sort_by
|
258
|
+
$log.trace("#{log_tag}: search_query: sort: start",
|
259
|
+
:by => sort_by)
|
260
|
+
if sort_by.is_a? Array
|
261
|
+
keys = parse_order_keys(sort_by)
|
262
|
+
offset = 0
|
263
|
+
limit = -1
|
264
|
+
elsif sort_by.is_a? Hash
|
265
|
+
keys = parse_order_keys(sort_by["keys"])
|
266
|
+
offset = sort_by["offset"]
|
267
|
+
limit = sort_by["limit"]
|
268
|
+
else
|
269
|
+
raise '"sortBy" parameter must be a Hash or an Array'
|
270
|
+
end
|
271
|
+
@result = @result.sort(keys, :offset => offset, :limit => limit)
|
272
|
+
$log.trace("#{log_tag}: search_query: sort: done",
|
273
|
+
:by => sort_by)
|
274
|
+
end
|
275
|
+
$log.trace("#{log_tag}: search_query: done")
|
276
|
+
@result
|
277
|
+
end
|
278
|
+
|
279
|
+
def need_element_output?(element)
|
280
|
+
params = @query["output"]
|
281
|
+
|
282
|
+
elements = params["elements"]
|
283
|
+
return false if elements.nil?
|
284
|
+
|
285
|
+
elements.include?(element)
|
286
|
+
end
|
287
|
+
|
288
|
+
def format_count(formatted_result)
|
289
|
+
return unless need_element_output?("count")
|
290
|
+
formatted_result["count"] = @count
|
291
|
+
end
|
292
|
+
|
293
|
+
def format_attributes(formatted_result)
|
294
|
+
return unless need_element_output?("attributes")
|
295
|
+
|
296
|
+
# XXX IMPLEMENT ME!!!
|
297
|
+
attributes = nil
|
298
|
+
if @query["output"]["format"] == "complex"
|
299
|
+
attributes = {}
|
300
|
+
else
|
301
|
+
attributes = []
|
302
|
+
end
|
303
|
+
|
304
|
+
formatted_result["attributes"] = attributes
|
305
|
+
end
|
306
|
+
|
307
|
+
def format_records(formatted_result)
|
308
|
+
return unless need_element_output?("records")
|
309
|
+
|
310
|
+
params = @query["output"]
|
311
|
+
|
312
|
+
attributes = params["attributes"]
|
313
|
+
target_attributes = normalize_target_attributes(attributes)
|
314
|
+
offset = params["offset"] || 0
|
315
|
+
limit = params["limit"] || 10
|
316
|
+
@result.open_cursor(:offset => offset, :limit => limit) do |cursor|
|
317
|
+
if params["format"] == "complex"
|
318
|
+
formatted_result["records"] = cursor.collect do |record|
|
319
|
+
complex_record(target_attributes, record)
|
320
|
+
end
|
321
|
+
else
|
322
|
+
formatted_result["records"] = cursor.collect do |record|
|
323
|
+
simple_record(target_attributes, record)
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
def complex_record(attributes, record)
|
330
|
+
values = {}
|
331
|
+
attributes.collect do |attribute|
|
332
|
+
values[attribute[:label]] = record_value(record, attribute)
|
333
|
+
end
|
334
|
+
values
|
335
|
+
end
|
336
|
+
|
337
|
+
def simple_record(attributes, record)
|
338
|
+
attributes.collect do |attribute|
|
339
|
+
record_value(record, attribute)
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
def record_value(record, attribute)
|
344
|
+
if attribute[:source] == "_subrecs"
|
345
|
+
if @query["output"]["format"] == "complex"
|
346
|
+
record.sub_records.collect do |sub_record|
|
347
|
+
target_attributes = resolve_attributes(attribute, sub_record)
|
348
|
+
complex_record(target_attributes, sub_record)
|
349
|
+
end
|
350
|
+
else
|
351
|
+
record.sub_records.collect do |sub_record|
|
352
|
+
target_attributes = resolve_attributes(attribute, sub_record)
|
353
|
+
simple_record(target_attributes, sub_record)
|
354
|
+
end
|
355
|
+
end
|
356
|
+
else
|
357
|
+
expression = attribute[:expression]
|
358
|
+
if expression
|
359
|
+
variable = attribute[:variable]
|
360
|
+
variable.value = record
|
361
|
+
expression.execute
|
362
|
+
else
|
363
|
+
record[attribute[:source]]
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
def resolve_attributes(attribute, record)
|
369
|
+
unless attribute[:target_attributes]
|
370
|
+
attribute[:target_attributes] =
|
371
|
+
normalize_target_attributes(attribute[:attributes], record.table)
|
372
|
+
end
|
373
|
+
return attribute[:target_attributes]
|
374
|
+
end
|
375
|
+
|
376
|
+
def accessor_name?(source)
|
377
|
+
/\A[a-zA-Z\#@$_][a-zA-Z\d\#@$_\-.]*\z/ === source
|
378
|
+
end
|
379
|
+
|
380
|
+
def normalize_target_attributes(attributes, domain = @result)
|
381
|
+
attributes.collect do |attribute|
|
382
|
+
if attribute.is_a?(String)
|
383
|
+
attribute = {
|
384
|
+
"source" => attribute,
|
385
|
+
}
|
386
|
+
end
|
387
|
+
source = attribute["source"]
|
388
|
+
if accessor_name?(source)
|
389
|
+
expression = nil
|
390
|
+
variable = nil
|
391
|
+
else
|
392
|
+
expression = Groonga::Expression.new(context: @context)
|
393
|
+
variable = expression.define_variable(domain: domain)
|
394
|
+
expression.parse(source, syntax: :script)
|
395
|
+
condition = expression.define_variable(name: "$condition",
|
396
|
+
reference: true)
|
397
|
+
condition.value = @condition
|
398
|
+
source = nil
|
399
|
+
end
|
400
|
+
{
|
401
|
+
label: attribute["label"] || attribute["source"],
|
402
|
+
source: source,
|
403
|
+
expression: expression,
|
404
|
+
variable: variable,
|
405
|
+
attributes: attribute["attributes"]
|
406
|
+
}
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
def log_tag
|
411
|
+
"[#{Process.ppid}][#{Process.pid}] query_searcher"
|
412
|
+
end
|
413
|
+
end
|
414
|
+
end
|
415
|
+
end
|