fluent-plugin-droonga 0.8.0 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile +7 -0
- data/benchmark/watch/benchmark-notify.rb +6 -6
- data/benchmark/watch/benchmark-notify.sh +3 -2
- data/bin/grn2jsons +0 -3
- data/doc/text/news.md +13 -0
- data/fluent-plugin-droonga.gemspec +1 -1
- data/lib/droonga/catalog/base.rb +8 -3
- data/lib/droonga/catalog.rb +1 -16
- data/lib/droonga/catalog_observer.rb +57 -0
- data/lib/droonga/collector.rb +1 -1
- data/lib/droonga/dispatcher.rb +3 -1
- data/lib/droonga/distributor_plugin.rb +47 -19
- data/lib/droonga/handler_messenger.rb +26 -2
- data/lib/droonga/message_processing_error.rb +6 -8
- data/lib/droonga/output_message.rb +17 -1
- data/lib/droonga/plugin/collector/basic.rb +44 -81
- data/lib/droonga/plugin/collector/groonga.rb +83 -0
- data/lib/droonga/plugin/collector/search.rb +104 -0
- data/lib/droonga/plugin/distributor/crud.rb +41 -0
- data/lib/droonga/plugin/distributor/distributed_search_planner.rb +4 -4
- data/lib/droonga/plugin/distributor/groonga.rb +38 -0
- data/lib/droonga/plugin/handler/add.rb +4 -1
- data/lib/droonga/plugin/handler/groonga/column_create.rb +7 -0
- data/lib/droonga/plugin/handler/groonga/table_remove.rb +42 -0
- data/lib/droonga/plugin/handler/groonga.rb +17 -2
- data/lib/droonga/plugin/handler/watch.rb +4 -4
- data/lib/droonga/plugin/input_adapter/crud.rb +27 -0
- data/lib/droonga/plugin/input_adapter/groonga.rb +16 -1
- data/lib/droonga/plugin/output_adapter/crud.rb +51 -0
- data/lib/droonga/plugin/output_adapter/groonga.rb +8 -1
- data/lib/droonga/plugin.rb +1 -1
- data/lib/droonga/replier.rb +7 -1
- data/lib/droonga/searcher.rb +168 -74
- data/lib/droonga/session.rb +1 -1
- data/lib/groonga_command_converter.rb +7 -1
- data/test/command/config/default/catalog.json +1 -1
- data/test/command/suite/add/error/invalid-integer.expected +31 -1
- data/test/command/suite/add/error/invalid-time.expected +31 -1
- data/test/command/suite/add/error/missing-key.expected +17 -1
- data/test/command/suite/add/error/missing-table.expected +17 -1
- data/test/command/suite/add/error/unknown-column.expected +31 -1
- data/test/command/suite/add/error/unknown-table.expected +17 -1
- data/test/command/suite/add/minimum.expected +1 -1
- data/test/command/suite/add/with-values.expected +1 -1
- data/test/command/suite/add/without-key.expected +1 -1
- data/test/command/suite/groonga/column_create/scalar.expected +2 -2
- data/test/command/suite/groonga/column_create/unknown-table.expected +18 -0
- data/test/command/suite/groonga/column_create/unknown-table.test +7 -0
- data/test/command/suite/groonga/column_create/vector.expected +2 -2
- data/test/command/suite/groonga/select/minimum.expected +1 -1
- data/test/command/suite/groonga/table_create/array.expected +1 -1
- data/test/command/suite/groonga/table_create/hash.expected +1 -1
- data/test/command/suite/groonga/table_remove/success.expected +17 -0
- data/test/command/suite/groonga/table_remove/success.test +8 -0
- data/test/command/suite/groonga/table_remove/unknown-table.expected +18 -0
- data/test/command/suite/groonga/table_remove/unknown-table.test +7 -0
- data/test/command/suite/message/error/missing-dataset.expected +1 -1
- data/test/command/suite/message/error/unknown-command.expected +1 -1
- data/test/command/suite/message/error/unknown-dataset.expected +1 -1
- data/test/command/suite/search/attributes/array.expected +1 -1
- data/test/command/suite/search/attributes/hash.expected +1 -1
- data/test/command/suite/search/complex.expected +1 -1
- data/test/command/suite/search/condition/nested.expected +1 -1
- data/test/command/suite/search/condition/query.expected +1 -1
- data/test/command/suite/search/condition/script.expected +1 -1
- data/test/command/suite/search/error/cyclic-source.expected +1 -1
- data/test/command/suite/search/error/deeply-cyclic-source.expected +1 -1
- data/test/command/suite/search/error/missing-source-parameter.expected +1 -1
- data/test/command/suite/search/error/unknown-source.expected +1 -1
- data/test/command/suite/search/group/count.expected +1 -1
- data/test/command/suite/search/group/limit.expected +1 -1
- data/test/command/suite/search/group/string.expected +1 -1
- data/test/command/suite/search/multiple/chained.expected +1 -1
- data/test/command/suite/search/multiple/parallel.expected +1 -1
- data/test/command/suite/search/range/only-output.expected +1 -1
- data/test/command/suite/search/range/only-sort.expected +1 -1
- data/test/command/suite/search/range/sort-and-output.expected +1 -1
- data/test/command/suite/search/range/too-large-output-offset.expected +1 -1
- data/test/command/suite/search/range/too-large-sort-offset.expected +1 -1
- data/test/command/suite/search/response/records/value/time.expected +1 -1
- data/test/command/suite/search/simple.expected +1 -1
- data/test/command/suite/search/sort/default-offset-limit.expected +1 -1
- data/test/command/suite/search/sort/invisible-column.expected +1 -1
- data/test/command/suite/watch/subscribe.expected +1 -1
- data/test/command/suite/watch/unsubscribe.expected +1 -1
- data/test/performance/run-test.rb +56 -0
- data/{benchmark → test/performance}/watch/catalog.json +0 -0
- data/test/performance/watch/feed.json +9 -0
- data/{benchmark → test/performance}/watch/fluentd.conf +0 -0
- data/test/performance/watch/subscribe.json +3 -0
- data/test/unit/catalog/test_version1.rb +34 -0
- data/test/unit/plugin/collector/test_basic.rb +300 -479
- data/test/unit/plugin/collector/test_search.rb +814 -0
- data/test/unit/plugin/distributor/test_search.rb +4 -4
- data/test/unit/plugin/distributor/test_search_planner.rb +260 -260
- data/test/unit/plugin/handler/groonga/test_column_create.rb +15 -1
- data/test/unit/plugin/handler/groonga/test_table_create.rb +6 -2
- data/test/unit/plugin/handler/groonga/test_table_remove.rb +58 -0
- data/test/unit/plugin/handler/test_add.rb +3 -1
- data/test/unit/plugin/handler/test_groonga.rb +24 -0
- data/test/unit/plugin/handler/test_search.rb +294 -0
- data/test/unit/plugin/handler/test_watch.rb +1 -1
- data/test/unit/plugin/{adapter → input_adapter}/groonga/test_select.rb +5 -37
- data/test/unit/plugin/output_adapter/groonga/test_select.rb +55 -0
- metadata +38 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 80594c4e021918c28b58451b90293961f7b65174
|
4
|
+
data.tar.gz: ff89eb342312841a7d9eaa8831a7d190bec34c35
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 39351f8baffeae2e33ac2b3d1ad6f06c39fe08a626ee8f260c62501e03b5acb05a67f2e0667833250f200679664d518da774b57c2356c30f260cd0afb83f8940
|
7
|
+
data.tar.gz: c26dd8db11b63e7f6130826f6eb4ce97b05dae064e39b9502b3cc7cbf0f54a912c5fc3885bab2855ace0a21cba9af851e64e9a4d40c136310790c848d9e0aadd
|
data/.gitignore
CHANGED
data/Gemfile
CHANGED
@@ -50,3 +50,10 @@ if File.exist?(drntest_dir)
|
|
50
50
|
else
|
51
51
|
gem "drntest", :github => "droonga/drntest"
|
52
52
|
end
|
53
|
+
|
54
|
+
drnbench_dir = File.join(parent_dir, "drnbench")
|
55
|
+
if File.exist?(drnbench_dir)
|
56
|
+
gem "drnbench", :path => drnbench_dir
|
57
|
+
else
|
58
|
+
gem "drnbench", :github => "droonga/drnbench"
|
59
|
+
end
|
@@ -54,11 +54,11 @@ class NotifyBenchmark
|
|
54
54
|
do_feed("#{WATCHING_KEYWORD} #{index}")
|
55
55
|
end
|
56
56
|
|
57
|
-
|
58
|
-
while
|
59
|
-
|
57
|
+
published_messages = []
|
58
|
+
while published_messages.size != @n_times
|
59
|
+
published_messages << @receiver.new_message
|
60
60
|
end
|
61
|
-
|
61
|
+
published_messages
|
62
62
|
end
|
63
63
|
|
64
64
|
def add_subscribers(n_subscribers)
|
@@ -118,8 +118,8 @@ options[:n_steps].times do |try_count|
|
|
118
118
|
percentage = nil
|
119
119
|
result = Benchmark.bm do |benchmark|
|
120
120
|
benchmark.report(label) do
|
121
|
-
|
122
|
-
percentage =
|
121
|
+
published_messages = notify_benchmark.run
|
122
|
+
percentage = published_messages.size.to_f / options[:n_times] * 100
|
123
123
|
end
|
124
124
|
end
|
125
125
|
puts "=> #{percentage} % feeds are notified"
|
@@ -1,13 +1,14 @@
|
|
1
1
|
#!/bin/sh
|
2
2
|
|
3
3
|
base_dir=$(cd $(dirname $0); pwd)
|
4
|
+
work_dir=$base_dir/../../performance/watch
|
4
5
|
|
5
6
|
rm -rf $base_dir/watch
|
6
7
|
mkdir -p $base_dir/watch
|
7
8
|
|
8
|
-
DROONGA_CATALOG=$
|
9
|
+
DROONGA_CATALOG=$work_dir/catalog.json \
|
9
10
|
bundle exec fluentd \
|
10
|
-
--config $
|
11
|
+
--config $work_dir/fluentd.conf &
|
11
12
|
FLUENTD_PID=$!
|
12
13
|
|
13
14
|
sleep 1
|
data/bin/grn2jsons
CHANGED
@@ -45,9 +45,6 @@ option_parser = OptionParser.new do |parser|
|
|
45
45
|
end
|
46
46
|
args = option_parser.parse!(ARGV)
|
47
47
|
|
48
|
-
if options.reply_to.nil?
|
49
|
-
raise "You must specify the value of the \"replyTo\" field by --reply-to option."
|
50
|
-
end
|
51
48
|
if options.dataset.nil?
|
52
49
|
raise "You must specify the name of the dataset by --dataset option."
|
53
50
|
end
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,18 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 0.9.0: 2014-01-29
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* `search`: Supported `"attributes"` for `elements` of `output`.
|
8
|
+
* `table_remove`: Implemented Groonga compatible `table_remove`
|
9
|
+
command.
|
10
|
+
* `column_create`: Implemented error handling.
|
11
|
+
* `catalog`: Supported auto reloading.
|
12
|
+
* Supported reducing responses from two or more nodes for Groonga
|
13
|
+
compatible commands.
|
14
|
+
* Supported three or more partitions.
|
15
|
+
|
3
16
|
## 0.8.0: 2013-12-29
|
4
17
|
|
5
18
|
### Improvements
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |gem|
|
19
19
|
gem.name = "fluent-plugin-droonga"
|
20
|
-
gem.version = "0.
|
20
|
+
gem.version = "0.9.0"
|
21
21
|
gem.authors = ["Droonga Project"]
|
22
22
|
gem.email = ["droonga@groonga.org"]
|
23
23
|
gem.description = "Droonga(distributed Groonga) plugin for Fluent event collector"
|
data/lib/droonga/catalog/base.rb
CHANGED
@@ -34,9 +34,7 @@ module Droonga
|
|
34
34
|
@data["datasets"].each do |name, dataset|
|
35
35
|
number_of_partitions = dataset["number_of_partitions"]
|
36
36
|
next if number_of_partitions < 2
|
37
|
-
total_weight = dataset
|
38
|
-
a[1]["weight"] + b[1]["weight"]
|
39
|
-
end
|
37
|
+
total_weight = compute_total_weight(dataset)
|
40
38
|
continuum = []
|
41
39
|
dataset["ring"].each do |key, value|
|
42
40
|
points = number_of_partitions * 160 * value["weight"] / total_weight
|
@@ -135,6 +133,13 @@ module Droonga
|
|
135
133
|
end
|
136
134
|
end
|
137
135
|
end
|
136
|
+
|
137
|
+
private
|
138
|
+
def compute_total_weight(dataset)
|
139
|
+
dataset["ring"].reduce(0) do |result, zone|
|
140
|
+
result + zone[1]["weight"]
|
141
|
+
end
|
142
|
+
end
|
138
143
|
end
|
139
144
|
end
|
140
145
|
end
|
data/lib/droonga/catalog.rb
CHANGED
@@ -19,21 +19,6 @@ require "droonga/catalog_loader"
|
|
19
19
|
|
20
20
|
module Droonga
|
21
21
|
class << self
|
22
|
-
|
23
|
-
@catalog ||= Catalog.load
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
module Catalog
|
28
|
-
PATH = "catalog.json"
|
29
|
-
|
30
|
-
class << self
|
31
|
-
def load(path=nil)
|
32
|
-
path = ENV["DROONGA_CATALOG"] || PATH
|
33
|
-
path = File.expand_path(path)
|
34
|
-
loader = CatalogLoader.new(path)
|
35
|
-
loader.load
|
36
|
-
end
|
37
|
-
end
|
22
|
+
attr_accessor :catalog
|
38
23
|
end
|
39
24
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2014 Droonga Project
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
+
|
18
|
+
module Droonga
|
19
|
+
class CatalogObserver
|
20
|
+
DEFAULT_CATALOG_PATH = "catalog.json"
|
21
|
+
CHECK_INTERVAL = 1
|
22
|
+
|
23
|
+
def initialize(loop)
|
24
|
+
@catalog_path = catalog_path
|
25
|
+
load_catalog
|
26
|
+
|
27
|
+
watcher = Cool.io::TimerWatcher.new(CHECK_INTERVAL, true)
|
28
|
+
observer = self
|
29
|
+
watcher.on_timer do
|
30
|
+
observer.ensure_latest_catalog_loaded
|
31
|
+
end
|
32
|
+
loop.attach(watcher)
|
33
|
+
end
|
34
|
+
|
35
|
+
def ensure_latest_catalog_loaded
|
36
|
+
if catalog_updated?
|
37
|
+
load_catalog
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def catalog_path
|
42
|
+
path = ENV["DROONGA_CATALOG"] || DEFAULT_CATALOG_PATH
|
43
|
+
File.expand_path(path)
|
44
|
+
end
|
45
|
+
|
46
|
+
def catalog_updated?
|
47
|
+
File.mtime(catalog_path) > @catalog_mtime
|
48
|
+
end
|
49
|
+
|
50
|
+
def load_catalog
|
51
|
+
loader = CatalogLoader.new(@catalog_path)
|
52
|
+
Droonga.catalog = loader.load
|
53
|
+
@catalog_mtime = File.mtime(@catalog_path)
|
54
|
+
$log.info "catalog loaded", path: @catalog_path, mtime: @catalog_mtime
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/lib/droonga/collector.rb
CHANGED
data/lib/droonga/dispatcher.rb
CHANGED
@@ -27,6 +27,7 @@ require "droonga/farm"
|
|
27
27
|
require "droonga/session"
|
28
28
|
require "droonga/replier"
|
29
29
|
require "droonga/message_processing_error"
|
30
|
+
require "droonga/catalog_observer"
|
30
31
|
|
31
32
|
module Droonga
|
32
33
|
class Dispatcher
|
@@ -48,6 +49,8 @@ module Droonga
|
|
48
49
|
def initialize(options)
|
49
50
|
@options = options
|
50
51
|
@name = @options[:name]
|
52
|
+
@loop = EventLoop.new
|
53
|
+
@catalog_observer = CatalogObserver.new(@loop)
|
51
54
|
@sessions = {}
|
52
55
|
@current_id = 0
|
53
56
|
@local = Regexp.new("^#{@name}")
|
@@ -55,7 +58,6 @@ module Droonga
|
|
55
58
|
InputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
|
56
59
|
@output_adapter =
|
57
60
|
OutputAdapter.new(self, :plugins => Droonga.catalog.option("plugins"))
|
58
|
-
@loop = EventLoop.new
|
59
61
|
@farm = Farm.new(name, @loop, :dispatcher => self)
|
60
62
|
@forwarder = Forwarder.new(@loop)
|
61
63
|
@replier = Replier.new(@forwarder)
|
@@ -31,28 +31,13 @@ module Droonga
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def scatter_all(message, key)
|
34
|
-
|
35
|
-
|
36
|
-
"dataset"=> message["dataset"],
|
37
|
-
"body"=> message["body"],
|
38
|
-
"key"=> key,
|
39
|
-
"type"=> "scatter",
|
40
|
-
"replica"=> "all",
|
41
|
-
"post"=> true
|
42
|
-
}]
|
43
|
-
distribute(distribute_message)
|
34
|
+
messages = [reducer(message), gatherer(message), scatterer(message, key)]
|
35
|
+
distribute(messages)
|
44
36
|
end
|
45
37
|
|
46
38
|
def broadcast_all(message)
|
47
|
-
|
48
|
-
|
49
|
-
"dataset"=> message["dataset"],
|
50
|
-
"body"=> message["body"],
|
51
|
-
"type"=> "broadcast",
|
52
|
-
"replica"=> "all",
|
53
|
-
"post"=> true
|
54
|
-
}]
|
55
|
-
distribute(distribute_message)
|
39
|
+
messages = [reducer(message), gatherer(message), broadcaster(message)]
|
40
|
+
distribute(messages)
|
56
41
|
end
|
57
42
|
|
58
43
|
private
|
@@ -63,5 +48,48 @@ module Droonga
|
|
63
48
|
super
|
64
49
|
end
|
65
50
|
end
|
51
|
+
|
52
|
+
def scatterer(message, key)
|
53
|
+
{
|
54
|
+
"command" => message["type"],
|
55
|
+
"dataset" => message["dataset"],
|
56
|
+
"body" => message["body"],
|
57
|
+
"key" => key,
|
58
|
+
"type" => "scatter",
|
59
|
+
"outputs" => [],
|
60
|
+
"replica" => "all",
|
61
|
+
"post" => true
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
def broadcaster(message)
|
66
|
+
{
|
67
|
+
"command" => message["type"],
|
68
|
+
"dataset" => message["dataset"],
|
69
|
+
"body" => message["body"],
|
70
|
+
"type" => "broadcast",
|
71
|
+
"outputs" => [],
|
72
|
+
"replica" => "all",
|
73
|
+
"post" => true
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
def reducer(message)
|
78
|
+
{
|
79
|
+
"type" => "reduce",
|
80
|
+
"body" => {},
|
81
|
+
"inputs" => [],
|
82
|
+
"outputs" => [],
|
83
|
+
}
|
84
|
+
end
|
85
|
+
|
86
|
+
def gatherer(message)
|
87
|
+
{
|
88
|
+
"type" => "gather",
|
89
|
+
"body" => {},
|
90
|
+
"inputs" => [],
|
91
|
+
"post" => true,
|
92
|
+
}
|
93
|
+
end
|
66
94
|
end
|
67
95
|
end
|
@@ -18,12 +18,15 @@ require "droonga/forwarder"
|
|
18
18
|
|
19
19
|
module Droonga
|
20
20
|
class HandlerMessenger
|
21
|
+
attr_reader :database_name
|
22
|
+
|
21
23
|
def initialize(forwarder, message, options={})
|
22
24
|
@forwarder = forwarder
|
23
25
|
@message = message
|
24
26
|
@options = options
|
25
27
|
@replier = Replier.new(@forwarder)
|
26
28
|
@dispatcher = @options[:dispatcher]
|
29
|
+
@database_name = options[:database]
|
27
30
|
end
|
28
31
|
|
29
32
|
def emit(value)
|
@@ -60,8 +63,29 @@ module Droonga
|
|
60
63
|
"body" => body)
|
61
64
|
@replier.reply(response)
|
62
65
|
else
|
63
|
-
|
64
|
-
|
66
|
+
body = {
|
67
|
+
"id" => @message.id,
|
68
|
+
"input" => "errors",
|
69
|
+
"value" => {
|
70
|
+
database_name => {
|
71
|
+
"statusCode" => status_code,
|
72
|
+
"body" => body,
|
73
|
+
},
|
74
|
+
},
|
75
|
+
}
|
76
|
+
all_dests = []
|
77
|
+
descendants.each do |name, dests|
|
78
|
+
all_dests += dests
|
79
|
+
end
|
80
|
+
all_dests.each do |dest|
|
81
|
+
if @dispatcher
|
82
|
+
@dispatcher.dispatch(body, dest)
|
83
|
+
else
|
84
|
+
message = raw_message.merge("statusCode" => status_code,
|
85
|
+
"body" => body,)
|
86
|
+
forward(message, "to" => dest, "type" => "dispatcher")
|
87
|
+
end
|
88
|
+
end
|
65
89
|
end
|
66
90
|
end
|
67
91
|
|
@@ -15,8 +15,10 @@
|
|
15
15
|
|
16
16
|
module Droonga
|
17
17
|
class MessageProcessingError < StandardError
|
18
|
+
STATUS_CODE = 500
|
19
|
+
|
18
20
|
attr_reader :message, :detail
|
19
|
-
|
21
|
+
|
20
22
|
def initialize(message, detail=nil)
|
21
23
|
@message = message
|
22
24
|
@detail = detail
|
@@ -27,7 +29,7 @@ module Droonga
|
|
27
29
|
end
|
28
30
|
|
29
31
|
def status_code
|
30
|
-
|
32
|
+
self.class::STATUS_CODE
|
31
33
|
end
|
32
34
|
|
33
35
|
def response_body
|
@@ -41,14 +43,10 @@ module Droonga
|
|
41
43
|
end
|
42
44
|
|
43
45
|
class BadRequest < MessageProcessingError
|
44
|
-
|
45
|
-
400
|
46
|
-
end
|
46
|
+
STATUS_CODE = 400
|
47
47
|
end
|
48
48
|
|
49
49
|
class NotFound < MessageProcessingError
|
50
|
-
|
51
|
-
404
|
52
|
-
end
|
50
|
+
STATUS_CODE = 404
|
53
51
|
end
|
54
52
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2013 Droonga Project
|
1
|
+
# Copyright (C) 2013-2014 Droonga Project
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -26,6 +26,22 @@ module Droonga
|
|
26
26
|
@raw_message
|
27
27
|
end
|
28
28
|
|
29
|
+
def status_code
|
30
|
+
@raw_message["statusCode"]
|
31
|
+
end
|
32
|
+
|
33
|
+
def status_code=(status_code)
|
34
|
+
@raw_message["statusCode"] = status_code
|
35
|
+
end
|
36
|
+
|
37
|
+
def errors
|
38
|
+
@raw_message["errors"]
|
39
|
+
end
|
40
|
+
|
41
|
+
def errors=(errors)
|
42
|
+
@raw_message["errors"] = errors
|
43
|
+
end
|
44
|
+
|
29
45
|
def body
|
30
46
|
@raw_message["body"]
|
31
47
|
end
|
@@ -27,42 +27,49 @@ module Droonga
|
|
27
27
|
def collector_gather(result)
|
28
28
|
output = body ? body[input_name] : input_name
|
29
29
|
if output.is_a?(Hash)
|
30
|
-
elements = output["elements"]
|
31
|
-
if elements && elements.is_a?(Hash)
|
32
|
-
# phase 1: pre-process
|
33
|
-
elements.each do |element, mapper|
|
34
|
-
case mapper["type"]
|
35
|
-
when "count"
|
36
|
-
result[element] = result[mapper["target"]].size
|
37
|
-
when "sort"
|
38
|
-
# do nothing on this phase!
|
39
|
-
end
|
40
|
-
end
|
41
|
-
# phase 2: post-process
|
42
|
-
elements.each do |element, mapper|
|
43
|
-
if mapper["no_output"]
|
44
|
-
result.delete(element)
|
45
|
-
next
|
46
|
-
end
|
47
|
-
|
48
|
-
case mapper["type"]
|
49
|
-
when "count"
|
50
|
-
# do nothing on this phase!
|
51
|
-
when "sort"
|
52
|
-
# because "count" type mapper requires all items of the array,
|
53
|
-
# I have to apply "sort" type mapper later.
|
54
|
-
if result[element]
|
55
|
-
result[element] = apply_output_range(result[element], mapper)
|
56
|
-
result[element] = apply_output_attributes_and_format(result[element], mapper)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
30
|
output = output["output"]
|
62
31
|
end
|
63
32
|
emit(output, result)
|
64
33
|
end
|
65
34
|
|
35
|
+
command :collector_reduce
|
36
|
+
def collector_reduce(request)
|
37
|
+
body[input_name].each do |output, deal|
|
38
|
+
left_value = output_values[output]
|
39
|
+
right_value = request
|
40
|
+
value = reduce(deal, left_value, right_value)
|
41
|
+
emit(output, value)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def reduce(deal, left_value, right_value)
|
46
|
+
if left_value.nil? || right_value.nil?
|
47
|
+
return right_value || left_value
|
48
|
+
end
|
49
|
+
|
50
|
+
reduced_value = nil
|
51
|
+
|
52
|
+
case deal["type"]
|
53
|
+
when "and"
|
54
|
+
reduced_value = left_value && right_value
|
55
|
+
when "sum"
|
56
|
+
reduced_value = sum(left_value, right_value)
|
57
|
+
reduced_value = apply_output_range(reduced_value,
|
58
|
+
"limit" => deal["limit"])
|
59
|
+
when "average"
|
60
|
+
reduced_value = (left_value.to_f + right_value.to_f) / 2
|
61
|
+
when "sort"
|
62
|
+
reduced_value = merge(left_value,
|
63
|
+
right_value,
|
64
|
+
:operators => deal["operators"],
|
65
|
+
:key_column => deal["key_column"])
|
66
|
+
reduced_value = apply_output_range(reduced_value,
|
67
|
+
"limit" => deal["limit"])
|
68
|
+
end
|
69
|
+
|
70
|
+
reduced_value
|
71
|
+
end
|
72
|
+
|
66
73
|
def apply_output_range(items, output)
|
67
74
|
if items && items.is_a?(Array)
|
68
75
|
offset = output["offset"] || 0
|
@@ -78,58 +85,14 @@ module Droonga
|
|
78
85
|
items
|
79
86
|
end
|
80
87
|
|
81
|
-
def
|
82
|
-
|
83
|
-
if attributes
|
84
|
-
format = output["format"]
|
85
|
-
if format == "complex"
|
86
|
-
items.collect! do |item|
|
87
|
-
complex_item = {}
|
88
|
-
attributes.each_with_index do |label, index|
|
89
|
-
complex_item[label] = item[index]
|
90
|
-
end
|
91
|
-
complex_item
|
92
|
-
end
|
93
|
-
else
|
94
|
-
items.collect! do |item|
|
95
|
-
item[0...attributes.size]
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
99
|
-
items
|
100
|
-
end
|
101
|
-
|
102
|
-
command :collector_reduce
|
103
|
-
def collector_reduce(request)
|
104
|
-
return unless request
|
105
|
-
body[input_name].each do |output, elements|
|
106
|
-
value = request
|
107
|
-
old_value = output_values[output]
|
108
|
-
value = reduce(elements, old_value, request) if old_value
|
109
|
-
emit(output, value)
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def reduce(elements, *values)
|
114
|
-
result = {}
|
115
|
-
elements.each do |key, deal|
|
116
|
-
reduced_values = nil
|
117
|
-
|
118
|
-
case deal["type"]
|
119
|
-
when "sum"
|
120
|
-
reduced_values = values[0][key] + values[1][key]
|
121
|
-
when "sort"
|
122
|
-
reduced_values = merge(values[0][key],
|
123
|
-
values[1][key],
|
124
|
-
:operators => deal["operators"],
|
125
|
-
:key_column => deal["key_column"])
|
126
|
-
end
|
127
|
-
|
128
|
-
reduced_values = apply_output_range(reduced_values, "limit" => deal["limit"])
|
88
|
+
def sum(x, y)
|
89
|
+
return x || y if x.nil? or y.nil?
|
129
90
|
|
130
|
-
|
91
|
+
if x.is_a?(Hash) && y.is_a?(Hash)
|
92
|
+
x.merge(y)
|
93
|
+
else
|
94
|
+
x + y
|
131
95
|
end
|
132
|
-
return result
|
133
96
|
end
|
134
97
|
|
135
98
|
def merge(x, y, options={})
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2014 Droonga Project
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
+
|
18
|
+
require "droonga/plugin/collector/basic"
|
19
|
+
|
20
|
+
module Droonga
|
21
|
+
class GroongaCollector < BasicCollector
|
22
|
+
repository.register("groonga", self)
|
23
|
+
|
24
|
+
command :collector_groonga_gather
|
25
|
+
def collector_groonga_gather(result)
|
26
|
+
collector_gather(result)
|
27
|
+
end
|
28
|
+
|
29
|
+
command :collector_groonga_reduce
|
30
|
+
def collector_groonga_reduce(request)
|
31
|
+
collector_reduce(request)
|
32
|
+
end
|
33
|
+
|
34
|
+
def reduce(deal, left_value, right_value)
|
35
|
+
reduced_value = nil
|
36
|
+
|
37
|
+
case deal["type"]
|
38
|
+
when "groonga_result"
|
39
|
+
#XXX how to merge multiple erros?
|
40
|
+
#XXX how to mix regular results and erros?
|
41
|
+
# reduced_value = merge_groonga_result(left_value, right_value)
|
42
|
+
reduced_value = left_value || right_value
|
43
|
+
else
|
44
|
+
reduced_value = super
|
45
|
+
end
|
46
|
+
|
47
|
+
reduced_value
|
48
|
+
end
|
49
|
+
|
50
|
+
def merge_groonga_result(left_value, right_value)
|
51
|
+
result = []
|
52
|
+
|
53
|
+
result << merge_groonga_header(left_value.shift, right_value.shift)
|
54
|
+
|
55
|
+
left_value.each_with_index do |left, index|
|
56
|
+
right = right_value[index]
|
57
|
+
result << reduce({ "type" => "and" }, left, right)
|
58
|
+
end
|
59
|
+
|
60
|
+
result
|
61
|
+
end
|
62
|
+
|
63
|
+
def merge_groonga_header(left_header, right_header)
|
64
|
+
status = [left_header.shift, right_header.shift].min
|
65
|
+
|
66
|
+
start_time = reduce({ "type" => "average" },
|
67
|
+
left_header.shift,
|
68
|
+
right_header.shift)
|
69
|
+
|
70
|
+
elapsed_time = reduce({ "type" => "average" },
|
71
|
+
left_header.shift,
|
72
|
+
right_header.shift)
|
73
|
+
|
74
|
+
#XXX we should merge error informations more smarter...
|
75
|
+
error_information = reduce({ "type" => "sum",
|
76
|
+
"limit" => UNLIMITED },
|
77
|
+
left_header,
|
78
|
+
right_header)
|
79
|
+
|
80
|
+
[status, start_time, elapsed_time] + error_information
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|