fluent-plugin-droonga 0.9.9 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dir-locals.el +3 -0
- data/.travis.yml +6 -2
- data/README.md +6 -7
- data/Rakefile +23 -6
- data/fluent-plugin-droonga.gemspec +2 -2
- data/lib/droonga/adapter.rb +12 -3
- data/lib/droonga/adapter_runner.rb +28 -23
- data/lib/droonga/catalog/base.rb +7 -111
- data/lib/droonga/catalog/dataset.rb +13 -25
- data/lib/droonga/catalog/errors.rb +94 -0
- data/lib/droonga/catalog/schema.rb +277 -0
- data/lib/droonga/catalog/version1.rb +404 -0
- data/lib/droonga/catalog/version2.rb +160 -0
- data/lib/droonga/catalog_loader.rb +27 -4
- data/lib/droonga/catalog_observer.rb +44 -6
- data/lib/droonga/collector.rb +12 -10
- data/lib/droonga/{handler_plugin.rb → collector_message.rb} +47 -20
- data/lib/droonga/collector_runner.rb +64 -0
- data/lib/droonga/collectors.rb +18 -0
- data/lib/droonga/{catalog.rb → collectors/add.rb} +9 -7
- data/lib/droonga/{command_repository.rb → collectors/and.rb} +7 -14
- data/lib/droonga/collectors/sum.rb +26 -0
- data/lib/droonga/dispatcher.rb +74 -41
- data/lib/droonga/distributed_command_planner.rb +2 -2
- data/lib/droonga/engine.rb +13 -5
- data/lib/droonga/{message_processing_error.rb → error.rb} +33 -12
- data/lib/droonga/{plugin/planner/search.rb → error_messages.rb} +12 -10
- data/lib/droonga/farm.rb +15 -14
- data/lib/droonga/fluent_message_sender.rb +15 -11
- data/lib/droonga/forwarder.rb +22 -18
- data/lib/droonga/handler.rb +8 -2
- data/lib/droonga/handler_runner.rb +47 -26
- data/lib/droonga/input_message.rb +6 -6
- data/lib/droonga/{command.rb → loggable.rb} +7 -14
- data/lib/droonga/logger.rb +56 -15
- data/lib/droonga/message_matcher.rb +12 -7
- data/lib/droonga/message_pusher.rb +8 -4
- data/lib/droonga/message_receiver.rb +11 -9
- data/lib/droonga/output_message.rb +2 -0
- data/lib/droonga/planner.rb +21 -10
- data/lib/droonga/plugin.rb +15 -0
- data/lib/droonga/plugin/metadata/{adapter_message.rb → adapter_input_message.rb} +6 -14
- data/lib/droonga/plugin/metadata/adapter_output_message.rb +39 -0
- data/lib/droonga/plugin/metadata/collector_message.rb +39 -0
- data/lib/droonga/plugin/metadata/input_message.rb +15 -0
- data/lib/droonga/plugin_loader.rb +33 -25
- data/lib/droonga/plugin_registry.rb +9 -1
- data/lib/droonga/plugins/basic.rb +54 -0
- data/lib/droonga/plugins/crud.rb +36 -15
- data/lib/droonga/plugins/error.rb +5 -4
- data/lib/droonga/plugins/groonga.rb +9 -6
- data/lib/droonga/plugins/groonga/column_create.rb +10 -5
- data/lib/droonga/plugins/groonga/generic_command.rb +2 -8
- data/lib/droonga/plugins/groonga/generic_response.rb +2 -2
- data/lib/droonga/plugins/groonga/select.rb +2 -2
- data/lib/droonga/plugins/groonga/table_create.rb +9 -4
- data/lib/droonga/plugins/groonga/table_remove.rb +10 -5
- data/lib/droonga/plugins/search.rb +106 -5
- data/lib/droonga/plugins/search/distributed_search_planner.rb +398 -0
- data/lib/droonga/plugins/watch.rb +41 -20
- data/lib/droonga/processor.rb +12 -9
- data/lib/droonga/{plugin/collector/basic.rb → reducer.rb} +36 -50
- data/lib/droonga/replier.rb +7 -4
- data/lib/droonga/searcher.rb +40 -37
- data/lib/droonga/server.rb +8 -6
- data/lib/droonga/session.rb +17 -7
- data/lib/droonga/single_step.rb +53 -0
- data/lib/droonga/{plugin/planner/watch.rb → single_step_definition.rb} +27 -26
- data/lib/droonga/{partition.rb → slice.rb} +23 -12
- data/lib/droonga/status_code.rb +25 -0
- data/lib/droonga/step_runner.rb +63 -0
- data/lib/droonga/watch_schema.rb +7 -3
- data/lib/droonga/watcher.rb +4 -4
- data/lib/droonga/worker.rb +6 -6
- data/lib/fluent/plugin/out_droonga.rb +27 -2
- data/sample/cluster/catalog.json +33 -32
- data/test/command/config/default/catalog.json +72 -45
- data/test/command/config/version1/catalog.json +68 -0
- data/test/command/config/version1/fluentd.conf +11 -0
- data/test/command/suite/message/error/missing-dataset.expected +1 -1
- data/test/command/suite/message/error/unknown-dataset.expected +1 -1
- data/test/command/suite/message/error/unknown-type.expected +13 -0
- data/test/command/suite/message/error/{unknown-command.test → unknown-type.test} +1 -1
- data/test/command/suite/search/error/missing-source-parameter.expected +1 -1
- data/test/command/suite/search/error/unknown-source.expected +15 -3
- data/test/command/suite/watch/subscribe.expected +1 -3
- data/test/command/suite/watch/unsubscribe.expected +1 -3
- data/test/performance/watch/catalog.json +1 -0
- data/test/unit/catalog/test_dataset.rb +16 -358
- data/test/unit/catalog/test_schema.rb +285 -0
- data/test/unit/catalog/test_version1.rb +222 -28
- data/test/unit/catalog/test_version2.rb +155 -0
- data/test/unit/fixtures/catalog/version2.json +62 -0
- data/test/unit/helper/distributed_search_planner_helper.rb +2 -2
- data/test/unit/plugins/crud/test_add.rb +13 -13
- data/test/unit/plugins/groonga/test_column_create.rb +14 -11
- data/test/unit/plugins/groonga/test_table_create.rb +4 -9
- data/test/unit/plugins/groonga/test_table_remove.rb +4 -9
- data/test/unit/{plugin/planner/search_planner → plugins/search/planner}/test_basic.rb +0 -0
- data/test/unit/{plugin/planner/search_planner → plugins/search/planner}/test_group_by.rb +0 -0
- data/test/unit/{plugin/planner/search_planner → plugins/search/planner}/test_output.rb +0 -0
- data/test/unit/{plugin/planner/search_planner → plugins/search/planner}/test_sort_by.rb +0 -0
- data/test/unit/{plugin/collector/test_search.rb → plugins/search/test_collector.rb} +40 -39
- data/test/unit/plugins/{test_search.rb → search/test_handler.rb} +6 -5
- data/test/unit/{plugin/planner/test_search.rb → plugins/search/test_planner.rb} +3 -3
- data/test/unit/{plugin/collector → plugins}/test_basic.rb +68 -50
- data/test/unit/plugins/test_groonga.rb +2 -15
- data/test/unit/plugins/test_watch.rb +25 -22
- data/test/unit/test_message_matcher.rb +29 -6
- data/test/unit/test_output.rb +4 -0
- metadata +58 -50
- data/lib/droonga/collector_plugin.rb +0 -50
- data/lib/droonga/legacy_pluggable.rb +0 -66
- data/lib/droonga/legacy_plugin.rb +0 -57
- data/lib/droonga/legacy_plugin_repository.rb +0 -54
- data/lib/droonga/planner_plugin.rb +0 -54
- data/lib/droonga/plugin/collector/search.rb +0 -98
- data/lib/droonga/plugin/planner/crud.rb +0 -49
- data/lib/droonga/plugin/planner/distributed_search_planner.rb +0 -393
- data/lib/droonga/plugin/planner/groonga.rb +0 -54
- data/lib/droonga/plugin_registerable.rb +0 -75
- data/test/command/suite/message/error/unknown-command.expected +0 -13
- data/test/unit/test_command_repository.rb +0 -39
- data/test/unit/test_legacy_plugin.rb +0 -50
- data/test/unit/test_legacy_plugin_repository.rb +0 -89
@@ -0,0 +1,53 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/planner"
|
17
|
+
require "droonga/collectors"
|
18
|
+
|
19
|
+
module Droonga
|
20
|
+
class SingleStep
|
21
|
+
def initialize(definition)
|
22
|
+
@definition = definition
|
23
|
+
end
|
24
|
+
|
25
|
+
def plan(message)
|
26
|
+
if message["type"] == "search"
|
27
|
+
# XXX: workaround
|
28
|
+
planner = Plugins::Search::Planner.new
|
29
|
+
return planner.plan(message)
|
30
|
+
end
|
31
|
+
|
32
|
+
# XXX: Re-implement me.
|
33
|
+
planner = Planner.new
|
34
|
+
options = {}
|
35
|
+
options[:write] = @definition.write?
|
36
|
+
collector_class = @definition.collector_class
|
37
|
+
if collector_class
|
38
|
+
reduce_key = "result"
|
39
|
+
options[:reduce] = {
|
40
|
+
reduce_key => collector_class.operator,
|
41
|
+
}
|
42
|
+
end
|
43
|
+
inputs = @definition.inputs
|
44
|
+
if inputs.empty?
|
45
|
+
planner.send(:broadcast, message, options)
|
46
|
+
else
|
47
|
+
input = inputs.values.first
|
48
|
+
options[:key] = message["body"][input[:filter]]["key"]
|
49
|
+
planner.send(:scatter, message, options)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
4
2
|
#
|
5
3
|
# This library is free software; you can redistribute it and/or
|
6
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -15,39 +13,42 @@
|
|
15
13
|
# License along with this library; if not, write to the Free Software
|
16
14
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
15
|
|
18
|
-
require "droonga/planner_plugin"
|
19
|
-
|
20
16
|
module Droonga
|
21
|
-
class
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
17
|
+
class SingleStepDefinition
|
18
|
+
attr_accessor :name
|
19
|
+
attr_accessor :handler
|
20
|
+
attr_accessor :collector
|
21
|
+
attr_writer :write
|
22
|
+
attr_accessor :inputs
|
23
|
+
attr_accessor :output
|
24
|
+
def initialize(plugin_module)
|
25
|
+
@plugin_module = plugin_module
|
26
|
+
@name = nil
|
27
|
+
@handler = nil
|
28
|
+
@collector = nil
|
29
|
+
@write = false
|
30
|
+
@inputs = []
|
31
|
+
@output = {}
|
32
|
+
yield(self)
|
27
33
|
end
|
28
34
|
|
29
|
-
|
30
|
-
|
31
|
-
broadcast(message)
|
35
|
+
def write?
|
36
|
+
@write
|
32
37
|
end
|
33
38
|
|
34
|
-
|
35
|
-
|
36
|
-
broadcast(message)
|
39
|
+
def handler_class
|
40
|
+
resolve_class(@handler)
|
37
41
|
end
|
38
42
|
|
39
|
-
|
40
|
-
|
41
|
-
broadcast(message)
|
43
|
+
def collector_class
|
44
|
+
resolve_class(@collector)
|
42
45
|
end
|
43
46
|
|
44
47
|
private
|
45
|
-
def
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
"success" => "and"
|
50
|
-
})
|
48
|
+
def resolve_class(target)
|
49
|
+
return nil if target.nil?
|
50
|
+
return target if target.is_a?(Class)
|
51
|
+
@plugin_module.const_get(target)
|
51
52
|
end
|
52
53
|
end
|
53
54
|
end
|
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
1
|
+
# Copyright (C) 2013-2014 Droonga Project
|
4
2
|
#
|
5
3
|
# This library is free software; you can redistribute it and/or
|
6
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -17,6 +15,7 @@
|
|
17
15
|
|
18
16
|
require "serverengine"
|
19
17
|
|
18
|
+
require "droonga/loggable"
|
20
19
|
require "droonga/server"
|
21
20
|
require "droonga/worker"
|
22
21
|
require "droonga/event_loop"
|
@@ -24,7 +23,9 @@ require "droonga/message_pusher"
|
|
24
23
|
require "droonga/processor"
|
25
24
|
|
26
25
|
module Droonga
|
27
|
-
class
|
26
|
+
class Slice
|
27
|
+
include Loggable
|
28
|
+
|
28
29
|
def initialize(loop, options={})
|
29
30
|
@options = options
|
30
31
|
@n_workers = @options[:n_workers] || 0
|
@@ -43,21 +44,22 @@ module Droonga
|
|
43
44
|
end
|
44
45
|
|
45
46
|
def shutdown
|
46
|
-
|
47
|
+
logger.trace("shutdown: start")
|
47
48
|
shutdown_supervisor if @supervisor
|
48
49
|
@message_pusher.shutdown
|
49
50
|
@processor.shutdown
|
50
|
-
|
51
|
+
logger.trace("shutdown: done")
|
51
52
|
end
|
52
53
|
|
53
54
|
def process(message)
|
54
|
-
|
55
|
+
logger.trace("process: start")
|
55
56
|
@processor.process(message)
|
56
|
-
|
57
|
+
logger.trace("process: done")
|
57
58
|
end
|
58
59
|
|
59
60
|
private
|
60
61
|
def ensure_database
|
62
|
+
enforce_umask
|
61
63
|
database_path = @options[:database]
|
62
64
|
return if File.exist?(database_path)
|
63
65
|
FileUtils.mkdir_p(File.dirname(database_path))
|
@@ -70,12 +72,16 @@ module Droonga
|
|
70
72
|
end
|
71
73
|
end
|
72
74
|
|
75
|
+
def enforce_umask
|
76
|
+
File.umask(022)
|
77
|
+
end
|
78
|
+
|
73
79
|
def start_supervisor
|
74
80
|
@supervisor = ServerEngine::Supervisor.new(Server, Worker) do
|
75
81
|
force_options = {
|
76
82
|
:worker_type => "process",
|
77
83
|
:workers => @options[:n_workers],
|
78
|
-
:log_level =>
|
84
|
+
:log_level => logger.level,
|
79
85
|
:server_process_name => "Server[#{@options[:database]}] #$0",
|
80
86
|
:worker_process_name => "Worker[#{@options[:database]}] #$0",
|
81
87
|
:message_receiver => @message_pusher.raw_receiver,
|
@@ -88,11 +94,16 @@ module Droonga
|
|
88
94
|
end
|
89
95
|
|
90
96
|
def shutdown_supervisor
|
91
|
-
|
97
|
+
logger.trace("supervisor: shutdown: start")
|
92
98
|
@supervisor.stop(true)
|
93
|
-
|
99
|
+
logger.trace("supervisor: shutdown: stopped")
|
94
100
|
@supervisor_thread.join
|
95
|
-
|
101
|
+
logger.trace("supervisor: shutdown: done")
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
def log_tag
|
106
|
+
"slice"
|
96
107
|
end
|
97
108
|
end
|
98
109
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2014 Droonga Project
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
+
|
18
|
+
module Droonga
|
19
|
+
module StatusCode
|
20
|
+
OK = 200
|
21
|
+
BAD_REQUEST = 400
|
22
|
+
NOT_FOUND = 404
|
23
|
+
INTERNAL_SERVER_ERROR = 500
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/loggable"
|
17
|
+
require "droonga/plugin"
|
18
|
+
require "droonga/single_step"
|
19
|
+
|
20
|
+
module Droonga
|
21
|
+
class StepRunner
|
22
|
+
include Loggable
|
23
|
+
|
24
|
+
def initialize(plugins)
|
25
|
+
@definitions = {}
|
26
|
+
plugins.each do |name|
|
27
|
+
plugin = Plugin.registry[name]
|
28
|
+
plugin.single_step_definitions.each do |definition|
|
29
|
+
@definitions[definition.name] = definition
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def shutdown
|
35
|
+
end
|
36
|
+
|
37
|
+
def plan(message)
|
38
|
+
type = message["type"]
|
39
|
+
logger.trace("plan: start",
|
40
|
+
:dataset => message["dataset"],
|
41
|
+
:type => type)
|
42
|
+
definition = find(type)
|
43
|
+
if definition.nil?
|
44
|
+
raise UnsupportedMessageError.new(:planner, message)
|
45
|
+
end
|
46
|
+
step = SingleStep.new(definition)
|
47
|
+
plan = step.plan(message)
|
48
|
+
logger.trace("plan: done",
|
49
|
+
:dataset => message["dataset"],
|
50
|
+
:type => type)
|
51
|
+
plan
|
52
|
+
end
|
53
|
+
|
54
|
+
def find(type)
|
55
|
+
@definitions[type]
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
def log_tag
|
60
|
+
"step-runner"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/droonga/watch_schema.rb
CHANGED
@@ -17,20 +17,24 @@
|
|
17
17
|
|
18
18
|
require "groonga"
|
19
19
|
|
20
|
+
require "droonga/loggable"
|
21
|
+
|
20
22
|
module Droonga
|
21
23
|
class WatchSchema
|
24
|
+
include Loggable
|
25
|
+
|
22
26
|
def initialize(context)
|
23
27
|
@context = context
|
24
28
|
end
|
25
29
|
|
26
30
|
def ensure_created
|
27
31
|
if @context["Keyword"]
|
28
|
-
|
32
|
+
logger.trace("skip table creation")
|
29
33
|
return
|
30
34
|
end
|
31
|
-
|
35
|
+
logger.trace("ensure_tables: start")
|
32
36
|
ensure_tables
|
33
|
-
|
37
|
+
logger.trace("ensure_tables: done")
|
34
38
|
end
|
35
39
|
|
36
40
|
private
|
data/lib/droonga/watcher.rb
CHANGED
@@ -34,8 +34,8 @@ module Droonga
|
|
34
34
|
route = request[:route]
|
35
35
|
|
36
36
|
# XXX better validation and error class must be written!!
|
37
|
-
if subscriber.nil?
|
38
|
-
query.nil?
|
37
|
+
if subscriber.nil? or subscriber.empty? or condition.nil? or
|
38
|
+
query.nil? or route.nil?
|
39
39
|
raise "invalid request"
|
40
40
|
end
|
41
41
|
raise "too long query" if query.size > 4095
|
@@ -65,14 +65,14 @@ module Droonga
|
|
65
65
|
subscriber = request[:subscriber]
|
66
66
|
query = request[:query]
|
67
67
|
|
68
|
-
if subscriber.nil?
|
68
|
+
if subscriber.nil? or subscriber.empty?
|
69
69
|
raise "invalid request"
|
70
70
|
end
|
71
71
|
|
72
72
|
subscriber_record = @subscriber_table[subscriber]
|
73
73
|
return unless subscriber_record
|
74
74
|
|
75
|
-
if query.nil?
|
75
|
+
if query.nil? or query.empty?
|
76
76
|
delete_subscriber(subscriber_record)
|
77
77
|
else
|
78
78
|
query_record = @query_table[query]
|
data/lib/droonga/worker.rb
CHANGED
@@ -30,26 +30,26 @@ module Droonga
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def run
|
33
|
-
|
33
|
+
Droonga.logger.trace("#{log_tag}: run: start")
|
34
34
|
@handler_runner.start
|
35
35
|
@message_receiver.start
|
36
36
|
@loop.run
|
37
37
|
@handler_runner.shutdown
|
38
|
-
|
38
|
+
Droonga.logger.trace("#{log_tag}: run: done")
|
39
39
|
end
|
40
40
|
|
41
41
|
def stop
|
42
|
-
|
42
|
+
Droonga.logger.trace("#{log_tag}: stop: start")
|
43
43
|
@message_receiver.shutdown
|
44
44
|
@loop.stop
|
45
|
-
|
45
|
+
Droonga.logger.trace("#{log_tag}: stop: done")
|
46
46
|
end
|
47
47
|
|
48
48
|
private
|
49
49
|
def process(message)
|
50
|
-
|
50
|
+
Droonga.logger.trace("#{log_tag}: process: start")
|
51
51
|
@handler_runner.process(message)
|
52
|
-
|
52
|
+
Droonga.logger.trace("#{log_tag}: process: done")
|
53
53
|
end
|
54
54
|
|
55
55
|
def log_tag
|
@@ -17,6 +17,7 @@
|
|
17
17
|
|
18
18
|
require "droonga/engine"
|
19
19
|
require "droonga/plugin_loader"
|
20
|
+
require "droonga/catalog_observer"
|
20
21
|
|
21
22
|
module Fluent
|
22
23
|
class DroongaOutput < Output
|
@@ -27,12 +28,20 @@ module Fluent
|
|
27
28
|
def start
|
28
29
|
super
|
29
30
|
Droonga::PluginLoader.load_all
|
30
|
-
@
|
31
|
+
@catalog_observer = Droonga::CatalogObserver.new
|
32
|
+
@catalog_observer.on_reload = lambda do |catalog|
|
33
|
+
graceful_engine_restart(catalog)
|
34
|
+
$log.info("engine restarted")
|
35
|
+
end
|
36
|
+
@catalog_observer.start
|
37
|
+
catalog = @catalog_observer.catalog
|
38
|
+
@engine = create_engine(catalog)
|
31
39
|
@engine.start
|
32
40
|
end
|
33
41
|
|
34
42
|
def shutdown
|
35
43
|
@engine.shutdown
|
44
|
+
@catalog_observer.stop
|
36
45
|
super
|
37
46
|
end
|
38
47
|
|
@@ -44,6 +53,22 @@ module Fluent
|
|
44
53
|
end
|
45
54
|
|
46
55
|
private
|
56
|
+
def create_engine(catalog)
|
57
|
+
Droonga::Engine.new(catalog, :name => @name)
|
58
|
+
end
|
59
|
+
|
60
|
+
def graceful_engine_restart(catalog)
|
61
|
+
$log.trace("out_droonga: start: graceful_engine_restart")
|
62
|
+
old_engine = @engine
|
63
|
+
$log.trace("out_droonga: creating new engine")
|
64
|
+
new_engine = create_engine(catalog)
|
65
|
+
new_engine.start
|
66
|
+
@engine = new_engine
|
67
|
+
$log.trace("out_droonga: shutdown old engine")
|
68
|
+
old_engine.shutdown
|
69
|
+
$log.trace("out_droonga: done: graceful_engine_restart")
|
70
|
+
end
|
71
|
+
|
47
72
|
def process_event(tag, record)
|
48
73
|
$log.trace("out_droonga: tag: <#{tag}>")
|
49
74
|
@engine.process(parse_record(tag, record))
|
@@ -51,7 +76,7 @@ module Fluent
|
|
51
76
|
|
52
77
|
def parse_record(tag, record)
|
53
78
|
prefix, type, *arguments = tag.split(/\./)
|
54
|
-
if type.nil?
|
79
|
+
if type.nil? or type.empty? or type == "message"
|
55
80
|
message = record
|
56
81
|
else
|
57
82
|
message = {
|