fluentd 0.10.36 → 0.10.37
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.travis.yml +2 -2
- data/ChangeLog +8 -0
- data/lib/fluent/config.rb +285 -289
- data/lib/fluent/config_dsl.rb +74 -0
- data/lib/fluent/engine.rb +250 -247
- data/lib/fluent/env.rb +5 -5
- data/lib/fluent/event.rb +120 -124
- data/lib/fluent/input.rb +13 -17
- data/lib/fluent/load.rb +1 -0
- data/lib/fluent/log.rb +238 -242
- data/lib/fluent/match.rb +132 -137
- data/lib/fluent/mixin.rb +151 -155
- data/lib/fluent/parser.rb +201 -205
- data/lib/fluent/plugin.rb +117 -121
- data/lib/fluent/plugin/buf_file.rb +8 -0
- data/lib/fluent/plugin/exec_util.rb +49 -0
- data/lib/fluent/plugin/in_exec.rb +44 -30
- data/lib/fluent/plugin/in_forward.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +5 -0
- data/lib/fluent/plugin/in_stream.rb +5 -2
- data/lib/fluent/plugin/out_exec_filter.rb +4 -47
- data/lib/fluent/plugin/out_stdout.rb +21 -1
- data/lib/fluent/process.rb +358 -362
- data/lib/fluent/status.rb +25 -30
- data/lib/fluent/supervisor.rb +277 -281
- data/lib/fluent/test/base.rb +35 -39
- data/lib/fluent/test/input_test.rb +68 -63
- data/lib/fluent/test/output_test.rb +98 -101
- data/lib/fluent/version.rb +1 -1
- data/test/configdsl.rb +77 -0
- data/test/plugin/in_exec.rb +73 -13
- data/test/plugin/in_gc_stat.rb +1 -1
- data/test/plugin/in_object_space.rb +2 -2
- data/test/plugin/out_stdout.rb +45 -2
- data/test/scripts/exec_script.rb +26 -0
- metadata +31 -46
@@ -124,18 +124,20 @@ module Fluent
|
|
124
124
|
# Forward
|
125
125
|
es = MultiEventStream.new
|
126
126
|
entries.each {|e|
|
127
|
+
record = e[1]
|
128
|
+
next if record.nil?
|
127
129
|
time = e[0].to_i
|
128
130
|
time = (now ||= Engine.now) if time == 0
|
129
|
-
record = e[1]
|
130
131
|
es.add(time, record)
|
131
132
|
}
|
132
133
|
Engine.emit_stream(tag, es)
|
133
134
|
|
134
135
|
else
|
135
136
|
# Message
|
137
|
+
record = msg[2]
|
138
|
+
return if record.nil?
|
136
139
|
time = msg[1]
|
137
140
|
time = Engine.now if time == 0
|
138
|
-
record = msg[2]
|
139
141
|
Engine.emit(tag, time, record)
|
140
142
|
end
|
141
143
|
end
|
@@ -117,6 +117,11 @@ module Fluent
|
|
117
117
|
raise "'json' or 'msgpack' parameter is required"
|
118
118
|
end
|
119
119
|
|
120
|
+
# Skip nil record
|
121
|
+
if record.nil?
|
122
|
+
return ["200 OK", {'Content-type'=>'text/plain'}, ""]
|
123
|
+
end
|
124
|
+
|
120
125
|
time = params['time']
|
121
126
|
time = time.to_i
|
122
127
|
if time == 0
|
@@ -84,18 +84,21 @@ module Fluent
|
|
84
84
|
# Forward
|
85
85
|
es = MultiEventStream.new
|
86
86
|
entries.each {|e|
|
87
|
+
record = e[1]
|
88
|
+
next if record.nil?
|
87
89
|
time = e[0].to_i
|
88
90
|
time = (now ||= Engine.now) if time == 0
|
89
|
-
record = e[1]
|
90
91
|
es.add(time, record)
|
91
92
|
}
|
92
93
|
Engine.emit_stream(tag, es)
|
93
94
|
|
94
95
|
else
|
95
96
|
# Message
|
97
|
+
record = msg[2]
|
98
|
+
return if record.nil?
|
99
|
+
|
96
100
|
time = msg[1]
|
97
101
|
time = Engine.now if time == 0
|
98
|
-
record = msg[2]
|
99
102
|
Engine.emit(tag, time, record)
|
100
103
|
end
|
101
104
|
end
|
@@ -21,6 +21,7 @@ module Fluent
|
|
21
21
|
|
22
22
|
def initialize
|
23
23
|
super
|
24
|
+
require 'fluent/plugin/exec_util'
|
24
25
|
end
|
25
26
|
|
26
27
|
SUPPORTED_FORMAT = {
|
@@ -151,11 +152,11 @@ module Fluent
|
|
151
152
|
if @out_keys.empty?
|
152
153
|
raise ConfigError, "out_keys option is required on exec_filter output for tsv in_format"
|
153
154
|
end
|
154
|
-
@parser = TSVParser.new(@out_keys, method(:on_message))
|
155
|
+
@parser = ExecUtil::TSVParser.new(@out_keys, method(:on_message))
|
155
156
|
when :json
|
156
|
-
@parser = JSONParser.new(method(:on_message))
|
157
|
+
@parser = ExecUtil::JSONParser.new(method(:on_message))
|
157
158
|
when :msgpack
|
158
|
-
@parser = MessagePackParser.new(method(:on_message))
|
159
|
+
@parser = ExecUtil::MessagePackParser.new(method(:on_message))
|
159
160
|
end
|
160
161
|
|
161
162
|
@respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
|
@@ -389,50 +390,6 @@ module Fluent
|
|
389
390
|
@next_log_time = Time.now.to_i + @suppress_error_log_interval
|
390
391
|
end
|
391
392
|
end
|
392
|
-
|
393
|
-
class Parser
|
394
|
-
def initialize(on_message)
|
395
|
-
@on_message = on_message
|
396
|
-
end
|
397
|
-
end
|
398
|
-
|
399
|
-
class TSVParser < Parser
|
400
|
-
def initialize(out_keys, on_message)
|
401
|
-
@out_keys = out_keys
|
402
|
-
super(on_message)
|
403
|
-
end
|
404
|
-
|
405
|
-
def call(io)
|
406
|
-
io.each_line(&method(:each_line))
|
407
|
-
end
|
408
|
-
|
409
|
-
def each_line(line)
|
410
|
-
line.chomp!
|
411
|
-
vals = line.split("\t")
|
412
|
-
|
413
|
-
record = Hash[@out_keys.zip(vals)]
|
414
|
-
|
415
|
-
@on_message.call(record)
|
416
|
-
end
|
417
|
-
end
|
418
|
-
|
419
|
-
class JSONParser < Parser
|
420
|
-
def call(io)
|
421
|
-
y = Yajl::Parser.new
|
422
|
-
y.on_parse_complete = @on_message
|
423
|
-
y.parse(io)
|
424
|
-
end
|
425
|
-
end
|
426
|
-
|
427
|
-
class MessagePackParser < Parser
|
428
|
-
def call(io)
|
429
|
-
@u = MessagePack::Unpacker.new(io)
|
430
|
-
begin
|
431
|
-
@u.each(&@on_message)
|
432
|
-
rescue EOFError
|
433
|
-
end
|
434
|
-
end
|
435
|
-
end
|
436
393
|
end
|
437
394
|
end
|
438
395
|
|
@@ -19,9 +19,29 @@ module Fluent
|
|
19
19
|
class StdoutOutput < Output
|
20
20
|
Plugin.register_output('stdout', self)
|
21
21
|
|
22
|
+
OUTPUT_PROCS = {
|
23
|
+
:json => Proc.new {|record| Yajl.dump(record) },
|
24
|
+
:hash => Proc.new {|record| record.to_s },
|
25
|
+
}
|
26
|
+
|
27
|
+
config_param :output_type, :default => :json do |val|
|
28
|
+
case val.downcase
|
29
|
+
when 'json'
|
30
|
+
:json
|
31
|
+
when 'hash'
|
32
|
+
:hash
|
33
|
+
else
|
34
|
+
raise ConfigError, "stdout output output_type should be 'json' or 'hash'"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
def configure(conf)
|
38
|
+
super
|
39
|
+
@output_proc = OUTPUT_PROCS[@output_type]
|
40
|
+
end
|
41
|
+
|
22
42
|
def emit(tag, es, chain)
|
23
43
|
es.each {|time,record|
|
24
|
-
$log.write "#{Time.at(time).localtime} #{tag}: #{
|
44
|
+
$log.write "#{Time.at(time).localtime} #{tag}: #{@output_proc.call(record)}\n"
|
25
45
|
}
|
26
46
|
$log.flush
|
27
47
|
|
data/lib/fluent/process.rb
CHANGED
@@ -16,462 +16,458 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class DetachProcessManager
|
20
|
+
require 'singleton'
|
21
|
+
include Singleton
|
19
22
|
|
23
|
+
class Broker
|
24
|
+
def initialize
|
25
|
+
end
|
20
26
|
|
21
|
-
|
22
|
-
|
23
|
-
|
27
|
+
def engine
|
28
|
+
Engine
|
29
|
+
end
|
30
|
+
end
|
24
31
|
|
25
|
-
class Broker
|
26
32
|
def initialize
|
33
|
+
require 'drb'
|
34
|
+
DRb.start_service(create_drb_uri, Broker.new)
|
35
|
+
@parent_uri = DRb.uri
|
27
36
|
end
|
28
37
|
|
29
|
-
def
|
30
|
-
Engine
|
31
|
-
|
32
|
-
end
|
38
|
+
def fork(delegate_object)
|
39
|
+
ipr, ipw = IO.pipe # child Engine.emit_stream -> parent Engine.emit_stream
|
40
|
+
opr, opw = IO.pipe # parent target.emit -> child target.emit
|
33
41
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
42
|
+
pid = Process.fork
|
43
|
+
if pid
|
44
|
+
# parent process
|
45
|
+
ipw.close
|
46
|
+
opr.close
|
47
|
+
forward_thread = process_parent(ipr, opw, pid, delegate_object)
|
48
|
+
return pid, forward_thread
|
49
|
+
end
|
39
50
|
|
40
|
-
|
41
|
-
|
42
|
-
|
51
|
+
# child process
|
52
|
+
ipr.close
|
53
|
+
opw.close
|
54
|
+
forward_thread = process_child(ipw, opr, delegate_object)
|
55
|
+
return nil, forward_thread
|
56
|
+
end
|
43
57
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
opr.close
|
49
|
-
forward_thread = process_parent(ipr, opw, pid, delegate_object)
|
50
|
-
return pid, forward_thread
|
58
|
+
private
|
59
|
+
def read_header(ipr)
|
60
|
+
sz = ipr.read(4).unpack('N')[0]
|
61
|
+
ipr.read(sz)
|
51
62
|
end
|
52
63
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
end
|
64
|
+
def send_header(ipw, data)
|
65
|
+
ipw.write [data.bytesize].pack('N')
|
66
|
+
ipw.write data
|
67
|
+
ipw.flush
|
68
|
+
end
|
59
69
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
ipr.read(sz)
|
64
|
-
end
|
70
|
+
def create_drb_uri
|
71
|
+
"drbunix:" # TODO
|
72
|
+
end
|
65
73
|
|
66
|
-
|
67
|
-
ipw
|
68
|
-
|
69
|
-
|
70
|
-
end
|
74
|
+
private
|
75
|
+
def process_child(ipw, opr, delegate_object)
|
76
|
+
DRb.start_service(create_drb_uri, delegate_object)
|
77
|
+
child_uri = DRb.uri
|
71
78
|
|
72
|
-
|
73
|
-
|
74
|
-
|
79
|
+
send_header(ipw, child_uri)
|
80
|
+
|
81
|
+
# override target.emit_stream to write event stream to the pipe
|
82
|
+
fwd = new_forwarder(ipw, 0.5) # TODO interval
|
83
|
+
Engine.define_singleton_method(:emit_stream) do |tag,es|
|
84
|
+
fwd.emit(tag, es)
|
85
|
+
end
|
75
86
|
|
76
|
-
|
77
|
-
|
78
|
-
DRb.start_service(create_drb_uri, delegate_object)
|
79
|
-
child_uri = DRb.uri
|
87
|
+
# read event stream from the pipe and forward to target.emit
|
88
|
+
forward_thread = Thread.new(opr, delegate_object, &method(:output_forward_main))
|
80
89
|
|
81
|
-
|
90
|
+
# override global methods to call parent process
|
91
|
+
override_shared_methods(@parent_uri)
|
82
92
|
|
83
|
-
|
84
|
-
fwd = new_forwarder(ipw, 0.5) # TODO interval
|
85
|
-
Engine.define_singleton_method(:emit_stream) do |tag,es|
|
86
|
-
fwd.emit(tag, es)
|
93
|
+
return forward_thread
|
87
94
|
end
|
88
95
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
96
|
+
def override_shared_methods(parent_uri)
|
97
|
+
broker = DRbObject.new_with_uri(parent_uri)
|
98
|
+
shared_methods.each {|(broker_accessor,target,name)|
|
99
|
+
remote = broker.send(broker_accessor)
|
100
|
+
target.define_singleton_method(name) do |*args,&block|
|
101
|
+
remote.send(name, *args, &block)
|
102
|
+
end
|
103
|
+
}
|
104
|
+
end
|
94
105
|
|
95
|
-
|
96
|
-
|
106
|
+
def shared_methods
|
107
|
+
[
|
108
|
+
#[:engine, Engine, :flush!],
|
109
|
+
#[:engine, Engine, :stop],
|
110
|
+
]
|
111
|
+
end
|
97
112
|
|
98
|
-
|
99
|
-
|
100
|
-
shared_methods.each {|(broker_accessor,target,name)|
|
101
|
-
remote = broker.send(broker_accessor)
|
102
|
-
target.define_singleton_method(name) do |*args,&block|
|
103
|
-
remote.send(name, *args, &block)
|
104
|
-
end
|
105
|
-
}
|
106
|
-
end
|
113
|
+
def process_parent(ipr, opw, pid, delegate_object)
|
114
|
+
child_uri = read_header(ipr)
|
107
115
|
|
108
|
-
|
109
|
-
|
110
|
-
#[:engine, Engine, :flush!],
|
111
|
-
#[:engine, Engine, :stop],
|
112
|
-
]
|
113
|
-
end
|
116
|
+
# read event stream from the pipe and forward to Engine.emit_stream
|
117
|
+
forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
|
114
118
|
|
115
|
-
|
116
|
-
|
119
|
+
# note: don't override methods in parent process
|
120
|
+
# because another process may fork after overriding
|
121
|
+
#override_delegate_methods(delegate_object, child_uri)
|
117
122
|
|
118
|
-
|
119
|
-
|
123
|
+
# return forwarder for DetachProcessMixin to
|
124
|
+
# override target.emit and write event stream to the pipe
|
125
|
+
fwd = new_forwarder(opw, 0.5) # TODO interval
|
126
|
+
# note: override emit method on DetachProcessMixin
|
127
|
+
forward_thread.define_singleton_method(:forwarder) do
|
128
|
+
fwd
|
129
|
+
end
|
120
130
|
|
121
|
-
|
122
|
-
|
123
|
-
#override_delegate_methods(delegate_object, child_uri)
|
131
|
+
return forward_thread
|
132
|
+
end
|
124
133
|
|
125
|
-
#
|
126
|
-
#
|
127
|
-
|
128
|
-
#
|
129
|
-
|
130
|
-
|
134
|
+
#def override_delegate_methods(target, child_uri)
|
135
|
+
# remote = DRbObject.new_with_uri(child_uri)
|
136
|
+
# delegate_methods(target).each {|name|
|
137
|
+
# target.define_singleton_method(name) do |*args,&block|
|
138
|
+
# remote.send(name, *args, &block)
|
139
|
+
# end
|
140
|
+
# }
|
141
|
+
#end
|
142
|
+
#
|
143
|
+
#def delegate_methods(target)
|
144
|
+
# target.methods - Object.public_instance_methods
|
145
|
+
#end
|
146
|
+
|
147
|
+
def output_forward_main(opr, target)
|
148
|
+
read_event_stream(opr) {|tag,es|
|
149
|
+
# FIXME error handling
|
150
|
+
begin
|
151
|
+
target.emit(tag, es, NullOutputChain.instance)
|
152
|
+
rescue
|
153
|
+
$log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
|
154
|
+
$log.warn_backtrace
|
155
|
+
end
|
156
|
+
}
|
157
|
+
rescue
|
158
|
+
$log.error "error on output process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
|
159
|
+
$log.error_backtrace
|
160
|
+
raise
|
131
161
|
end
|
132
162
|
|
133
|
-
|
134
|
-
|
163
|
+
def input_forward_main(ipr, pid)
|
164
|
+
read_event_stream(ipr) {|tag,es|
|
165
|
+
# FIXME error handling
|
166
|
+
begin
|
167
|
+
Engine.emit_stream(tag, es)
|
168
|
+
rescue
|
169
|
+
$log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
|
170
|
+
$log.warn_backtrace
|
171
|
+
end
|
172
|
+
}
|
173
|
+
rescue
|
174
|
+
$log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
|
175
|
+
$log.error_backtrace
|
176
|
+
raise
|
177
|
+
end
|
135
178
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
# target.define_singleton_method(name) do |*args,&block|
|
140
|
-
# remote.send(name, *args, &block)
|
141
|
-
# end
|
142
|
-
# }
|
143
|
-
#end
|
144
|
-
#
|
145
|
-
#def delegate_methods(target)
|
146
|
-
# target.methods - Object.public_instance_methods
|
147
|
-
#end
|
148
|
-
|
149
|
-
def output_forward_main(opr, target)
|
150
|
-
read_event_stream(opr) {|tag,es|
|
151
|
-
# FIXME error handling
|
179
|
+
def read_event_stream(r, &block)
|
180
|
+
u = MessagePack::Unpacker.new(r)
|
181
|
+
cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
|
152
182
|
begin
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
183
|
+
#buf = ''
|
184
|
+
#map = {}
|
185
|
+
#while true
|
186
|
+
# r.readpartial(64*1024, buf)
|
187
|
+
# u.feed_each(buf) {|tag,ms|
|
188
|
+
# if msbuf = map[tag]
|
189
|
+
# msbuf << ms
|
190
|
+
# else
|
191
|
+
# map[tag] = ms
|
192
|
+
# end
|
193
|
+
# }
|
194
|
+
# unless map.empty?
|
195
|
+
# map.each_pair {|tag,ms|
|
196
|
+
# es = MessagePackEventStream.new(ms, cached_unpacker)
|
197
|
+
# block.call(tag, es)
|
198
|
+
# }
|
199
|
+
# map.clear
|
200
|
+
# end
|
201
|
+
#end
|
202
|
+
u.each {|tag,ms|
|
203
|
+
es = MessagePackEventStream.new(ms, cached_unpacker)
|
204
|
+
block.call(tag, es)
|
205
|
+
}
|
206
|
+
rescue EOFError
|
207
|
+
ensure
|
208
|
+
r.close
|
157
209
|
end
|
158
|
-
|
159
|
-
rescue
|
160
|
-
$log.error "error on output process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
|
161
|
-
$log.error_backtrace
|
162
|
-
raise
|
163
|
-
end
|
210
|
+
end
|
164
211
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
rescue
|
171
|
-
$log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
|
172
|
-
$log.warn_backtrace
|
212
|
+
def new_forwarder(w, interval)
|
213
|
+
if interval < 0.2 # TODO interval
|
214
|
+
Forwarder.new(w)
|
215
|
+
else
|
216
|
+
DelayedForwarder.new(w, interval)
|
173
217
|
end
|
174
|
-
}
|
175
|
-
rescue
|
176
|
-
$log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
|
177
|
-
$log.error_backtrace
|
178
|
-
raise
|
179
|
-
end
|
180
|
-
|
181
|
-
def read_event_stream(r, &block)
|
182
|
-
u = MessagePack::Unpacker.new(r)
|
183
|
-
cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
|
184
|
-
begin
|
185
|
-
#buf = ''
|
186
|
-
#map = {}
|
187
|
-
#while true
|
188
|
-
# r.readpartial(64*1024, buf)
|
189
|
-
# u.feed_each(buf) {|tag,ms|
|
190
|
-
# if msbuf = map[tag]
|
191
|
-
# msbuf << ms
|
192
|
-
# else
|
193
|
-
# map[tag] = ms
|
194
|
-
# end
|
195
|
-
# }
|
196
|
-
# unless map.empty?
|
197
|
-
# map.each_pair {|tag,ms|
|
198
|
-
# es = MessagePackEventStream.new(ms, cached_unpacker)
|
199
|
-
# block.call(tag, es)
|
200
|
-
# }
|
201
|
-
# map.clear
|
202
|
-
# end
|
203
|
-
#end
|
204
|
-
u.each {|tag,ms|
|
205
|
-
es = MessagePackEventStream.new(ms, cached_unpacker)
|
206
|
-
block.call(tag, es)
|
207
|
-
}
|
208
|
-
rescue EOFError
|
209
|
-
ensure
|
210
|
-
r.close
|
211
218
|
end
|
212
|
-
end
|
213
219
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
DelayedForwarder.new(w, interval)
|
219
|
-
end
|
220
|
-
end
|
220
|
+
class Forwarder
|
221
|
+
def initialize(w)
|
222
|
+
@w = w
|
223
|
+
end
|
221
224
|
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
+
def emit(tag, es)
|
226
|
+
ms = es.to_msgpack_stream
|
227
|
+
#[tag, ms].to_msgpack(@w) # not thread safe
|
228
|
+
@w.write [tag, ms].to_msgpack
|
229
|
+
end
|
225
230
|
end
|
226
231
|
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
232
|
+
class DelayedForwarder
|
233
|
+
def initialize(w, interval)
|
234
|
+
@w = w
|
235
|
+
@interval = interval
|
236
|
+
@buffer = {}
|
237
|
+
Thread.new(&method(:run))
|
238
|
+
end
|
233
239
|
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
240
|
+
def emit(tag, es)
|
241
|
+
if ms = @buffer[tag]
|
242
|
+
ms << es.to_msgpack_stream
|
243
|
+
else
|
244
|
+
@buffer[tag] = es.to_msgpack_stream
|
245
|
+
end
|
246
|
+
end
|
241
247
|
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
248
|
+
def run
|
249
|
+
while true
|
250
|
+
sleep @interval
|
251
|
+
@buffer.keys.each {|tag|
|
252
|
+
if ms = @buffer.delete(tag)
|
253
|
+
[tag, ms].to_msgpack(@w)
|
254
|
+
#@w.write [tag, ms].to_msgpack
|
255
|
+
end
|
256
|
+
}
|
257
|
+
end
|
258
|
+
rescue
|
259
|
+
$log.error "error on forwerder thread", :error=>$!.to_s
|
260
|
+
$log.error_backtrace
|
261
|
+
raise
|
247
262
|
end
|
248
263
|
end
|
249
264
|
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
@
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
265
|
+
class MultiForwarder
|
266
|
+
def initialize(forwarders)
|
267
|
+
@forwarders = forwarders
|
268
|
+
@rr = 1
|
269
|
+
end
|
270
|
+
|
271
|
+
def emit(tag, es)
|
272
|
+
forwarder = @forwarders[@rr]
|
273
|
+
@rr = (@rr + 1) % @forwarders.length
|
274
|
+
forwarder.emit(tag, es)
|
259
275
|
end
|
260
|
-
rescue
|
261
|
-
$log.error "error on forwerder thread", :error=>$!.to_s
|
262
|
-
$log.error_backtrace
|
263
|
-
raise
|
264
276
|
end
|
265
277
|
end
|
266
278
|
|
267
|
-
class MultiForwarder
|
268
|
-
def initialize(forwarders)
|
269
|
-
@forwarders = forwarders
|
270
|
-
@rr = 1
|
271
|
-
end
|
272
279
|
|
273
|
-
|
274
|
-
|
275
|
-
@rr = (@rr + 1) % @forwarders.length
|
276
|
-
forwarder.emit(tag, es)
|
280
|
+
module DetachProcessImpl
|
281
|
+
def on_detach_process(i)
|
277
282
|
end
|
278
|
-
end
|
279
|
-
end
|
280
283
|
|
284
|
+
protected
|
285
|
+
def detach_process_impl(num, &block)
|
286
|
+
children = []
|
281
287
|
|
282
|
-
|
283
|
-
|
284
|
-
end
|
285
|
-
|
286
|
-
protected
|
287
|
-
def detach_process_impl(num, &block)
|
288
|
-
children = []
|
288
|
+
num.times do |i|
|
289
|
+
pid, forward_thread = DetachProcessManager.instance.fork(self)
|
289
290
|
|
290
|
-
|
291
|
-
|
291
|
+
if pid
|
292
|
+
# parent process
|
293
|
+
$log.info "detached process", :class=>self.class, :pid=>pid
|
294
|
+
children << [pid, forward_thread]
|
295
|
+
next
|
296
|
+
end
|
292
297
|
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
children << [pid, forward_thread]
|
297
|
-
next
|
298
|
-
end
|
298
|
+
# child process
|
299
|
+
begin
|
300
|
+
on_detach_process(i)
|
299
301
|
|
300
|
-
|
301
|
-
begin
|
302
|
-
on_detach_process(i)
|
302
|
+
block.call
|
303
303
|
|
304
|
-
|
304
|
+
# disable Engine.stop called by signal handler
|
305
|
+
Engine.define_singleton_method(:stop) do
|
306
|
+
# do nothing
|
307
|
+
end
|
305
308
|
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
309
|
+
# override signal handlers called by parent process
|
310
|
+
fin = FinishWait.new
|
311
|
+
trap :INT do
|
312
|
+
fin.stop
|
313
|
+
end
|
314
|
+
trap :TERM do
|
315
|
+
fin.stop
|
316
|
+
end
|
317
|
+
#forward_thread.join # TODO this thread won't stop because parent doesn't close pipe
|
318
|
+
fin.wait
|
310
319
|
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
end
|
316
|
-
trap :TERM do
|
317
|
-
fin.stop
|
320
|
+
exit! 0
|
321
|
+
ensure
|
322
|
+
$log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
|
323
|
+
$log.error_backtrace
|
318
324
|
end
|
319
|
-
#forward_thread.join # TODO this thread won't stop because parent doesn't close pipe
|
320
|
-
fin.wait
|
321
325
|
|
322
|
-
exit!
|
323
|
-
ensure
|
324
|
-
$log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
|
325
|
-
$log.error_backtrace
|
326
|
+
exit! 1
|
326
327
|
end
|
327
328
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
329
|
+
# parent process
|
330
|
+
# override shutdown method to kill child processes
|
331
|
+
define_singleton_method(:shutdown) do
|
332
|
+
children.each {|pair|
|
333
|
+
begin
|
334
|
+
pid = pair[0]
|
335
|
+
forward_thread = pair[1]
|
336
|
+
if pid
|
337
|
+
Process.kill(:TERM, pid)
|
338
|
+
forward_thread.join # wait until child closes pipe
|
339
|
+
Process.waitpid(pid)
|
340
|
+
pair[0] = nil
|
341
|
+
end
|
342
|
+
rescue
|
343
|
+
$log.error "unknown error while shutting down remote child process", :error=>$!.to_s
|
344
|
+
$log.error_backtrace
|
343
345
|
end
|
344
|
-
|
345
|
-
|
346
|
-
$log.error_backtrace
|
347
|
-
end
|
348
|
-
}
|
349
|
-
end
|
346
|
+
}
|
347
|
+
end
|
350
348
|
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
349
|
+
# override target.emit and write event stream to the pipe
|
350
|
+
forwarders = children.map {|pair| pair[1].forwarder }
|
351
|
+
if forwarders.length > 1
|
352
|
+
# use roundrobin
|
353
|
+
fwd = DetachProcessManager::MultiForwarder.new(forwarders)
|
354
|
+
else
|
355
|
+
fwd = forwarders[0]
|
356
|
+
end
|
357
|
+
define_singleton_method(:emit) do |tag,es,chain|
|
358
|
+
chain.next
|
359
|
+
fwd.emit(tag, es)
|
360
|
+
end
|
362
361
|
end
|
363
|
-
end
|
364
362
|
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
363
|
+
class FinishWait
|
364
|
+
def initialize
|
365
|
+
@finished = false
|
366
|
+
@mutex = Mutex.new
|
367
|
+
@cond = ConditionVariable.new
|
368
|
+
end
|
371
369
|
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
370
|
+
def wait
|
371
|
+
@mutex.synchronize do
|
372
|
+
until @finished
|
373
|
+
@cond.wait(@mutex, 1.0)
|
374
|
+
end
|
376
375
|
end
|
377
376
|
end
|
378
|
-
end
|
379
377
|
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
378
|
+
def stop
|
379
|
+
return if @finished
|
380
|
+
@finished = true
|
381
|
+
@mutex.synchronize do
|
382
|
+
@cond.broadcast
|
383
|
+
end
|
385
384
|
end
|
386
|
-
end
|
387
385
|
|
388
|
-
|
389
|
-
|
386
|
+
def finished?
|
387
|
+
@finished
|
388
|
+
end
|
390
389
|
end
|
391
390
|
end
|
392
|
-
end
|
393
|
-
|
394
|
-
|
395
|
-
module DetachProcessMixin
|
396
|
-
include DetachProcessImpl
|
397
391
|
|
398
|
-
def configure(conf)
|
399
|
-
super
|
400
392
|
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
393
|
+
module DetachProcessMixin
|
394
|
+
include DetachProcessImpl
|
395
|
+
|
396
|
+
def configure(conf)
|
397
|
+
super
|
398
|
+
|
399
|
+
if detach_process = conf['detach_process']
|
400
|
+
b3v = Config.bool_value(detach_process)
|
401
|
+
case b3v
|
402
|
+
when nil
|
403
|
+
num = detach_process.to_i
|
404
|
+
if num > 1
|
405
|
+
$log.warn "'detach_process' parameter supports only 1 process on this plugin: #{conf}"
|
406
|
+
elsif num > 0
|
407
|
+
@detach_process = true
|
408
|
+
elsif detach_process =~ /0+/
|
409
|
+
@detach_process = false
|
410
|
+
else
|
411
|
+
@detach_process = true
|
412
|
+
end
|
413
|
+
when true
|
409
414
|
@detach_process = true
|
410
|
-
|
415
|
+
when false
|
411
416
|
@detach_process = false
|
412
|
-
else
|
413
|
-
@detach_process = true
|
414
417
|
end
|
415
|
-
when true
|
416
|
-
@detach_process = true
|
417
|
-
when false
|
418
|
-
@detach_process = false
|
419
418
|
end
|
420
419
|
end
|
421
|
-
end
|
422
420
|
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
421
|
+
def detach_process(&block)
|
422
|
+
if @detach_process
|
423
|
+
detach_process_impl(1, &block)
|
424
|
+
else
|
425
|
+
block.call
|
426
|
+
end
|
428
427
|
end
|
429
428
|
end
|
430
|
-
end
|
431
429
|
|
432
430
|
|
433
|
-
module DetachMultiProcessMixin
|
434
|
-
|
431
|
+
module DetachMultiProcessMixin
|
432
|
+
include DetachProcessImpl
|
435
433
|
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
def configure(conf)
|
442
|
-
super
|
434
|
+
def initialize
|
435
|
+
@detach_process_num = 2
|
436
|
+
super
|
437
|
+
end
|
443
438
|
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
439
|
+
def configure(conf)
|
440
|
+
super
|
441
|
+
|
442
|
+
if detach_process = conf['detach_process']
|
443
|
+
b3v = Config.bool_value(detach_process)
|
444
|
+
case b3v
|
445
|
+
when nil
|
446
|
+
num = detach_process.to_i
|
447
|
+
if num > 0
|
448
|
+
@detach_process = true
|
449
|
+
@detach_process_num = num
|
450
|
+
elsif detach_process =~ /0+/
|
451
|
+
@detach_process = false
|
452
|
+
else
|
453
|
+
@detach_process = true
|
454
|
+
end
|
455
|
+
when true
|
450
456
|
@detach_process = true
|
451
|
-
|
452
|
-
elsif detach_process =~ /0+/
|
457
|
+
when false
|
453
458
|
@detach_process = false
|
454
|
-
else
|
455
|
-
@detach_process = true
|
456
459
|
end
|
457
|
-
when true
|
458
|
-
@detach_process = true
|
459
|
-
when false
|
460
|
-
@detach_process = false
|
461
460
|
end
|
462
461
|
end
|
463
|
-
end
|
464
462
|
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
463
|
+
protected
|
464
|
+
def detach_multi_process(&block)
|
465
|
+
if @detach_process
|
466
|
+
detach_process_impl(@detach_process_num, &block)
|
467
|
+
else
|
468
|
+
block.call
|
469
|
+
end
|
471
470
|
end
|
472
471
|
end
|
473
472
|
end
|
474
473
|
|
475
|
-
|
476
|
-
end
|
477
|
-
|