fluentd 0.10.35 → 0.10.36
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- data/.travis.yml +13 -0
- data/ChangeLog +9 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/buffer.rb +210 -214
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/config.rb +1 -0
- data/lib/fluent/engine.rb +10 -10
- data/lib/fluent/output.rb +404 -406
- data/lib/fluent/plugin/buf_file.rb +146 -151
- data/lib/fluent/plugin/buf_memory.rb +62 -67
- data/lib/fluent/plugin/in_debug_agent.rb +27 -31
- data/lib/fluent/plugin/in_exec.rb +86 -90
- data/lib/fluent/plugin/in_forward.rb +171 -171
- data/lib/fluent/plugin/in_gc_stat.rb +43 -47
- data/lib/fluent/plugin/in_http.rb +214 -216
- data/lib/fluent/plugin/in_monitor_agent.rb +212 -214
- data/lib/fluent/plugin/in_object_space.rb +75 -79
- data/lib/fluent/plugin/in_status.rb +44 -50
- data/lib/fluent/plugin/in_stream.rb +159 -160
- data/lib/fluent/plugin/in_syslog.rb +149 -153
- data/lib/fluent/plugin/in_tail.rb +382 -387
- data/lib/fluent/plugin/out_copy.rb +40 -45
- data/lib/fluent/plugin/out_exec.rb +52 -57
- data/lib/fluent/plugin/out_exec_filter.rb +327 -331
- data/lib/fluent/plugin/out_file.rb +78 -74
- data/lib/fluent/plugin/out_forward.rb +410 -414
- data/lib/fluent/plugin/out_null.rb +15 -19
- data/lib/fluent/plugin/out_roundrobin.rb +63 -68
- data/lib/fluent/plugin/out_stdout.rb +9 -14
- data/lib/fluent/plugin/out_stream.rb +83 -90
- data/lib/fluent/plugin/out_test.rb +42 -46
- data/lib/fluent/supervisor.rb +15 -0
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/in_stream.rb +2 -0
- data/test/plugin/out_file.rb +19 -1
- metadata +6 -5
@@ -16,29 +16,25 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class NullOutput < Output
|
20
|
+
Plugin.register_output('null', self)
|
19
21
|
|
22
|
+
def initialize
|
23
|
+
super
|
24
|
+
end
|
20
25
|
|
21
|
-
|
22
|
-
|
26
|
+
def configure(conf)
|
27
|
+
super
|
28
|
+
end
|
23
29
|
|
24
|
-
|
25
|
-
|
26
|
-
end
|
30
|
+
def start
|
31
|
+
end
|
27
32
|
|
28
|
-
|
29
|
-
|
30
|
-
end
|
33
|
+
def shutdown
|
34
|
+
end
|
31
35
|
|
32
|
-
|
36
|
+
def emit(tag, es, chain)
|
37
|
+
chain.next
|
38
|
+
end
|
33
39
|
end
|
34
|
-
|
35
|
-
def shutdown
|
36
|
-
end
|
37
|
-
|
38
|
-
def emit(tag, es, chain)
|
39
|
-
chain.next
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
|
44
40
|
end
|
@@ -16,85 +16,80 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class RoundRobinOutput < MultiOutput
|
20
|
+
Plugin.register_output('roundrobin', self)
|
19
21
|
|
22
|
+
def initialize
|
23
|
+
@outputs = []
|
24
|
+
@weights = []
|
25
|
+
end
|
20
26
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
output = Plugin.new_output(type)
|
46
|
-
output.configure(e)
|
47
|
-
@outputs << output
|
48
|
-
@weights << weight
|
49
|
-
}
|
50
|
-
@rr = -1 # starts from @output[0]
|
51
|
-
@rand_seed = Random.new.seed
|
52
|
-
end
|
27
|
+
attr_reader :outputs, :weights
|
28
|
+
attr_accessor :rand_seed
|
29
|
+
|
30
|
+
def configure(conf)
|
31
|
+
conf.elements.select {|e|
|
32
|
+
e.name == 'store'
|
33
|
+
}.each {|e|
|
34
|
+
type = e['type']
|
35
|
+
unless type
|
36
|
+
raise ConfigError, "Missing 'type' parameter on <store> directive"
|
37
|
+
end
|
38
|
+
|
39
|
+
weight = e['weight']
|
40
|
+
weight = weight ? weight.to_i : 1
|
41
|
+
$log.debug "adding store type=#{type.dump}, weight=#{weight}"
|
42
|
+
|
43
|
+
output = Plugin.new_output(type)
|
44
|
+
output.configure(e)
|
45
|
+
@outputs << output
|
46
|
+
@weights << weight
|
47
|
+
}
|
48
|
+
@rr = -1 # starts from @output[0]
|
49
|
+
@rand_seed = Random.new.seed
|
50
|
+
end
|
53
51
|
|
54
|
-
|
55
|
-
|
52
|
+
def start
|
53
|
+
rebuild_weight_array
|
56
54
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
55
|
+
@outputs.each {|o|
|
56
|
+
o.start
|
57
|
+
}
|
58
|
+
end
|
61
59
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
60
|
+
def shutdown
|
61
|
+
@outputs.each {|o|
|
62
|
+
o.shutdown
|
63
|
+
}
|
64
|
+
end
|
67
65
|
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
def emit(tag, es, chain)
|
67
|
+
next_output.emit(tag, es, chain)
|
68
|
+
end
|
71
69
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
70
|
+
protected
|
71
|
+
def next_output
|
72
|
+
@rr = 0 if (@rr += 1) >= @weight_array.size
|
73
|
+
@weight_array[@rr]
|
74
|
+
end
|
77
75
|
|
78
|
-
|
79
|
-
|
76
|
+
def rebuild_weight_array
|
77
|
+
gcd = @weights.inject(0) {|r,w| r.gcd(w) }
|
80
78
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
79
|
+
weight_array = []
|
80
|
+
@outputs.zip(@weights).each {|output,weight|
|
81
|
+
(weight / gcd).times {
|
82
|
+
weight_array << output
|
83
|
+
}
|
85
84
|
}
|
86
|
-
}
|
87
85
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
86
|
+
# don't randomize order if all weight is 1 (=default)
|
87
|
+
if @weights.any? {|w| w > 1 }
|
88
|
+
r = Random.new(@rand_seed)
|
89
|
+
weight_array.sort_by! { r.rand }
|
90
|
+
end
|
93
91
|
|
94
|
-
|
92
|
+
@weight_array = weight_array
|
93
|
+
end
|
95
94
|
end
|
96
95
|
end
|
97
|
-
|
98
|
-
|
99
|
-
end
|
100
|
-
|
@@ -16,21 +16,16 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class StdoutOutput < Output
|
20
|
+
Plugin.register_output('stdout', self)
|
19
21
|
|
22
|
+
def emit(tag, es, chain)
|
23
|
+
es.each {|time,record|
|
24
|
+
$log.write "#{Time.at(time).localtime} #{tag}: #{Yajl.dump(record)}\n"
|
25
|
+
}
|
26
|
+
$log.flush
|
20
27
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
def emit(tag, es, chain)
|
25
|
-
es.each {|time,record|
|
26
|
-
$log.write "#{Time.at(time).localtime} #{tag}: #{Yajl.dump(record)}\n"
|
27
|
-
}
|
28
|
-
$log.flush
|
29
|
-
|
30
|
-
chain.next
|
28
|
+
chain.next
|
29
|
+
end
|
31
30
|
end
|
32
31
|
end
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
|
@@ -16,118 +16,111 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
# obsolete
|
20
|
+
class StreamOutput < BufferedOutput
|
21
|
+
def initialize
|
22
|
+
require 'socket'
|
23
|
+
require 'fileutils'
|
24
|
+
super
|
25
|
+
end
|
19
26
|
|
27
|
+
config_param :send_timeout, :time, :default => 60
|
20
28
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
require 'socket'
|
25
|
-
require 'fileutils'
|
26
|
-
super
|
27
|
-
end
|
28
|
-
|
29
|
-
config_param :send_timeout, :time, :default => 60
|
30
|
-
|
31
|
-
def configure(conf)
|
32
|
-
super
|
33
|
-
end
|
34
|
-
|
35
|
-
def format_stream(tag, es)
|
36
|
-
# use PackedForward
|
37
|
-
[tag, es.to_msgpack_stream].to_msgpack
|
38
|
-
end
|
39
|
-
|
40
|
-
def write(chunk)
|
41
|
-
sock = connect
|
42
|
-
begin
|
43
|
-
opt = [1, @send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
|
44
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
45
|
-
|
46
|
-
opt = [@send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
47
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
29
|
+
def configure(conf)
|
30
|
+
super
|
31
|
+
end
|
48
32
|
|
49
|
-
|
50
|
-
|
51
|
-
|
33
|
+
def format_stream(tag, es)
|
34
|
+
# use PackedForward
|
35
|
+
[tag, es.to_msgpack_stream].to_msgpack
|
52
36
|
end
|
53
|
-
end
|
54
37
|
|
55
|
-
|
56
|
-
|
57
|
-
|
38
|
+
def write(chunk)
|
39
|
+
sock = connect
|
40
|
+
begin
|
41
|
+
opt = [1, @send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
|
42
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
43
|
+
|
44
|
+
opt = [@send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
45
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
46
|
+
|
47
|
+
chunk.write_to(sock)
|
48
|
+
ensure
|
49
|
+
sock.close
|
50
|
+
end
|
58
51
|
end
|
59
|
-
@buffer.pop(secondary)
|
60
|
-
end
|
61
52
|
|
62
|
-
|
63
|
-
|
64
|
-
|
53
|
+
def flush_secondary(secondary)
|
54
|
+
unless secondary.is_a?(StreamOutput)
|
55
|
+
secondary = ReformatWriter.new(secondary)
|
56
|
+
end
|
57
|
+
@buffer.pop(secondary)
|
65
58
|
end
|
66
59
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
60
|
+
class ReformatWriter
|
61
|
+
def initialize(secondary)
|
62
|
+
@secondary = secondary
|
63
|
+
end
|
64
|
+
|
65
|
+
def write(chunk)
|
66
|
+
chain = NullOutputChain.instance
|
67
|
+
chunk.open {|io|
|
68
|
+
# TODO use MessagePackIoEventStream
|
69
|
+
u = MessagePack::Unpacker.new(io)
|
70
|
+
begin
|
71
|
+
u.each {|(tag,entries)|
|
72
|
+
es = MultiEventStream.new
|
73
|
+
entries.each {|o|
|
74
|
+
es.add(o[0], o[1])
|
75
|
+
}
|
76
|
+
@secondary.emit(tag, es, chain)
|
77
77
|
}
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
}
|
78
|
+
rescue EOFError
|
79
|
+
end
|
80
|
+
}
|
81
|
+
end
|
83
82
|
end
|
84
83
|
end
|
85
|
-
end
|
86
|
-
|
87
84
|
|
88
|
-
# obsolete
|
89
|
-
class TcpOutput < StreamOutput
|
90
|
-
|
85
|
+
# obsolete
|
86
|
+
class TcpOutput < StreamOutput
|
87
|
+
Plugin.register_output('tcp', self)
|
91
88
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
89
|
+
def initialize
|
90
|
+
super
|
91
|
+
$log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
|
92
|
+
$log.warn "see 'forward' section in http://fluentd.org/doc/plugin.html for the high-availability configuration."
|
93
|
+
end
|
97
94
|
|
98
|
-
|
99
|
-
|
95
|
+
config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
|
96
|
+
config_param :host, :string
|
100
97
|
|
101
|
-
|
102
|
-
|
103
|
-
|
98
|
+
def configure(conf)
|
99
|
+
super
|
100
|
+
end
|
104
101
|
|
105
|
-
|
106
|
-
|
102
|
+
def connect
|
103
|
+
TCPSocket.new(@host, @port)
|
104
|
+
end
|
107
105
|
end
|
108
|
-
end
|
109
106
|
|
107
|
+
# obsolete
|
108
|
+
class UnixOutput < StreamOutput
|
109
|
+
Plugin.register_output('unix', self)
|
110
110
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
def initialize
|
116
|
-
super
|
117
|
-
$log.warn "'unix' output is obsoleted and will be removed."
|
118
|
-
end
|
111
|
+
def initialize
|
112
|
+
super
|
113
|
+
$log.warn "'unix' output is obsoleted and will be removed."
|
114
|
+
end
|
119
115
|
|
120
|
-
|
116
|
+
config_param :path, :string
|
121
117
|
|
122
|
-
|
123
|
-
|
124
|
-
|
118
|
+
def configure(conf)
|
119
|
+
super
|
120
|
+
end
|
125
121
|
|
126
|
-
|
127
|
-
|
122
|
+
def connect
|
123
|
+
UNIXSocket.new(@path)
|
124
|
+
end
|
128
125
|
end
|
129
126
|
end
|
130
|
-
|
131
|
-
|
132
|
-
end
|
133
|
-
|
@@ -16,63 +16,59 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class TestOutput < Output
|
20
|
+
Plugin.register_output('test', self)
|
19
21
|
|
22
|
+
def initialize
|
23
|
+
@emit_streams = []
|
24
|
+
@name = nil
|
25
|
+
end
|
20
26
|
|
21
|
-
|
22
|
-
Plugin.register_output('test', self)
|
23
|
-
|
24
|
-
def initialize
|
25
|
-
@emit_streams = []
|
26
|
-
@name = nil
|
27
|
-
end
|
28
|
-
|
29
|
-
attr_reader :emit_streams, :name
|
27
|
+
attr_reader :emit_streams, :name
|
30
28
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
29
|
+
def emits
|
30
|
+
all = []
|
31
|
+
@emit_streams.each {|tag,events|
|
32
|
+
events.each {|time,record|
|
33
|
+
all << [tag, time, record]
|
34
|
+
}
|
36
35
|
}
|
37
|
-
|
38
|
-
|
39
|
-
end
|
36
|
+
all
|
37
|
+
end
|
40
38
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
39
|
+
def events
|
40
|
+
all = []
|
41
|
+
@emit_streams.each {|tag,events|
|
42
|
+
all.concat events
|
43
|
+
}
|
44
|
+
all
|
45
|
+
end
|
48
46
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
def records
|
48
|
+
all = []
|
49
|
+
@emit_streams.each {|tag,events|
|
50
|
+
events.each {|time,record|
|
51
|
+
all << record
|
52
|
+
}
|
54
53
|
}
|
55
|
-
|
56
|
-
|
57
|
-
end
|
54
|
+
all
|
55
|
+
end
|
58
56
|
|
59
|
-
|
60
|
-
|
61
|
-
|
57
|
+
def configure(conf)
|
58
|
+
if name = conf['name']
|
59
|
+
@name = name
|
60
|
+
end
|
62
61
|
end
|
63
|
-
end
|
64
62
|
|
65
|
-
|
66
|
-
|
63
|
+
def start
|
64
|
+
end
|
67
65
|
|
68
|
-
|
69
|
-
|
66
|
+
def shutdown
|
67
|
+
end
|
70
68
|
|
71
|
-
|
72
|
-
|
73
|
-
|
69
|
+
def emit(tag, es, chain)
|
70
|
+
chain.next
|
71
|
+
@emit_streams << [tag, es.to_a]
|
72
|
+
end
|
74
73
|
end
|
75
74
|
end
|
76
|
-
|
77
|
-
|
78
|
-
end
|