fluentd 0.10.36 → 0.10.37
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.travis.yml +2 -2
- data/ChangeLog +8 -0
- data/lib/fluent/config.rb +285 -289
- data/lib/fluent/config_dsl.rb +74 -0
- data/lib/fluent/engine.rb +250 -247
- data/lib/fluent/env.rb +5 -5
- data/lib/fluent/event.rb +120 -124
- data/lib/fluent/input.rb +13 -17
- data/lib/fluent/load.rb +1 -0
- data/lib/fluent/log.rb +238 -242
- data/lib/fluent/match.rb +132 -137
- data/lib/fluent/mixin.rb +151 -155
- data/lib/fluent/parser.rb +201 -205
- data/lib/fluent/plugin.rb +117 -121
- data/lib/fluent/plugin/buf_file.rb +8 -0
- data/lib/fluent/plugin/exec_util.rb +49 -0
- data/lib/fluent/plugin/in_exec.rb +44 -30
- data/lib/fluent/plugin/in_forward.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +5 -0
- data/lib/fluent/plugin/in_stream.rb +5 -2
- data/lib/fluent/plugin/out_exec_filter.rb +4 -47
- data/lib/fluent/plugin/out_stdout.rb +21 -1
- data/lib/fluent/process.rb +358 -362
- data/lib/fluent/status.rb +25 -30
- data/lib/fluent/supervisor.rb +277 -281
- data/lib/fluent/test/base.rb +35 -39
- data/lib/fluent/test/input_test.rb +68 -63
- data/lib/fluent/test/output_test.rb +98 -101
- data/lib/fluent/version.rb +1 -1
- data/test/configdsl.rb +77 -0
- data/test/plugin/in_exec.rb +73 -13
- data/test/plugin/in_gc_stat.rb +1 -1
- data/test/plugin/in_object_space.rb +2 -2
- data/test/plugin/out_stdout.rb +45 -2
- data/test/scripts/exec_script.rb +26 -0
- metadata +31 -46
data/lib/fluent/test/base.rb
CHANGED
@@ -16,49 +16,45 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
-
module Test
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
19
|
+
module Test
|
20
|
+
class TestDriver
|
21
|
+
include ::Test::Unit::Assertions
|
22
|
+
|
23
|
+
def initialize(klass, &block)
|
24
|
+
if klass.is_a?(Class)
|
25
|
+
if block
|
26
|
+
klass = klass.dup
|
27
|
+
klass.module_eval(&block)
|
28
|
+
end
|
29
|
+
@instance = klass.new
|
30
|
+
else
|
31
|
+
@instance = klass
|
32
|
+
end
|
33
|
+
@config = Config.new
|
30
34
|
end
|
31
|
-
@instance = klass.new
|
32
|
-
else
|
33
|
-
@instance = klass
|
34
|
-
end
|
35
|
-
@config = Config.new
|
36
|
-
end
|
37
35
|
|
38
|
-
|
36
|
+
attr_reader :instance, :config
|
39
37
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
38
|
+
def configure(str)
|
39
|
+
if str.is_a?(Fluent::Config::Element)
|
40
|
+
@config = str
|
41
|
+
else
|
42
|
+
@config = Config.parse(str, "(test)")
|
43
|
+
end
|
44
|
+
@instance.configure(@config)
|
45
|
+
self
|
46
|
+
end
|
49
47
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
48
|
+
def run(&block)
|
49
|
+
@instance.start
|
50
|
+
begin
|
51
|
+
# wait until thread starts
|
52
|
+
10.times { sleep 0.05 }
|
53
|
+
return yield
|
54
|
+
ensure
|
55
|
+
@instance.shutdown
|
56
|
+
end
|
57
|
+
end
|
58
58
|
end
|
59
59
|
end
|
60
60
|
end
|
61
|
-
|
62
|
-
|
63
|
-
end
|
64
|
-
end
|
@@ -16,84 +16,89 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
-
|
19
|
+
require 'fluent/plugin/buf_file'
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@emit_streams = []
|
26
|
-
@expects = nil
|
27
|
-
end
|
28
|
-
|
29
|
-
def expect_emit(tag, time, record)
|
30
|
-
(@expects ||= []) << [tag, time, record]
|
31
|
-
self
|
32
|
-
end
|
33
|
-
|
34
|
-
def expected_emits
|
35
|
-
@expects ||= []
|
21
|
+
class FileBuffer
|
22
|
+
def self.clear_buffer_paths
|
23
|
+
@@buffer_paths = {}
|
24
|
+
end
|
36
25
|
end
|
37
26
|
|
38
|
-
|
27
|
+
module Test
|
28
|
+
class InputTestDriver < TestDriver
|
29
|
+
def initialize(klass, &block)
|
30
|
+
FileBuffer.clear_buffer_paths
|
31
|
+
super(klass, &block)
|
32
|
+
@emit_streams = []
|
33
|
+
@expects = nil
|
34
|
+
end
|
39
35
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
all << [tag, time, record]
|
45
|
-
}
|
46
|
-
}
|
47
|
-
all
|
48
|
-
end
|
36
|
+
def expect_emit(tag, time, record)
|
37
|
+
(@expects ||= []) << [tag, time, record]
|
38
|
+
self
|
39
|
+
end
|
49
40
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
all.concat events
|
54
|
-
}
|
55
|
-
all
|
56
|
-
end
|
41
|
+
def expected_emits
|
42
|
+
@expects ||= []
|
43
|
+
end
|
57
44
|
|
58
|
-
|
59
|
-
all = []
|
60
|
-
@emit_streams.each {|tag,events|
|
61
|
-
events.each {|time,record|
|
62
|
-
all << record
|
63
|
-
}
|
64
|
-
}
|
65
|
-
all
|
66
|
-
end
|
45
|
+
attr_reader :emit_streams
|
67
46
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
47
|
+
def emits
|
48
|
+
all = []
|
49
|
+
@emit_streams.each {|tag,events|
|
50
|
+
events.each {|time,record|
|
51
|
+
all << [tag, time, record]
|
52
|
+
}
|
53
|
+
}
|
54
|
+
all
|
55
|
+
end
|
74
56
|
|
75
|
-
|
57
|
+
def events
|
58
|
+
all = []
|
59
|
+
@emit_streams.each {|tag,events|
|
60
|
+
all.concat events
|
61
|
+
}
|
62
|
+
all
|
63
|
+
end
|
76
64
|
|
77
|
-
|
78
|
-
|
65
|
+
def records
|
66
|
+
all = []
|
79
67
|
@emit_streams.each {|tag,events|
|
80
68
|
events.each {|time,record|
|
81
|
-
|
82
|
-
i += 1
|
69
|
+
all << record
|
83
70
|
}
|
84
71
|
}
|
85
|
-
|
72
|
+
all
|
86
73
|
end
|
87
|
-
}
|
88
|
-
self
|
89
|
-
end
|
90
74
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
75
|
+
def run(&block)
|
76
|
+
m = method(:emit_stream)
|
77
|
+
super {
|
78
|
+
Engine.define_singleton_method(:emit_stream) {|tag,es|
|
79
|
+
m.call(tag, es)
|
80
|
+
}
|
96
81
|
|
82
|
+
block.call if block
|
97
83
|
|
98
|
-
|
84
|
+
if @expects
|
85
|
+
i = 0
|
86
|
+
@emit_streams.each {|tag,events|
|
87
|
+
events.each {|time,record|
|
88
|
+
assert_equal(@expects[i], [tag, time, record])
|
89
|
+
i += 1
|
90
|
+
}
|
91
|
+
}
|
92
|
+
assert_equal @expects.length, i
|
93
|
+
end
|
94
|
+
}
|
95
|
+
self
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
def emit_stream(tag, es)
|
100
|
+
@emit_streams << [tag, es.to_a]
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
99
104
|
end
|
@@ -16,131 +16,128 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
-
module Test
|
19
|
+
module Test
|
20
|
+
class TestOutputChain
|
21
|
+
def initialize
|
22
|
+
@called = 0
|
23
|
+
end
|
20
24
|
|
25
|
+
def next
|
26
|
+
@called += 1
|
27
|
+
end
|
21
28
|
|
22
|
-
|
23
|
-
|
24
|
-
@called = 0
|
25
|
-
end
|
29
|
+
attr_reader :called
|
30
|
+
end
|
26
31
|
|
27
|
-
def next
|
28
|
-
@called += 1
|
29
|
-
end
|
30
|
-
|
31
|
-
attr_reader :called
|
32
|
-
end
|
33
32
|
|
33
|
+
class OutputTestDriver < InputTestDriver
|
34
|
+
def initialize(klass, tag='test', &block)
|
35
|
+
super(klass, &block)
|
36
|
+
@tag = tag
|
37
|
+
end
|
34
38
|
|
35
|
-
|
36
|
-
def initialize(klass, tag='test', &block)
|
37
|
-
super(klass, &block)
|
38
|
-
@tag = tag
|
39
|
-
end
|
39
|
+
attr_accessor :tag
|
40
40
|
|
41
|
-
|
41
|
+
def emit(record, time=Time.now)
|
42
|
+
es = OneEventStream.new(time.to_i, record)
|
43
|
+
chain = TestOutputChain.new
|
44
|
+
@instance.emit(@tag, es, chain)
|
45
|
+
assert_equal 1, chain.called
|
46
|
+
end
|
47
|
+
end
|
42
48
|
|
43
|
-
def emit(record, time=Time.now)
|
44
|
-
es = OneEventStream.new(time.to_i, record)
|
45
|
-
chain = TestOutputChain.new
|
46
|
-
@instance.emit(@tag, es, chain)
|
47
|
-
assert_equal 1, chain.called
|
48
|
-
end
|
49
|
-
end
|
50
49
|
|
50
|
+
class BufferedOutputTestDriver < InputTestDriver
|
51
|
+
def initialize(klass, tag='test', &block)
|
52
|
+
super(klass, &block)
|
53
|
+
@entries = []
|
54
|
+
@expected_buffer = nil
|
55
|
+
@tag = tag
|
56
|
+
end
|
51
57
|
|
52
|
-
|
53
|
-
def initialize(klass, tag='test', &block)
|
54
|
-
super(klass, &block)
|
55
|
-
@entries = []
|
56
|
-
@expected_buffer = nil
|
57
|
-
@tag = tag
|
58
|
-
end
|
58
|
+
attr_accessor :tag
|
59
59
|
|
60
|
-
|
60
|
+
def emit(record, time=Time.now)
|
61
|
+
@entries << [time.to_i, record]
|
62
|
+
self
|
63
|
+
end
|
61
64
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
end
|
65
|
+
def expect_format(str)
|
66
|
+
(@expected_buffer ||= '') << str
|
67
|
+
end
|
66
68
|
|
67
|
-
|
68
|
-
|
69
|
-
|
69
|
+
def run(&block)
|
70
|
+
result = nil
|
71
|
+
super {
|
72
|
+
es = ArrayEventStream.new(@entries)
|
73
|
+
buffer = @instance.format_stream(@tag, es)
|
70
74
|
|
71
|
-
|
72
|
-
result = nil
|
73
|
-
super {
|
74
|
-
es = ArrayEventStream.new(@entries)
|
75
|
-
buffer = @instance.format_stream(@tag, es)
|
75
|
+
block.call if block
|
76
76
|
|
77
|
-
|
77
|
+
if @expected_buffer
|
78
|
+
assert_equal(@expected_buffer, buffer)
|
79
|
+
end
|
78
80
|
|
79
|
-
|
80
|
-
|
81
|
+
chunk = MemoryBufferChunk.new('', buffer)
|
82
|
+
result = @instance.write(chunk)
|
83
|
+
}
|
84
|
+
result
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
class TimeSlicedOutputTestDriver < InputTestDriver
|
89
|
+
def initialize(klass, tag='test', &block)
|
90
|
+
super(klass, &block)
|
91
|
+
@entries = {}
|
92
|
+
@expected_buffer = nil
|
93
|
+
@tag = tag
|
81
94
|
end
|
82
95
|
|
83
|
-
|
84
|
-
result = @instance.write(chunk)
|
85
|
-
}
|
86
|
-
result
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
class TimeSlicedOutputTestDriver < InputTestDriver
|
91
|
-
def initialize(klass, tag='test', &block)
|
92
|
-
super(klass, &block)
|
93
|
-
@entries = {}
|
94
|
-
@expected_buffer = nil
|
95
|
-
@tag = tag
|
96
|
-
end
|
97
|
-
|
98
|
-
attr_accessor :tag
|
96
|
+
attr_accessor :tag
|
99
97
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
98
|
+
def emit(record, time=Time.now)
|
99
|
+
slicer = @instance.instance_eval{@time_slicer}
|
100
|
+
key = slicer.call(time.to_i)
|
101
|
+
@entries[key] = [] unless @entries.has_key?(key)
|
102
|
+
@entries[key] << [time.to_i, record]
|
103
|
+
self
|
104
|
+
end
|
107
105
|
|
108
|
-
|
109
|
-
|
110
|
-
|
106
|
+
def expect_format(str)
|
107
|
+
(@expected_buffer ||= '') << str
|
108
|
+
end
|
111
109
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
110
|
+
def run(&block)
|
111
|
+
result = []
|
112
|
+
super {
|
113
|
+
buffer = ''
|
114
|
+
@entries.keys.each {|key|
|
115
|
+
es = ArrayEventStream.new(@entries[key])
|
116
|
+
@instance.emit(@tag, es, NullOutputChain.instance)
|
117
|
+
buffer << @instance.format_stream(@tag, es)
|
118
|
+
}
|
121
119
|
|
122
|
-
|
120
|
+
block.call if block
|
123
121
|
|
124
|
-
|
125
|
-
|
126
|
-
|
122
|
+
if @expected_buffer
|
123
|
+
assert_equal(@expected_buffer, buffer)
|
124
|
+
end
|
127
125
|
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
126
|
+
chunks = @instance.instance_eval {
|
127
|
+
@buffer.instance_eval {
|
128
|
+
chunks = []
|
129
|
+
@map.keys.each {|key|
|
130
|
+
chunks.push(@map.delete(key))
|
131
|
+
}
|
132
|
+
chunks
|
133
|
+
}
|
134
|
+
}
|
135
|
+
chunks.each { |chunk|
|
136
|
+
result.push(@instance.write(chunk))
|
133
137
|
}
|
134
|
-
chunks
|
135
138
|
}
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
}
|
140
|
-
}
|
141
|
-
result
|
139
|
+
result
|
140
|
+
end
|
141
|
+
end
|
142
142
|
end
|
143
143
|
end
|
144
|
-
|
145
|
-
end
|
146
|
-
end
|