fluentd 0.12.0.pre.1 → 0.12.0.pre.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/.travis.yml +1 -0
- data/ChangeLog +21 -0
- data/README.md +10 -2
- data/Rakefile +4 -13
- data/example/v1_literal_example.conf +36 -0
- data/fluentd.gemspec +4 -1
- data/lib/fluent/buffer.rb +73 -46
- data/lib/fluent/command/fluentd.rb +7 -2
- data/lib/fluent/config/basic_parser.rb +5 -0
- data/lib/fluent/config/element.rb +2 -5
- data/lib/fluent/config/literal_parser.rb +26 -7
- data/lib/fluent/config/section.rb +2 -0
- data/lib/fluent/config/v1_parser.rb +9 -2
- data/lib/fluent/formatter.rb +2 -1
- data/lib/fluent/mixin.rb +22 -7
- data/lib/fluent/output.rb +17 -8
- data/lib/fluent/parser.rb +14 -3
- data/lib/fluent/plugin/buf_file.rb +30 -15
- data/lib/fluent/plugin/filter_grep.rb +69 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +183 -0
- data/lib/fluent/plugin/in_exec.rb +6 -0
- data/lib/fluent/plugin/in_forward.rb +34 -4
- data/lib/fluent/plugin/in_http.rb +1 -1
- data/lib/fluent/plugin/out_exec.rb +1 -1
- data/lib/fluent/plugin/out_exec_filter.rb +8 -1
- data/lib/fluent/plugin/out_forward.rb +82 -4
- data/lib/fluent/supervisor.rb +1 -1
- data/lib/fluent/timezone.rb +131 -0
- data/lib/fluent/version.rb +1 -1
- data/test/config/assertions.rb +42 -0
- data/test/config/test_config_parser.rb +385 -0
- data/test/config/test_configurable.rb +530 -0
- data/test/config/test_configure_proxy.rb +99 -0
- data/test/config/test_dsl.rb +237 -0
- data/test/config/test_literal_parser.rb +293 -0
- data/test/config/test_section.rb +112 -0
- data/test/config/test_system_config.rb +49 -0
- data/test/helper.rb +25 -0
- data/test/plugin/test_buf_file.rb +604 -0
- data/test/plugin/test_buf_memory.rb +204 -0
- data/test/plugin/test_filter_grep.rb +124 -0
- data/test/plugin/test_filter_record_transformer.rb +251 -0
- data/test/plugin/test_in_exec.rb +1 -0
- data/test/plugin/test_in_forward.rb +205 -2
- data/test/plugin/test_in_gc_stat.rb +1 -0
- data/test/plugin/test_in_http.rb +58 -2
- data/test/plugin/test_in_object_space.rb +1 -0
- data/test/plugin/test_in_status.rb +1 -0
- data/test/plugin/test_in_stream.rb +1 -1
- data/test/plugin/test_in_syslog.rb +1 -1
- data/test/plugin/test_in_tail.rb +1 -0
- data/test/plugin/test_in_tcp.rb +1 -1
- data/test/plugin/test_in_udp.rb +1 -1
- data/test/plugin/test_out_copy.rb +1 -0
- data/test/plugin/test_out_exec.rb +1 -0
- data/test/plugin/test_out_exec_filter.rb +1 -0
- data/test/plugin/test_out_file.rb +36 -0
- data/test/plugin/test_out_forward.rb +279 -8
- data/test/plugin/test_out_roundrobin.rb +1 -0
- data/test/plugin/test_out_stdout.rb +1 -0
- data/test/plugin/test_out_stream.rb +1 -1
- data/test/test_buffer.rb +530 -0
- data/test/test_config.rb +1 -1
- data/test/test_configdsl.rb +1 -1
- data/test/test_formatter.rb +223 -0
- data/test/test_match.rb +1 -2
- data/test/test_mixin.rb +74 -2
- data/test/test_parser.rb +7 -1
- metadata +88 -35
- data/lib/fluent/plugin/buf_zfile.rb +0 -75
- data/spec/config/config_parser_spec.rb +0 -314
- data/spec/config/configurable_spec.rb +0 -524
- data/spec/config/configure_proxy_spec.rb +0 -96
- data/spec/config/dsl_spec.rb +0 -239
- data/spec/config/helper.rb +0 -49
- data/spec/config/literal_parser_spec.rb +0 -222
- data/spec/config/section_spec.rb +0 -97
- data/spec/config/system_config_spec.rb +0 -49
- data/spec/spec_helper.rb +0 -60
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/config/section'
|
3
|
+
|
4
|
+
module Fluent::Config
|
5
|
+
class TestSection < ::Test::Unit::TestCase
|
6
|
+
sub_test_case Fluent::Config::Section do
|
7
|
+
sub_test_case 'class' do
|
8
|
+
sub_test_case '.name' do
|
9
|
+
test 'returns its full module name as String' do
|
10
|
+
assert_equal('Fluent::Config::Section', Fluent::Config::Section.name)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
sub_test_case 'instance object' do
|
16
|
+
sub_test_case '#initialize' do
|
17
|
+
test 'creates blank object without argument' do
|
18
|
+
s = Fluent::Config::Section.new
|
19
|
+
assert_equal({}, s.instance_eval{ @params })
|
20
|
+
end
|
21
|
+
|
22
|
+
test 'creates object which contains specified hash object itself' do
|
23
|
+
hash = {
|
24
|
+
name: 'tagomoris',
|
25
|
+
age: 34,
|
26
|
+
send: 'email',
|
27
|
+
class: 'normal',
|
28
|
+
keys: 5,
|
29
|
+
}
|
30
|
+
s1 = Fluent::Config::Section.new(hash)
|
31
|
+
assert_equal(hash, s1.instance_eval { @params })
|
32
|
+
assert_equal("tagomoris", s1[:name])
|
33
|
+
assert_equal(34, s1[:age])
|
34
|
+
assert_equal("email", s1[:send])
|
35
|
+
assert_equal("normal", s1[:class])
|
36
|
+
assert_equal(5, s1[:keys])
|
37
|
+
|
38
|
+
assert_equal("tagomoris", s1.name)
|
39
|
+
assert_equal(34, s1.age)
|
40
|
+
assert_equal("email", s1.send)
|
41
|
+
assert_equal("normal", s1.class)
|
42
|
+
assert_equal(5, s1.keys)
|
43
|
+
|
44
|
+
assert_raise(NoMethodError) { s1.dup }
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
sub_test_case '#object_id' do
|
49
|
+
test 'returns its object id' do
|
50
|
+
s1 = Fluent::Config::Section.new({})
|
51
|
+
assert s1.object_id
|
52
|
+
s2 = Fluent::Config::Section.new({})
|
53
|
+
assert s2.object_id
|
54
|
+
assert_not_equal s1.object_id, s2.object_id
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
sub_test_case '#to_h' do
|
59
|
+
test 'returns internal hash itself' do
|
60
|
+
hash = {
|
61
|
+
name: 'tagomoris',
|
62
|
+
age: 34,
|
63
|
+
send: 'email',
|
64
|
+
class: 'normal',
|
65
|
+
keys: 5,
|
66
|
+
}
|
67
|
+
s = Fluent::Config::Section.new(hash)
|
68
|
+
assert_equal(hash, s.to_h)
|
69
|
+
assert_instance_of(Hash, s.to_h)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
sub_test_case '#instance_of?' do
|
74
|
+
test 'can judge whether it is a Section object or not' do
|
75
|
+
s = Fluent::Config::Section.new
|
76
|
+
assert_true(s.instance_of?(Fluent::Config::Section))
|
77
|
+
assert_false(s.instance_of?(BasicObject))
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
sub_test_case '#is_a?' do
|
82
|
+
test 'can judge whether it belongs to or not' do
|
83
|
+
s = Fluent::Config::Section.new
|
84
|
+
assert_true(s.is_a?(Fluent::Config::Section))
|
85
|
+
assert_true(s.kind_of?(Fluent::Config::Section))
|
86
|
+
assert_true(s.is_a?(BasicObject))
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
sub_test_case '#+' do
|
91
|
+
test 'can merge 2 sections: argument side is primary, internal hash is newly created' do
|
92
|
+
h1 = {name: "s1", num: 10, class: "A"}
|
93
|
+
s1 = Fluent::Config::Section.new(h1)
|
94
|
+
|
95
|
+
h2 = {name: "s2", class: "A", num2: "5", num3: "8"}
|
96
|
+
s2 = Fluent::Config::Section.new(h2)
|
97
|
+
s = s1 + s2
|
98
|
+
|
99
|
+
assert_not_equal(h1.object_id, s.to_h.object_id)
|
100
|
+
assert_not_equal(h2.object_id, s.to_h.object_id)
|
101
|
+
|
102
|
+
assert_equal("s2", s.name)
|
103
|
+
assert_equal(10, s.num)
|
104
|
+
assert_equal("A", s.class)
|
105
|
+
assert_equal("5", s.num2)
|
106
|
+
assert_equal("8", s.num3)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/configurable'
|
3
|
+
require 'fluent/config/element'
|
4
|
+
require 'fluent/config/section'
|
5
|
+
require 'fluent/supervisor'
|
6
|
+
|
7
|
+
module Fluent::Config
|
8
|
+
class TestSystemConfig < ::Test::Unit::TestCase
|
9
|
+
def parse_text(text)
|
10
|
+
basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
|
11
|
+
Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
|
12
|
+
end
|
13
|
+
|
14
|
+
test 'should not override default configurations when no parameters' do
|
15
|
+
conf = parse_text(<<EOS)
|
16
|
+
<system>
|
17
|
+
</system>
|
18
|
+
EOS
|
19
|
+
sc = Fluent::Supervisor::SystemConfig.new(conf)
|
20
|
+
assert_nil(sc.log_level)
|
21
|
+
assert_nil(sc.suppress_repeated_stacktrace)
|
22
|
+
assert_nil(sc.emit_error_log_interval)
|
23
|
+
assert_nil(sc.suppress_config_dump)
|
24
|
+
assert_nil(sc.without_source)
|
25
|
+
assert_empty(sc.to_opt)
|
26
|
+
end
|
27
|
+
|
28
|
+
{'log_level' => 'error', 'suppress_repeated_stacktrace' => true, 'emit_error_log_interval' => 60, 'suppress_config_dump' => true, 'without_source' => true}.each { |k, v|
|
29
|
+
test "accepts #{k} parameter" do
|
30
|
+
conf = parse_text(<<EOS)
|
31
|
+
<system>
|
32
|
+
#{k} #{v}
|
33
|
+
</system>
|
34
|
+
EOS
|
35
|
+
sc = Fluent::Supervisor::SystemConfig.new(conf)
|
36
|
+
assert_not_nil(sc.instance_variable_get("@#{k}"))
|
37
|
+
key = (k == 'emit_error_log_interval' ? :suppress_interval : k.to_sym)
|
38
|
+
assert_include(sc.to_opt, key)
|
39
|
+
end
|
40
|
+
}
|
41
|
+
|
42
|
+
{'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
|
43
|
+
test "should not affect settable parameters with unknown #{k} parameter" do
|
44
|
+
sc = Fluent::Supervisor::SystemConfig.new({k => v})
|
45
|
+
assert_empty(sc.to_opt)
|
46
|
+
end
|
47
|
+
}
|
48
|
+
end
|
49
|
+
end
|
data/test/helper.rb
CHANGED
@@ -1,3 +1,28 @@
|
|
1
|
+
# simplecov must be loaded before any of target code
|
2
|
+
if ENV['SIMPLE_COV']
|
3
|
+
require 'simplecov'
|
4
|
+
if defined?(SimpleCov::SourceFile)
|
5
|
+
mod = SimpleCov::SourceFile
|
6
|
+
def mod.new(*args, &block)
|
7
|
+
m = allocate
|
8
|
+
m.instance_eval do
|
9
|
+
begin
|
10
|
+
initialize(*args, &block)
|
11
|
+
rescue Encoding::UndefinedConversionError
|
12
|
+
@src = "".force_encoding('UTF-8')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
m
|
16
|
+
end
|
17
|
+
end
|
18
|
+
unless SimpleCov.running
|
19
|
+
SimpleCov.start do
|
20
|
+
add_filter '/test/'
|
21
|
+
add_filter '/gems/'
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
1
26
|
require 'test/unit'
|
2
27
|
require 'fileutils'
|
3
28
|
require 'fluent/log'
|
@@ -0,0 +1,604 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'helper'
|
3
|
+
require 'fluent/test'
|
4
|
+
require 'fluent/plugin/buf_file'
|
5
|
+
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
require 'stringio'
|
9
|
+
require 'msgpack'
|
10
|
+
|
11
|
+
module FluentFileBufferTest
|
12
|
+
class FileBufferChunkTest < Test::Unit::TestCase
|
13
|
+
BUF_FILE_TMPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp', 'buf_file_chunk'))
|
14
|
+
|
15
|
+
def setup
|
16
|
+
if Dir.exists? BUF_FILE_TMPDIR
|
17
|
+
FileUtils.remove_entry_secure BUF_FILE_TMPDIR
|
18
|
+
end
|
19
|
+
FileUtils.mkdir_p BUF_FILE_TMPDIR
|
20
|
+
end
|
21
|
+
|
22
|
+
def bufpath(unique, link=false)
|
23
|
+
File.join(BUF_FILE_TMPDIR, unique + '.log' + (link ? '.link' : ''))
|
24
|
+
end
|
25
|
+
|
26
|
+
def filebufferchunk(key, unique, opts={})
|
27
|
+
Fluent::FileBufferChunk.new(key, bufpath(unique), unique, opts[:mode] || "a+", opts[:symlink])
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_init
|
31
|
+
chunk = filebufferchunk('key', 'init1')
|
32
|
+
assert_equal 'key', chunk.key
|
33
|
+
assert_equal 'init1', chunk.unique_id
|
34
|
+
assert_equal bufpath('init1'), chunk.path
|
35
|
+
|
36
|
+
chunk.close # size==0, then, unlinked
|
37
|
+
|
38
|
+
symlink_path = bufpath('init2', true)
|
39
|
+
|
40
|
+
chunk = filebufferchunk('key2', 'init2', symlink: symlink_path)
|
41
|
+
assert_equal 'key2', chunk.key
|
42
|
+
assert_equal 'init2', chunk.unique_id
|
43
|
+
assert File.exists?(symlink_path) && File.symlink?(symlink_path)
|
44
|
+
|
45
|
+
chunk.close # unlink
|
46
|
+
|
47
|
+
assert File.symlink?(symlink_path)
|
48
|
+
File.unlink(symlink_path)
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_buffer_chunk_interface
|
52
|
+
chunk = filebufferchunk('key', 'interface1')
|
53
|
+
|
54
|
+
assert chunk.respond_to?(:empty?)
|
55
|
+
assert chunk.respond_to?(:<<)
|
56
|
+
assert chunk.respond_to?(:size)
|
57
|
+
assert chunk.respond_to?(:close)
|
58
|
+
assert chunk.respond_to?(:purge)
|
59
|
+
assert chunk.respond_to?(:read)
|
60
|
+
assert chunk.respond_to?(:open)
|
61
|
+
assert chunk.respond_to?(:write_to)
|
62
|
+
assert chunk.respond_to?(:msgpack_each)
|
63
|
+
|
64
|
+
chunk.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_empty?
|
68
|
+
chunk = filebufferchunk('e1', 'empty1')
|
69
|
+
assert chunk.empty?
|
70
|
+
chunk.close
|
71
|
+
|
72
|
+
open(bufpath('empty2'), 'w') do |file|
|
73
|
+
file.write "data1\ndata2\n"
|
74
|
+
end
|
75
|
+
chunk = filebufferchunk('e2', 'empty2')
|
76
|
+
assert !(chunk.empty?)
|
77
|
+
chunk.close
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_append_close_purge
|
81
|
+
chunk = filebufferchunk('a1', 'append1')
|
82
|
+
assert chunk.empty?
|
83
|
+
|
84
|
+
test_data1 = ("1" * 9 + "\n" + "2" * 9 + "\n").force_encoding('ASCII-8BIT')
|
85
|
+
test_data2 = "日本語Japanese\n".force_encoding('UTF-8')
|
86
|
+
chunk << test_data1
|
87
|
+
chunk << test_data2
|
88
|
+
assert_equal 38, chunk.size
|
89
|
+
chunk.close
|
90
|
+
|
91
|
+
assert File.exists?(bufpath('append1'))
|
92
|
+
|
93
|
+
chunk = filebufferchunk('a1', 'append1', mode: 'r')
|
94
|
+
test_data = test_data1.force_encoding('ASCII-8BIT') + test_data2.force_encoding('ASCII-8BIT')
|
95
|
+
|
96
|
+
#### TODO: This assertion currently fails. Oops.
|
97
|
+
# FileBuffer#read does NOT do force_encoding('ASCII-8BIT'). So encoding of output string instance are 'UTF-8'.
|
98
|
+
# I think it is a kind of bug, but fixing it may break some behavior of buf_file. So I cannot be sure to fix it just now.
|
99
|
+
#
|
100
|
+
# assert_equal test_data, chunk.read
|
101
|
+
|
102
|
+
chunk.purge
|
103
|
+
|
104
|
+
assert !(File.exists?(bufpath('append1')))
|
105
|
+
end
|
106
|
+
|
107
|
+
def test_empty_chunk_key # for BufferedOutput#emit
|
108
|
+
chunk = filebufferchunk('', 'append1')
|
109
|
+
assert chunk.empty?
|
110
|
+
|
111
|
+
test_data1 = ("1" * 9 + "\n" + "2" * 9 + "\n").force_encoding('ASCII-8BIT')
|
112
|
+
test_data2 = "日本語Japanese\n".force_encoding('UTF-8')
|
113
|
+
chunk << test_data1
|
114
|
+
chunk << test_data2
|
115
|
+
assert_equal 38, chunk.size
|
116
|
+
chunk.close
|
117
|
+
end
|
118
|
+
|
119
|
+
def test_read
|
120
|
+
chunk = filebufferchunk('r1', 'read1')
|
121
|
+
assert chunk.empty?
|
122
|
+
|
123
|
+
d1 = "abcde" * 200 + "\n"
|
124
|
+
chunk << d1
|
125
|
+
d2 = "12345" * 200 + "\n"
|
126
|
+
chunk << d2
|
127
|
+
assert_equal (d1.size + d2.size), chunk.size
|
128
|
+
|
129
|
+
read_data = chunk.read
|
130
|
+
assert_equal (d1 + d2), read_data
|
131
|
+
|
132
|
+
chunk.purge
|
133
|
+
end
|
134
|
+
|
135
|
+
def test_open
|
136
|
+
chunk = filebufferchunk('o1', 'open1')
|
137
|
+
assert chunk.empty?
|
138
|
+
|
139
|
+
d1 = "abcde" * 200 + "\n"
|
140
|
+
chunk << d1
|
141
|
+
d2 = "12345" * 200 + "\n"
|
142
|
+
chunk << d2
|
143
|
+
assert_equal (d1.size + d2.size), chunk.size
|
144
|
+
|
145
|
+
read_data = chunk.open do |io|
|
146
|
+
io.read
|
147
|
+
end
|
148
|
+
assert_equal (d1 + d2), read_data
|
149
|
+
|
150
|
+
chunk.purge
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_write_to
|
154
|
+
chunk = filebufferchunk('w1', 'write1')
|
155
|
+
assert chunk.empty?
|
156
|
+
|
157
|
+
d1 = "abcde" * 200 + "\n"
|
158
|
+
chunk << d1
|
159
|
+
d2 = "12345" * 200 + "\n"
|
160
|
+
chunk << d2
|
161
|
+
assert_equal (d1.size + d2.size), chunk.size
|
162
|
+
|
163
|
+
dummy_dst = StringIO.new
|
164
|
+
|
165
|
+
chunk.write_to(dummy_dst)
|
166
|
+
assert_equal (d1 + d2), dummy_dst.string
|
167
|
+
|
168
|
+
chunk.purge
|
169
|
+
end
|
170
|
+
|
171
|
+
def test_msgpack_each
|
172
|
+
chunk = filebufferchunk('m1', 'msgpack1')
|
173
|
+
assert chunk.empty?
|
174
|
+
|
175
|
+
d0 = MessagePack.pack([[1, "foo"], [2, "bar"], [3, "baz"]])
|
176
|
+
d1 = MessagePack.pack({"key1" => "value1", "key2" => "value2"})
|
177
|
+
d2 = MessagePack.pack("string1")
|
178
|
+
d3 = MessagePack.pack(1)
|
179
|
+
d4 = MessagePack.pack(nil)
|
180
|
+
chunk << d0
|
181
|
+
chunk << d1
|
182
|
+
chunk << d2
|
183
|
+
chunk << d3
|
184
|
+
chunk << d4
|
185
|
+
|
186
|
+
store = []
|
187
|
+
chunk.msgpack_each do |data|
|
188
|
+
store << data
|
189
|
+
end
|
190
|
+
|
191
|
+
assert_equal 5, store.size
|
192
|
+
assert_equal [[1, "foo"], [2, "bar"], [3, "baz"]], store[0]
|
193
|
+
assert_equal({"key1" => "value1", "key2" => "value2"}, store[1])
|
194
|
+
assert_equal "string1", store[2]
|
195
|
+
assert_equal 1, store[3]
|
196
|
+
assert_equal nil, store[4]
|
197
|
+
|
198
|
+
chunk.purge
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_mv
|
202
|
+
chunk = filebufferchunk('m1', 'move1')
|
203
|
+
assert chunk.empty?
|
204
|
+
|
205
|
+
d1 = "abcde" * 200 + "\n"
|
206
|
+
chunk << d1
|
207
|
+
d2 = "12345" * 200 + "\n"
|
208
|
+
chunk << d2
|
209
|
+
assert_equal (d1.size + d2.size), chunk.size
|
210
|
+
|
211
|
+
assert_equal bufpath('move1'), chunk.path
|
212
|
+
|
213
|
+
assert File.exists?( bufpath( 'move1' ) )
|
214
|
+
assert !(File.exists?( bufpath( 'move2' ) ))
|
215
|
+
|
216
|
+
chunk.mv(bufpath('move2'))
|
217
|
+
|
218
|
+
assert !(File.exists?( bufpath( 'move1' ) ))
|
219
|
+
assert File.exists?( bufpath( 'move2' ) )
|
220
|
+
|
221
|
+
assert_equal bufpath('move2'), chunk.path
|
222
|
+
|
223
|
+
chunk.purge
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
class FileBufferTest < Test::Unit::TestCase
|
228
|
+
BUF_FILE_TMPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp', 'buf_file'))
|
229
|
+
|
230
|
+
def setup
|
231
|
+
if Dir.exists? BUF_FILE_TMPDIR
|
232
|
+
FileUtils.remove_entry_secure BUF_FILE_TMPDIR
|
233
|
+
end
|
234
|
+
FileUtils.mkdir_p BUF_FILE_TMPDIR
|
235
|
+
end
|
236
|
+
|
237
|
+
def bufpath(basename)
|
238
|
+
File.join(BUF_FILE_TMPDIR, basename)
|
239
|
+
end
|
240
|
+
|
241
|
+
def filebuffer(key, unique, opts={})
|
242
|
+
Fluent::FileBufferChunk.new(key, bufpath(unique), unique, opts[:mode] || "a+", opts[:symlink])
|
243
|
+
end
|
244
|
+
|
245
|
+
def test_init_configure
|
246
|
+
buf = Fluent::FileBuffer.new
|
247
|
+
|
248
|
+
assert_raise(Fluent::ConfigError){ buf.configure({}) }
|
249
|
+
|
250
|
+
buf.configure({'buffer_path' => bufpath('configure1.*.log')})
|
251
|
+
assert_equal bufpath('configure1.*.log'), buf.buffer_path
|
252
|
+
assert_equal nil, buf.symlink_path
|
253
|
+
assert_equal false, buf.instance_eval{ @flush_at_shutdown }
|
254
|
+
|
255
|
+
buf2 = Fluent::FileBuffer.new
|
256
|
+
|
257
|
+
# Same buffer_path value is rejected, not to overwrite exisitng buffer file.
|
258
|
+
assert_raise(Fluent::ConfigError){ buf2.configure({'buffer_path' => bufpath('configure1.*.log')}) }
|
259
|
+
|
260
|
+
buf2.configure({'buffer_path' => bufpath('configure2.*.log'), 'flush_at_shutdown' => ''})
|
261
|
+
assert_equal bufpath('configure2.*.log'), buf2.buffer_path
|
262
|
+
assert_equal true, buf2.instance_eval{ @flush_at_shutdown }
|
263
|
+
end
|
264
|
+
|
265
|
+
def test_configure_path_prefix_suffix
|
266
|
+
# With '*' in path, prefix is the part before '*', suffix is the part after '*'
|
267
|
+
buf = Fluent::FileBuffer.new
|
268
|
+
|
269
|
+
path1 = bufpath('suffpref1.*.log')
|
270
|
+
prefix1, suffix1 = path1.split('*', 2)
|
271
|
+
buf.configure({'buffer_path' => path1})
|
272
|
+
assert_equal prefix1, buf.instance_eval{ @buffer_path_prefix }
|
273
|
+
assert_equal suffix1, buf.instance_eval{ @buffer_path_suffix }
|
274
|
+
|
275
|
+
# Without '*', prefix is the string of whole path + '.', suffix is '.log'
|
276
|
+
path2 = bufpath('suffpref2')
|
277
|
+
buf.configure({'buffer_path' => path2})
|
278
|
+
assert_equal path2 + '.', buf.instance_eval{ @buffer_path_prefix }
|
279
|
+
assert_equal '.log', buf.instance_eval{ @buffer_path_suffix }
|
280
|
+
end
|
281
|
+
|
282
|
+
class DummyOutput
|
283
|
+
attr_accessor :written
|
284
|
+
|
285
|
+
def write(chunk)
|
286
|
+
@written ||= []
|
287
|
+
@written.push chunk
|
288
|
+
"return value"
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
def test_encode_key
|
293
|
+
buf = Fluent::FileBuffer.new
|
294
|
+
safe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789'
|
295
|
+
assert_equal safe_chars, buf.send(:encode_key, safe_chars)
|
296
|
+
unsafe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789 ~/*()'
|
297
|
+
assert_equal safe_chars + '%20%7E%2F%2A%28%29', buf.send(:encode_key, unsafe_chars)
|
298
|
+
end
|
299
|
+
|
300
|
+
def test_decode_key
|
301
|
+
buf = Fluent::FileBuffer.new
|
302
|
+
safe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789'
|
303
|
+
assert_equal safe_chars, buf.send(:decode_key, safe_chars)
|
304
|
+
unsafe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789 ~/*()'
|
305
|
+
assert_equal unsafe_chars, buf.send(:decode_key, safe_chars + '%20%7E%2F%2A%28%29')
|
306
|
+
|
307
|
+
assert_equal safe_chars, buf.send(:decode_key, buf.send(:encode_key, safe_chars))
|
308
|
+
assert_equal unsafe_chars, buf.send(:decode_key, buf.send(:encode_key, unsafe_chars))
|
309
|
+
end
|
310
|
+
|
311
|
+
def test_make_path
|
312
|
+
buf = Fluent::FileBuffer.new
|
313
|
+
buf.configure({'buffer_path' => bufpath('makepath.*.log')})
|
314
|
+
prefix = buf.instance_eval{ @buffer_path_prefix }
|
315
|
+
suffix = buf.instance_eval{ @buffer_path_suffix }
|
316
|
+
|
317
|
+
path,tsuffix = buf.send(:make_path, buf.send(:encode_key, 'foo bar'), 'b')
|
318
|
+
assert path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.[bq][0-9a-f]+#{suffix}\Z/, "invalid format:#{path}"
|
319
|
+
assert tsuffix =~ /\A[0-9a-f]+\Z/, "invalid hexadecimal:#{tsuffix}"
|
320
|
+
|
321
|
+
path,tsuffix = buf.send(:make_path, buf.send(:encode_key, 'baz 123'), 'q')
|
322
|
+
assert path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.[bq][0-9a-f]+#{suffix}\Z/, "invalid format:#{path}"
|
323
|
+
assert tsuffix =~ /\A[0-9a-f]+\Z/, "invalid hexadecimal:#{tsuffix}"
|
324
|
+
end
|
325
|
+
|
326
|
+
def test_tsuffix_to_unique_id
|
327
|
+
buf = Fluent::FileBuffer.new
|
328
|
+
# why *2 ? frsyuki said "I forgot why completely."
|
329
|
+
assert_equal "\xFF\xFF\xFF\xFF".force_encoding('ASCII-8BIT'), buf.send(:tsuffix_to_unique_id, 'ffff')
|
330
|
+
assert_equal "\x88\x00\xFF\x00\x11\xEE\x88\x00\xFF\x00\x11\xEE".force_encoding('ASCII-8BIT'), buf.send(:tsuffix_to_unique_id, '8800ff0011ee')
|
331
|
+
end
|
332
|
+
|
333
|
+
def test_start_makes_parent_directories
|
334
|
+
buf = Fluent::FileBuffer.new
|
335
|
+
buf.configure({'buffer_path' => bufpath('start/base.*.log')})
|
336
|
+
parent_dirname = File.dirname(buf.instance_eval{ @buffer_path_prefix })
|
337
|
+
assert !(Dir.exists?(parent_dirname))
|
338
|
+
buf.start
|
339
|
+
assert Dir.exists?(parent_dirname)
|
340
|
+
end
|
341
|
+
|
342
|
+
def test_new_chunk
|
343
|
+
buf = Fluent::FileBuffer.new
|
344
|
+
buf.configure({'buffer_path' => bufpath('new_chunk_1')})
|
345
|
+
prefix = buf.instance_eval{ @buffer_path_prefix }
|
346
|
+
suffix = buf.instance_eval{ @buffer_path_suffix }
|
347
|
+
|
348
|
+
chunk = buf.new_chunk('key1')
|
349
|
+
assert chunk
|
350
|
+
assert File.exists?(chunk.path)
|
351
|
+
assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
|
352
|
+
chunk.close
|
353
|
+
end
|
354
|
+
|
355
|
+
def test_chunk_identifier_in_path
|
356
|
+
buf1 = Fluent::FileBuffer.new
|
357
|
+
buf1.configure({'buffer_path' => bufpath('chunkid1')})
|
358
|
+
prefix1 = buf1.instance_eval{ @buffer_path_prefix }
|
359
|
+
suffix1 = buf1.instance_eval{ @buffer_path_suffix }
|
360
|
+
|
361
|
+
chunk1 = buf1.new_chunk('key1')
|
362
|
+
assert_equal chunk1.path, prefix1 + buf1.chunk_identifier_in_path(chunk1.path) + suffix1
|
363
|
+
|
364
|
+
buf2 = Fluent::FileBuffer.new
|
365
|
+
buf2.configure({'buffer_path' => bufpath('chunkid2')})
|
366
|
+
prefix2 = buf2.instance_eval{ @buffer_path_prefix }
|
367
|
+
suffix2 = buf2.instance_eval{ @buffer_path_suffix }
|
368
|
+
|
369
|
+
chunk2 = buf2.new_chunk('key2')
|
370
|
+
assert_equal chunk2.path, prefix2 + buf2.chunk_identifier_in_path(chunk2.path) + suffix2
|
371
|
+
end
|
372
|
+
|
373
|
+
def test_enqueue_moves_chunk_from_b_to_q
|
374
|
+
buf = Fluent::FileBuffer.new
|
375
|
+
buf.configure({'buffer_path' => bufpath('enqueue1')})
|
376
|
+
prefix = buf.instance_eval{ @buffer_path_prefix }
|
377
|
+
suffix = buf.instance_eval{ @buffer_path_suffix }
|
378
|
+
|
379
|
+
chunk = buf.new_chunk('key1')
|
380
|
+
chunk << "data1\ndata2\n"
|
381
|
+
|
382
|
+
assert chunk
|
383
|
+
old_path = chunk.path.dup
|
384
|
+
assert File.exists?(chunk.path)
|
385
|
+
assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
|
386
|
+
|
387
|
+
buf.enqueue(chunk)
|
388
|
+
|
389
|
+
assert chunk
|
390
|
+
assert File.exists?(chunk.path)
|
391
|
+
assert !(File.exists?(old_path))
|
392
|
+
assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "enqueued chunk's path must be a 'q' buffer chunk"
|
393
|
+
|
394
|
+
data = chunk.read
|
395
|
+
assert "data1\ndata2\n", data
|
396
|
+
end
|
397
|
+
|
398
|
+
# empty chunk keys are used w/ BufferedOutput
|
399
|
+
# * ObjectBufferedOutput's keys are tag
|
400
|
+
# * TimeSlicedOutput's keys are time_key
|
401
|
+
def test_enqueue_chunk_with_empty_key
|
402
|
+
buf = Fluent::FileBuffer.new
|
403
|
+
buf.configure({'buffer_path' => bufpath('enqueue2')})
|
404
|
+
prefix = buf.instance_eval{ @buffer_path_prefix }
|
405
|
+
suffix = buf.instance_eval{ @buffer_path_suffix }
|
406
|
+
|
407
|
+
chunk = buf.new_chunk('')
|
408
|
+
chunk << "data1\ndata2\n"
|
409
|
+
|
410
|
+
assert chunk
|
411
|
+
old_path = chunk.path.dup
|
412
|
+
assert File.exists?(chunk.path)
|
413
|
+
# chunk key is empty
|
414
|
+
assert chunk.path =~ /\A#{prefix}\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
|
415
|
+
|
416
|
+
buf.enqueue(chunk)
|
417
|
+
|
418
|
+
assert chunk
|
419
|
+
assert File.exists?(chunk.path)
|
420
|
+
assert !(File.exists?(old_path))
|
421
|
+
# chunk key is empty
|
422
|
+
assert chunk.path =~ /\A#{prefix}\.q[0-9a-f]+#{suffix}\Z/, "enqueued chunk's path must be a 'q' buffer chunk"
|
423
|
+
|
424
|
+
data = chunk.read
|
425
|
+
assert "data1\ndata2\n", data
|
426
|
+
end
|
427
|
+
|
428
|
+
def test_before_shutdown_without_flush_at_shutdown
|
429
|
+
buf = Fluent::FileBuffer.new
|
430
|
+
buf.configure({'buffer_path' => bufpath('before_shutdown1')})
|
431
|
+
buf.start
|
432
|
+
|
433
|
+
# before_shutdown does nothing
|
434
|
+
|
435
|
+
c1 = [ buf.new_chunk('k0'), buf.new_chunk('k1'), buf.new_chunk('k2'), buf.new_chunk('k3') ]
|
436
|
+
c2 = [ buf.new_chunk('q0'), buf.new_chunk('q1') ]
|
437
|
+
|
438
|
+
buf.instance_eval do
|
439
|
+
@map = {
|
440
|
+
'k0' => c1[0], 'k1' => c1[1], 'k2' => c1[2], 'k3' => c1[3],
|
441
|
+
'q0' => c2[0], 'q1' => c2[1]
|
442
|
+
}
|
443
|
+
end
|
444
|
+
c1[0] << "data1\ndata2\n"
|
445
|
+
c1[1] << "data1\ndata2\n"
|
446
|
+
c1[2] << "data1\ndata2\n"
|
447
|
+
# k3 chunk is empty!
|
448
|
+
|
449
|
+
c2[0] << "data1\ndata2\n"
|
450
|
+
c2[1] << "data1\ndata2\n"
|
451
|
+
buf.push('q0')
|
452
|
+
buf.push('q1')
|
453
|
+
|
454
|
+
buf.instance_eval do
|
455
|
+
@enqueue_hook_times = 0
|
456
|
+
def enqueue(chunk)
|
457
|
+
@enqueue_hook_times += 1
|
458
|
+
end
|
459
|
+
end
|
460
|
+
assert_equal 0, buf.instance_eval{ @enqueue_hook_times }
|
461
|
+
|
462
|
+
out = DummyOutput.new
|
463
|
+
assert_equal nil, out.written
|
464
|
+
|
465
|
+
buf.before_shutdown(out)
|
466
|
+
|
467
|
+
assert_equal 0, buf.instance_eval{ @enqueue_hook_times } # k0, k1, k2
|
468
|
+
assert_nil out.written
|
469
|
+
end
|
470
|
+
|
471
|
+
def test_before_shutdown_with_flush_at_shutdown
|
472
|
+
buf = Fluent::FileBuffer.new
|
473
|
+
buf.configure({'buffer_path' => bufpath('before_shutdown2'), 'flush_at_shutdown' => 'true'})
|
474
|
+
buf.start
|
475
|
+
|
476
|
+
# before_shutdown flushes all chunks in @map and @queue
|
477
|
+
|
478
|
+
c1 = [ buf.new_chunk('k0'), buf.new_chunk('k1'), buf.new_chunk('k2'), buf.new_chunk('k3') ]
|
479
|
+
c2 = [ buf.new_chunk('q0'), buf.new_chunk('q1') ]
|
480
|
+
|
481
|
+
buf.instance_eval do
|
482
|
+
@map = {
|
483
|
+
'k0' => c1[0], 'k1' => c1[1], 'k2' => c1[2], 'k3' => c1[3],
|
484
|
+
'q0' => c2[0], 'q1' => c2[1]
|
485
|
+
}
|
486
|
+
end
|
487
|
+
c1[0] << "data1\ndata2\n"
|
488
|
+
c1[1] << "data1\ndata2\n"
|
489
|
+
c1[2] << "data1\ndata2\n"
|
490
|
+
# k3 chunk is empty!
|
491
|
+
|
492
|
+
c2[0] << "data1\ndata2\n"
|
493
|
+
c2[1] << "data1\ndata2\n"
|
494
|
+
buf.push('q0')
|
495
|
+
buf.push('q1')
|
496
|
+
|
497
|
+
buf.instance_eval do
|
498
|
+
@enqueue_hook_times = 0
|
499
|
+
def enqueue(chunk)
|
500
|
+
@enqueue_hook_times += 1
|
501
|
+
end
|
502
|
+
end
|
503
|
+
assert_equal 0, buf.instance_eval{ @enqueue_hook_times }
|
504
|
+
|
505
|
+
out = DummyOutput.new
|
506
|
+
assert_equal nil, out.written
|
507
|
+
|
508
|
+
buf.before_shutdown(out)
|
509
|
+
|
510
|
+
assert_equal 3, buf.instance_eval{ @enqueue_hook_times } # k0, k1, k2
|
511
|
+
assert_equal 5, out.written.size
|
512
|
+
assert_equal [c2[0], c2[1], c1[0], c1[1], c1[2]], out.written
|
513
|
+
end
|
514
|
+
|
515
|
+
def test_resume
|
516
|
+
buffer_path_for_resume_test = bufpath('resume')
|
517
|
+
|
518
|
+
buf1 = Fluent::FileBuffer.new
|
519
|
+
buf1.configure({'buffer_path' => buffer_path_for_resume_test})
|
520
|
+
prefix = buf1.instance_eval{ @buffer_path_prefix }
|
521
|
+
suffix = buf1.instance_eval{ @buffer_path_suffix }
|
522
|
+
|
523
|
+
buf1.start
|
524
|
+
|
525
|
+
chunk1 = buf1.new_chunk('key1')
|
526
|
+
chunk1 << "data1\ndata2\n"
|
527
|
+
|
528
|
+
chunk2 = buf1.new_chunk('key2')
|
529
|
+
chunk2 << "data3\ndata4\n"
|
530
|
+
|
531
|
+
assert chunk1
|
532
|
+
assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
|
533
|
+
|
534
|
+
buf1.enqueue(chunk1)
|
535
|
+
|
536
|
+
assert chunk1
|
537
|
+
assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "chunk1 must be enqueued"
|
538
|
+
assert chunk2
|
539
|
+
assert chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "chunk2 is not enqueued yet"
|
540
|
+
|
541
|
+
buf1.shutdown
|
542
|
+
|
543
|
+
buf2 = Fluent::FileBuffer.new
|
544
|
+
Fluent::FileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
|
545
|
+
buf2.configure({'buffer_path' => buffer_path_for_resume_test})
|
546
|
+
prefix = buf2.instance_eval{ @buffer_path_prefix }
|
547
|
+
suffix = buf2.instance_eval{ @buffer_path_suffix }
|
548
|
+
|
549
|
+
# buf1.start -> resume is normal operation, but now, we cannot it.
|
550
|
+
queue, map = buf2.resume
|
551
|
+
|
552
|
+
assert_equal 1, queue.size
|
553
|
+
assert_equal 1, map.size
|
554
|
+
|
555
|
+
resumed_chunk1 = queue.first
|
556
|
+
assert_equal chunk1.path, resumed_chunk1.path
|
557
|
+
resumed_chunk2 = map['key2']
|
558
|
+
assert_equal chunk2.path, resumed_chunk2.path
|
559
|
+
|
560
|
+
assert_equal "data1\ndata2\n", resumed_chunk1.read
|
561
|
+
assert_equal "data3\ndata4\n", resumed_chunk2.read
|
562
|
+
end
|
563
|
+
|
564
|
+
class DummyChain
|
565
|
+
def next
|
566
|
+
true
|
567
|
+
end
|
568
|
+
end
|
569
|
+
|
570
|
+
def test_resume_only_for_my_buffer_path
|
571
|
+
chain = DummyChain.new
|
572
|
+
|
573
|
+
buffer_path_for_resume_test_1 = bufpath('resume_fix.1.*.log')
|
574
|
+
buffer_path_for_resume_test_2 = bufpath('resume_fix.*.log')
|
575
|
+
|
576
|
+
buf1 = Fluent::FileBuffer.new
|
577
|
+
buf1.configure({'buffer_path' => buffer_path_for_resume_test_1})
|
578
|
+
buf1.start
|
579
|
+
|
580
|
+
buf1.emit('key1', "x1\ty1\tz1\n", chain)
|
581
|
+
buf1.emit('key1', "x2\ty2\tz2\n", chain)
|
582
|
+
|
583
|
+
assert buf1.instance_eval{ @map['key1'] }
|
584
|
+
|
585
|
+
buf1.shutdown
|
586
|
+
|
587
|
+
buf2 = Fluent::FileBuffer.new
|
588
|
+
buf2.configure({'buffer_path' => buffer_path_for_resume_test_2}) # other buffer_path
|
589
|
+
|
590
|
+
queue, map = buf2.resume
|
591
|
+
|
592
|
+
assert_equal 0, queue.size
|
593
|
+
|
594
|
+
### TODO: This map size MUST be 0, but actually, 1
|
595
|
+
# This is because 1.XXXXX is misunderstood like chunk key of resume_fix.*.log.
|
596
|
+
# This may be a kind of bug, but we cannot decide whether 1. is a part of chunk key or not,
|
597
|
+
# because current version of buffer plugin uses '.'(dot) as a one of chars for chunk encoding.
|
598
|
+
# I think that this is a mistake of design, but we cannot fix it because updated plugin become
|
599
|
+
# not to be able to resume existing file buffer chunk.
|
600
|
+
# We will fix it in next API version of buffer plugin.
|
601
|
+
assert_equal 1, map.size
|
602
|
+
end
|
603
|
+
end
|
604
|
+
end
|