fluentd 0.12.0.pre.1 → 0.12.0.pre.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/.travis.yml +1 -0
  4. data/ChangeLog +21 -0
  5. data/README.md +10 -2
  6. data/Rakefile +4 -13
  7. data/example/v1_literal_example.conf +36 -0
  8. data/fluentd.gemspec +4 -1
  9. data/lib/fluent/buffer.rb +73 -46
  10. data/lib/fluent/command/fluentd.rb +7 -2
  11. data/lib/fluent/config/basic_parser.rb +5 -0
  12. data/lib/fluent/config/element.rb +2 -5
  13. data/lib/fluent/config/literal_parser.rb +26 -7
  14. data/lib/fluent/config/section.rb +2 -0
  15. data/lib/fluent/config/v1_parser.rb +9 -2
  16. data/lib/fluent/formatter.rb +2 -1
  17. data/lib/fluent/mixin.rb +22 -7
  18. data/lib/fluent/output.rb +17 -8
  19. data/lib/fluent/parser.rb +14 -3
  20. data/lib/fluent/plugin/buf_file.rb +30 -15
  21. data/lib/fluent/plugin/filter_grep.rb +69 -0
  22. data/lib/fluent/plugin/filter_record_transformer.rb +183 -0
  23. data/lib/fluent/plugin/in_exec.rb +6 -0
  24. data/lib/fluent/plugin/in_forward.rb +34 -4
  25. data/lib/fluent/plugin/in_http.rb +1 -1
  26. data/lib/fluent/plugin/out_exec.rb +1 -1
  27. data/lib/fluent/plugin/out_exec_filter.rb +8 -1
  28. data/lib/fluent/plugin/out_forward.rb +82 -4
  29. data/lib/fluent/supervisor.rb +1 -1
  30. data/lib/fluent/timezone.rb +131 -0
  31. data/lib/fluent/version.rb +1 -1
  32. data/test/config/assertions.rb +42 -0
  33. data/test/config/test_config_parser.rb +385 -0
  34. data/test/config/test_configurable.rb +530 -0
  35. data/test/config/test_configure_proxy.rb +99 -0
  36. data/test/config/test_dsl.rb +237 -0
  37. data/test/config/test_literal_parser.rb +293 -0
  38. data/test/config/test_section.rb +112 -0
  39. data/test/config/test_system_config.rb +49 -0
  40. data/test/helper.rb +25 -0
  41. data/test/plugin/test_buf_file.rb +604 -0
  42. data/test/plugin/test_buf_memory.rb +204 -0
  43. data/test/plugin/test_filter_grep.rb +124 -0
  44. data/test/plugin/test_filter_record_transformer.rb +251 -0
  45. data/test/plugin/test_in_exec.rb +1 -0
  46. data/test/plugin/test_in_forward.rb +205 -2
  47. data/test/plugin/test_in_gc_stat.rb +1 -0
  48. data/test/plugin/test_in_http.rb +58 -2
  49. data/test/plugin/test_in_object_space.rb +1 -0
  50. data/test/plugin/test_in_status.rb +1 -0
  51. data/test/plugin/test_in_stream.rb +1 -1
  52. data/test/plugin/test_in_syslog.rb +1 -1
  53. data/test/plugin/test_in_tail.rb +1 -0
  54. data/test/plugin/test_in_tcp.rb +1 -1
  55. data/test/plugin/test_in_udp.rb +1 -1
  56. data/test/plugin/test_out_copy.rb +1 -0
  57. data/test/plugin/test_out_exec.rb +1 -0
  58. data/test/plugin/test_out_exec_filter.rb +1 -0
  59. data/test/plugin/test_out_file.rb +36 -0
  60. data/test/plugin/test_out_forward.rb +279 -8
  61. data/test/plugin/test_out_roundrobin.rb +1 -0
  62. data/test/plugin/test_out_stdout.rb +1 -0
  63. data/test/plugin/test_out_stream.rb +1 -1
  64. data/test/test_buffer.rb +530 -0
  65. data/test/test_config.rb +1 -1
  66. data/test/test_configdsl.rb +1 -1
  67. data/test/test_formatter.rb +223 -0
  68. data/test/test_match.rb +1 -2
  69. data/test/test_mixin.rb +74 -2
  70. data/test/test_parser.rb +7 -1
  71. metadata +88 -35
  72. data/lib/fluent/plugin/buf_zfile.rb +0 -75
  73. data/spec/config/config_parser_spec.rb +0 -314
  74. data/spec/config/configurable_spec.rb +0 -524
  75. data/spec/config/configure_proxy_spec.rb +0 -96
  76. data/spec/config/dsl_spec.rb +0 -239
  77. data/spec/config/helper.rb +0 -49
  78. data/spec/config/literal_parser_spec.rb +0 -222
  79. data/spec/config/section_spec.rb +0 -97
  80. data/spec/config/system_config_spec.rb +0 -49
  81. data/spec/spec_helper.rb +0 -60
@@ -0,0 +1,112 @@
1
+ require 'helper'
2
+ require 'fluent/config/section'
3
+
4
+ module Fluent::Config
5
+ class TestSection < ::Test::Unit::TestCase
6
+ sub_test_case Fluent::Config::Section do
7
+ sub_test_case 'class' do
8
+ sub_test_case '.name' do
9
+ test 'returns its full module name as String' do
10
+ assert_equal('Fluent::Config::Section', Fluent::Config::Section.name)
11
+ end
12
+ end
13
+ end
14
+
15
+ sub_test_case 'instance object' do
16
+ sub_test_case '#initialize' do
17
+ test 'creates blank object without argument' do
18
+ s = Fluent::Config::Section.new
19
+ assert_equal({}, s.instance_eval{ @params })
20
+ end
21
+
22
+ test 'creates object which contains specified hash object itself' do
23
+ hash = {
24
+ name: 'tagomoris',
25
+ age: 34,
26
+ send: 'email',
27
+ class: 'normal',
28
+ keys: 5,
29
+ }
30
+ s1 = Fluent::Config::Section.new(hash)
31
+ assert_equal(hash, s1.instance_eval { @params })
32
+ assert_equal("tagomoris", s1[:name])
33
+ assert_equal(34, s1[:age])
34
+ assert_equal("email", s1[:send])
35
+ assert_equal("normal", s1[:class])
36
+ assert_equal(5, s1[:keys])
37
+
38
+ assert_equal("tagomoris", s1.name)
39
+ assert_equal(34, s1.age)
40
+ assert_equal("email", s1.send)
41
+ assert_equal("normal", s1.class)
42
+ assert_equal(5, s1.keys)
43
+
44
+ assert_raise(NoMethodError) { s1.dup }
45
+ end
46
+ end
47
+
48
+ sub_test_case '#object_id' do
49
+ test 'returns its object id' do
50
+ s1 = Fluent::Config::Section.new({})
51
+ assert s1.object_id
52
+ s2 = Fluent::Config::Section.new({})
53
+ assert s2.object_id
54
+ assert_not_equal s1.object_id, s2.object_id
55
+ end
56
+ end
57
+
58
+ sub_test_case '#to_h' do
59
+ test 'returns internal hash itself' do
60
+ hash = {
61
+ name: 'tagomoris',
62
+ age: 34,
63
+ send: 'email',
64
+ class: 'normal',
65
+ keys: 5,
66
+ }
67
+ s = Fluent::Config::Section.new(hash)
68
+ assert_equal(hash, s.to_h)
69
+ assert_instance_of(Hash, s.to_h)
70
+ end
71
+ end
72
+
73
+ sub_test_case '#instance_of?' do
74
+ test 'can judge whether it is a Section object or not' do
75
+ s = Fluent::Config::Section.new
76
+ assert_true(s.instance_of?(Fluent::Config::Section))
77
+ assert_false(s.instance_of?(BasicObject))
78
+ end
79
+ end
80
+
81
+ sub_test_case '#is_a?' do
82
+ test 'can judge whether it belongs to or not' do
83
+ s = Fluent::Config::Section.new
84
+ assert_true(s.is_a?(Fluent::Config::Section))
85
+ assert_true(s.kind_of?(Fluent::Config::Section))
86
+ assert_true(s.is_a?(BasicObject))
87
+ end
88
+ end
89
+
90
+ sub_test_case '#+' do
91
+ test 'can merge 2 sections: argument side is primary, internal hash is newly created' do
92
+ h1 = {name: "s1", num: 10, class: "A"}
93
+ s1 = Fluent::Config::Section.new(h1)
94
+
95
+ h2 = {name: "s2", class: "A", num2: "5", num3: "8"}
96
+ s2 = Fluent::Config::Section.new(h2)
97
+ s = s1 + s2
98
+
99
+ assert_not_equal(h1.object_id, s.to_h.object_id)
100
+ assert_not_equal(h2.object_id, s.to_h.object_id)
101
+
102
+ assert_equal("s2", s.name)
103
+ assert_equal(10, s.num)
104
+ assert_equal("A", s.class)
105
+ assert_equal("5", s.num2)
106
+ assert_equal("8", s.num3)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,49 @@
1
+ require 'helper'
2
+ require 'fluent/configurable'
3
+ require 'fluent/config/element'
4
+ require 'fluent/config/section'
5
+ require 'fluent/supervisor'
6
+
7
+ module Fluent::Config
8
+ class TestSystemConfig < ::Test::Unit::TestCase
9
+ def parse_text(text)
10
+ basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
11
+ Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
12
+ end
13
+
14
+ test 'should not override default configurations when no parameters' do
15
+ conf = parse_text(<<EOS)
16
+ <system>
17
+ </system>
18
+ EOS
19
+ sc = Fluent::Supervisor::SystemConfig.new(conf)
20
+ assert_nil(sc.log_level)
21
+ assert_nil(sc.suppress_repeated_stacktrace)
22
+ assert_nil(sc.emit_error_log_interval)
23
+ assert_nil(sc.suppress_config_dump)
24
+ assert_nil(sc.without_source)
25
+ assert_empty(sc.to_opt)
26
+ end
27
+
28
+ {'log_level' => 'error', 'suppress_repeated_stacktrace' => true, 'emit_error_log_interval' => 60, 'suppress_config_dump' => true, 'without_source' => true}.each { |k, v|
29
+ test "accepts #{k} parameter" do
30
+ conf = parse_text(<<EOS)
31
+ <system>
32
+ #{k} #{v}
33
+ </system>
34
+ EOS
35
+ sc = Fluent::Supervisor::SystemConfig.new(conf)
36
+ assert_not_nil(sc.instance_variable_get("@#{k}"))
37
+ key = (k == 'emit_error_log_interval' ? :suppress_interval : k.to_sym)
38
+ assert_include(sc.to_opt, key)
39
+ end
40
+ }
41
+
42
+ {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
43
+ test "should not affect settable parameters with unknown #{k} parameter" do
44
+ sc = Fluent::Supervisor::SystemConfig.new({k => v})
45
+ assert_empty(sc.to_opt)
46
+ end
47
+ }
48
+ end
49
+ end
data/test/helper.rb CHANGED
@@ -1,3 +1,28 @@
1
+ # simplecov must be loaded before any of target code
2
+ if ENV['SIMPLE_COV']
3
+ require 'simplecov'
4
+ if defined?(SimpleCov::SourceFile)
5
+ mod = SimpleCov::SourceFile
6
+ def mod.new(*args, &block)
7
+ m = allocate
8
+ m.instance_eval do
9
+ begin
10
+ initialize(*args, &block)
11
+ rescue Encoding::UndefinedConversionError
12
+ @src = "".force_encoding('UTF-8')
13
+ end
14
+ end
15
+ m
16
+ end
17
+ end
18
+ unless SimpleCov.running
19
+ SimpleCov.start do
20
+ add_filter '/test/'
21
+ add_filter '/gems/'
22
+ end
23
+ end
24
+ end
25
+
1
26
  require 'test/unit'
2
27
  require 'fileutils'
3
28
  require 'fluent/log'
@@ -0,0 +1,604 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'helper'
3
+ require 'fluent/test'
4
+ require 'fluent/plugin/buf_file'
5
+
6
+ require 'fileutils'
7
+
8
+ require 'stringio'
9
+ require 'msgpack'
10
+
11
+ module FluentFileBufferTest
12
+ class FileBufferChunkTest < Test::Unit::TestCase
13
+ BUF_FILE_TMPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp', 'buf_file_chunk'))
14
+
15
+ def setup
16
+ if Dir.exists? BUF_FILE_TMPDIR
17
+ FileUtils.remove_entry_secure BUF_FILE_TMPDIR
18
+ end
19
+ FileUtils.mkdir_p BUF_FILE_TMPDIR
20
+ end
21
+
22
+ def bufpath(unique, link=false)
23
+ File.join(BUF_FILE_TMPDIR, unique + '.log' + (link ? '.link' : ''))
24
+ end
25
+
26
+ def filebufferchunk(key, unique, opts={})
27
+ Fluent::FileBufferChunk.new(key, bufpath(unique), unique, opts[:mode] || "a+", opts[:symlink])
28
+ end
29
+
30
+ def test_init
31
+ chunk = filebufferchunk('key', 'init1')
32
+ assert_equal 'key', chunk.key
33
+ assert_equal 'init1', chunk.unique_id
34
+ assert_equal bufpath('init1'), chunk.path
35
+
36
+ chunk.close # size==0, then, unlinked
37
+
38
+ symlink_path = bufpath('init2', true)
39
+
40
+ chunk = filebufferchunk('key2', 'init2', symlink: symlink_path)
41
+ assert_equal 'key2', chunk.key
42
+ assert_equal 'init2', chunk.unique_id
43
+ assert File.exists?(symlink_path) && File.symlink?(symlink_path)
44
+
45
+ chunk.close # unlink
46
+
47
+ assert File.symlink?(symlink_path)
48
+ File.unlink(symlink_path)
49
+ end
50
+
51
+ def test_buffer_chunk_interface
52
+ chunk = filebufferchunk('key', 'interface1')
53
+
54
+ assert chunk.respond_to?(:empty?)
55
+ assert chunk.respond_to?(:<<)
56
+ assert chunk.respond_to?(:size)
57
+ assert chunk.respond_to?(:close)
58
+ assert chunk.respond_to?(:purge)
59
+ assert chunk.respond_to?(:read)
60
+ assert chunk.respond_to?(:open)
61
+ assert chunk.respond_to?(:write_to)
62
+ assert chunk.respond_to?(:msgpack_each)
63
+
64
+ chunk.close
65
+ end
66
+
67
+ def test_empty?
68
+ chunk = filebufferchunk('e1', 'empty1')
69
+ assert chunk.empty?
70
+ chunk.close
71
+
72
+ open(bufpath('empty2'), 'w') do |file|
73
+ file.write "data1\ndata2\n"
74
+ end
75
+ chunk = filebufferchunk('e2', 'empty2')
76
+ assert !(chunk.empty?)
77
+ chunk.close
78
+ end
79
+
80
+ def test_append_close_purge
81
+ chunk = filebufferchunk('a1', 'append1')
82
+ assert chunk.empty?
83
+
84
+ test_data1 = ("1" * 9 + "\n" + "2" * 9 + "\n").force_encoding('ASCII-8BIT')
85
+ test_data2 = "日本語Japanese\n".force_encoding('UTF-8')
86
+ chunk << test_data1
87
+ chunk << test_data2
88
+ assert_equal 38, chunk.size
89
+ chunk.close
90
+
91
+ assert File.exists?(bufpath('append1'))
92
+
93
+ chunk = filebufferchunk('a1', 'append1', mode: 'r')
94
+ test_data = test_data1.force_encoding('ASCII-8BIT') + test_data2.force_encoding('ASCII-8BIT')
95
+
96
+ #### TODO: This assertion currently fails. Oops.
97
+ # FileBuffer#read does NOT do force_encoding('ASCII-8BIT'). So encoding of output string instance are 'UTF-8'.
98
+ # I think it is a kind of bug, but fixing it may break some behavior of buf_file. So I cannot be sure to fix it just now.
99
+ #
100
+ # assert_equal test_data, chunk.read
101
+
102
+ chunk.purge
103
+
104
+ assert !(File.exists?(bufpath('append1')))
105
+ end
106
+
107
+ def test_empty_chunk_key # for BufferedOutput#emit
108
+ chunk = filebufferchunk('', 'append1')
109
+ assert chunk.empty?
110
+
111
+ test_data1 = ("1" * 9 + "\n" + "2" * 9 + "\n").force_encoding('ASCII-8BIT')
112
+ test_data2 = "日本語Japanese\n".force_encoding('UTF-8')
113
+ chunk << test_data1
114
+ chunk << test_data2
115
+ assert_equal 38, chunk.size
116
+ chunk.close
117
+ end
118
+
119
+ def test_read
120
+ chunk = filebufferchunk('r1', 'read1')
121
+ assert chunk.empty?
122
+
123
+ d1 = "abcde" * 200 + "\n"
124
+ chunk << d1
125
+ d2 = "12345" * 200 + "\n"
126
+ chunk << d2
127
+ assert_equal (d1.size + d2.size), chunk.size
128
+
129
+ read_data = chunk.read
130
+ assert_equal (d1 + d2), read_data
131
+
132
+ chunk.purge
133
+ end
134
+
135
+ def test_open
136
+ chunk = filebufferchunk('o1', 'open1')
137
+ assert chunk.empty?
138
+
139
+ d1 = "abcde" * 200 + "\n"
140
+ chunk << d1
141
+ d2 = "12345" * 200 + "\n"
142
+ chunk << d2
143
+ assert_equal (d1.size + d2.size), chunk.size
144
+
145
+ read_data = chunk.open do |io|
146
+ io.read
147
+ end
148
+ assert_equal (d1 + d2), read_data
149
+
150
+ chunk.purge
151
+ end
152
+
153
+ def test_write_to
154
+ chunk = filebufferchunk('w1', 'write1')
155
+ assert chunk.empty?
156
+
157
+ d1 = "abcde" * 200 + "\n"
158
+ chunk << d1
159
+ d2 = "12345" * 200 + "\n"
160
+ chunk << d2
161
+ assert_equal (d1.size + d2.size), chunk.size
162
+
163
+ dummy_dst = StringIO.new
164
+
165
+ chunk.write_to(dummy_dst)
166
+ assert_equal (d1 + d2), dummy_dst.string
167
+
168
+ chunk.purge
169
+ end
170
+
171
+ def test_msgpack_each
172
+ chunk = filebufferchunk('m1', 'msgpack1')
173
+ assert chunk.empty?
174
+
175
+ d0 = MessagePack.pack([[1, "foo"], [2, "bar"], [3, "baz"]])
176
+ d1 = MessagePack.pack({"key1" => "value1", "key2" => "value2"})
177
+ d2 = MessagePack.pack("string1")
178
+ d3 = MessagePack.pack(1)
179
+ d4 = MessagePack.pack(nil)
180
+ chunk << d0
181
+ chunk << d1
182
+ chunk << d2
183
+ chunk << d3
184
+ chunk << d4
185
+
186
+ store = []
187
+ chunk.msgpack_each do |data|
188
+ store << data
189
+ end
190
+
191
+ assert_equal 5, store.size
192
+ assert_equal [[1, "foo"], [2, "bar"], [3, "baz"]], store[0]
193
+ assert_equal({"key1" => "value1", "key2" => "value2"}, store[1])
194
+ assert_equal "string1", store[2]
195
+ assert_equal 1, store[3]
196
+ assert_equal nil, store[4]
197
+
198
+ chunk.purge
199
+ end
200
+
201
+ def test_mv
202
+ chunk = filebufferchunk('m1', 'move1')
203
+ assert chunk.empty?
204
+
205
+ d1 = "abcde" * 200 + "\n"
206
+ chunk << d1
207
+ d2 = "12345" * 200 + "\n"
208
+ chunk << d2
209
+ assert_equal (d1.size + d2.size), chunk.size
210
+
211
+ assert_equal bufpath('move1'), chunk.path
212
+
213
+ assert File.exists?( bufpath( 'move1' ) )
214
+ assert !(File.exists?( bufpath( 'move2' ) ))
215
+
216
+ chunk.mv(bufpath('move2'))
217
+
218
+ assert !(File.exists?( bufpath( 'move1' ) ))
219
+ assert File.exists?( bufpath( 'move2' ) )
220
+
221
+ assert_equal bufpath('move2'), chunk.path
222
+
223
+ chunk.purge
224
+ end
225
+ end
226
+
227
+ class FileBufferTest < Test::Unit::TestCase
228
+ BUF_FILE_TMPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp', 'buf_file'))
229
+
230
+ def setup
231
+ if Dir.exists? BUF_FILE_TMPDIR
232
+ FileUtils.remove_entry_secure BUF_FILE_TMPDIR
233
+ end
234
+ FileUtils.mkdir_p BUF_FILE_TMPDIR
235
+ end
236
+
237
+ def bufpath(basename)
238
+ File.join(BUF_FILE_TMPDIR, basename)
239
+ end
240
+
241
+ def filebuffer(key, unique, opts={})
242
+ Fluent::FileBufferChunk.new(key, bufpath(unique), unique, opts[:mode] || "a+", opts[:symlink])
243
+ end
244
+
245
+ def test_init_configure
246
+ buf = Fluent::FileBuffer.new
247
+
248
+ assert_raise(Fluent::ConfigError){ buf.configure({}) }
249
+
250
+ buf.configure({'buffer_path' => bufpath('configure1.*.log')})
251
+ assert_equal bufpath('configure1.*.log'), buf.buffer_path
252
+ assert_equal nil, buf.symlink_path
253
+ assert_equal false, buf.instance_eval{ @flush_at_shutdown }
254
+
255
+ buf2 = Fluent::FileBuffer.new
256
+
257
+ # Same buffer_path value is rejected, not to overwrite exisitng buffer file.
258
+ assert_raise(Fluent::ConfigError){ buf2.configure({'buffer_path' => bufpath('configure1.*.log')}) }
259
+
260
+ buf2.configure({'buffer_path' => bufpath('configure2.*.log'), 'flush_at_shutdown' => ''})
261
+ assert_equal bufpath('configure2.*.log'), buf2.buffer_path
262
+ assert_equal true, buf2.instance_eval{ @flush_at_shutdown }
263
+ end
264
+
265
+ def test_configure_path_prefix_suffix
266
+ # With '*' in path, prefix is the part before '*', suffix is the part after '*'
267
+ buf = Fluent::FileBuffer.new
268
+
269
+ path1 = bufpath('suffpref1.*.log')
270
+ prefix1, suffix1 = path1.split('*', 2)
271
+ buf.configure({'buffer_path' => path1})
272
+ assert_equal prefix1, buf.instance_eval{ @buffer_path_prefix }
273
+ assert_equal suffix1, buf.instance_eval{ @buffer_path_suffix }
274
+
275
+ # Without '*', prefix is the string of whole path + '.', suffix is '.log'
276
+ path2 = bufpath('suffpref2')
277
+ buf.configure({'buffer_path' => path2})
278
+ assert_equal path2 + '.', buf.instance_eval{ @buffer_path_prefix }
279
+ assert_equal '.log', buf.instance_eval{ @buffer_path_suffix }
280
+ end
281
+
282
+ class DummyOutput
283
+ attr_accessor :written
284
+
285
+ def write(chunk)
286
+ @written ||= []
287
+ @written.push chunk
288
+ "return value"
289
+ end
290
+ end
291
+
292
+ def test_encode_key
293
+ buf = Fluent::FileBuffer.new
294
+ safe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789'
295
+ assert_equal safe_chars, buf.send(:encode_key, safe_chars)
296
+ unsafe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789 ~/*()'
297
+ assert_equal safe_chars + '%20%7E%2F%2A%28%29', buf.send(:encode_key, unsafe_chars)
298
+ end
299
+
300
+ def test_decode_key
301
+ buf = Fluent::FileBuffer.new
302
+ safe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789'
303
+ assert_equal safe_chars, buf.send(:decode_key, safe_chars)
304
+ unsafe_chars = '-_.abcdefgxyzABCDEFGXYZ0123456789 ~/*()'
305
+ assert_equal unsafe_chars, buf.send(:decode_key, safe_chars + '%20%7E%2F%2A%28%29')
306
+
307
+ assert_equal safe_chars, buf.send(:decode_key, buf.send(:encode_key, safe_chars))
308
+ assert_equal unsafe_chars, buf.send(:decode_key, buf.send(:encode_key, unsafe_chars))
309
+ end
310
+
311
+ def test_make_path
312
+ buf = Fluent::FileBuffer.new
313
+ buf.configure({'buffer_path' => bufpath('makepath.*.log')})
314
+ prefix = buf.instance_eval{ @buffer_path_prefix }
315
+ suffix = buf.instance_eval{ @buffer_path_suffix }
316
+
317
+ path,tsuffix = buf.send(:make_path, buf.send(:encode_key, 'foo bar'), 'b')
318
+ assert path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.[bq][0-9a-f]+#{suffix}\Z/, "invalid format:#{path}"
319
+ assert tsuffix =~ /\A[0-9a-f]+\Z/, "invalid hexadecimal:#{tsuffix}"
320
+
321
+ path,tsuffix = buf.send(:make_path, buf.send(:encode_key, 'baz 123'), 'q')
322
+ assert path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.[bq][0-9a-f]+#{suffix}\Z/, "invalid format:#{path}"
323
+ assert tsuffix =~ /\A[0-9a-f]+\Z/, "invalid hexadecimal:#{tsuffix}"
324
+ end
325
+
326
+ def test_tsuffix_to_unique_id
327
+ buf = Fluent::FileBuffer.new
328
+ # why *2 ? frsyuki said "I forgot why completely."
329
+ assert_equal "\xFF\xFF\xFF\xFF".force_encoding('ASCII-8BIT'), buf.send(:tsuffix_to_unique_id, 'ffff')
330
+ assert_equal "\x88\x00\xFF\x00\x11\xEE\x88\x00\xFF\x00\x11\xEE".force_encoding('ASCII-8BIT'), buf.send(:tsuffix_to_unique_id, '8800ff0011ee')
331
+ end
332
+
333
+ def test_start_makes_parent_directories
334
+ buf = Fluent::FileBuffer.new
335
+ buf.configure({'buffer_path' => bufpath('start/base.*.log')})
336
+ parent_dirname = File.dirname(buf.instance_eval{ @buffer_path_prefix })
337
+ assert !(Dir.exists?(parent_dirname))
338
+ buf.start
339
+ assert Dir.exists?(parent_dirname)
340
+ end
341
+
342
+ def test_new_chunk
343
+ buf = Fluent::FileBuffer.new
344
+ buf.configure({'buffer_path' => bufpath('new_chunk_1')})
345
+ prefix = buf.instance_eval{ @buffer_path_prefix }
346
+ suffix = buf.instance_eval{ @buffer_path_suffix }
347
+
348
+ chunk = buf.new_chunk('key1')
349
+ assert chunk
350
+ assert File.exists?(chunk.path)
351
+ assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
352
+ chunk.close
353
+ end
354
+
355
+ def test_chunk_identifier_in_path
356
+ buf1 = Fluent::FileBuffer.new
357
+ buf1.configure({'buffer_path' => bufpath('chunkid1')})
358
+ prefix1 = buf1.instance_eval{ @buffer_path_prefix }
359
+ suffix1 = buf1.instance_eval{ @buffer_path_suffix }
360
+
361
+ chunk1 = buf1.new_chunk('key1')
362
+ assert_equal chunk1.path, prefix1 + buf1.chunk_identifier_in_path(chunk1.path) + suffix1
363
+
364
+ buf2 = Fluent::FileBuffer.new
365
+ buf2.configure({'buffer_path' => bufpath('chunkid2')})
366
+ prefix2 = buf2.instance_eval{ @buffer_path_prefix }
367
+ suffix2 = buf2.instance_eval{ @buffer_path_suffix }
368
+
369
+ chunk2 = buf2.new_chunk('key2')
370
+ assert_equal chunk2.path, prefix2 + buf2.chunk_identifier_in_path(chunk2.path) + suffix2
371
+ end
372
+
373
+ def test_enqueue_moves_chunk_from_b_to_q
374
+ buf = Fluent::FileBuffer.new
375
+ buf.configure({'buffer_path' => bufpath('enqueue1')})
376
+ prefix = buf.instance_eval{ @buffer_path_prefix }
377
+ suffix = buf.instance_eval{ @buffer_path_suffix }
378
+
379
+ chunk = buf.new_chunk('key1')
380
+ chunk << "data1\ndata2\n"
381
+
382
+ assert chunk
383
+ old_path = chunk.path.dup
384
+ assert File.exists?(chunk.path)
385
+ assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
386
+
387
+ buf.enqueue(chunk)
388
+
389
+ assert chunk
390
+ assert File.exists?(chunk.path)
391
+ assert !(File.exists?(old_path))
392
+ assert chunk.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "enqueued chunk's path must be a 'q' buffer chunk"
393
+
394
+ data = chunk.read
395
+ assert "data1\ndata2\n", data
396
+ end
397
+
398
+ # empty chunk keys are used w/ BufferedOutput
399
+ # * ObjectBufferedOutput's keys are tag
400
+ # * TimeSlicedOutput's keys are time_key
401
+ def test_enqueue_chunk_with_empty_key
402
+ buf = Fluent::FileBuffer.new
403
+ buf.configure({'buffer_path' => bufpath('enqueue2')})
404
+ prefix = buf.instance_eval{ @buffer_path_prefix }
405
+ suffix = buf.instance_eval{ @buffer_path_suffix }
406
+
407
+ chunk = buf.new_chunk('')
408
+ chunk << "data1\ndata2\n"
409
+
410
+ assert chunk
411
+ old_path = chunk.path.dup
412
+ assert File.exists?(chunk.path)
413
+ # chunk key is empty
414
+ assert chunk.path =~ /\A#{prefix}\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
415
+
416
+ buf.enqueue(chunk)
417
+
418
+ assert chunk
419
+ assert File.exists?(chunk.path)
420
+ assert !(File.exists?(old_path))
421
+ # chunk key is empty
422
+ assert chunk.path =~ /\A#{prefix}\.q[0-9a-f]+#{suffix}\Z/, "enqueued chunk's path must be a 'q' buffer chunk"
423
+
424
+ data = chunk.read
425
+ assert "data1\ndata2\n", data
426
+ end
427
+
428
+ def test_before_shutdown_without_flush_at_shutdown
429
+ buf = Fluent::FileBuffer.new
430
+ buf.configure({'buffer_path' => bufpath('before_shutdown1')})
431
+ buf.start
432
+
433
+ # before_shutdown does nothing
434
+
435
+ c1 = [ buf.new_chunk('k0'), buf.new_chunk('k1'), buf.new_chunk('k2'), buf.new_chunk('k3') ]
436
+ c2 = [ buf.new_chunk('q0'), buf.new_chunk('q1') ]
437
+
438
+ buf.instance_eval do
439
+ @map = {
440
+ 'k0' => c1[0], 'k1' => c1[1], 'k2' => c1[2], 'k3' => c1[3],
441
+ 'q0' => c2[0], 'q1' => c2[1]
442
+ }
443
+ end
444
+ c1[0] << "data1\ndata2\n"
445
+ c1[1] << "data1\ndata2\n"
446
+ c1[2] << "data1\ndata2\n"
447
+ # k3 chunk is empty!
448
+
449
+ c2[0] << "data1\ndata2\n"
450
+ c2[1] << "data1\ndata2\n"
451
+ buf.push('q0')
452
+ buf.push('q1')
453
+
454
+ buf.instance_eval do
455
+ @enqueue_hook_times = 0
456
+ def enqueue(chunk)
457
+ @enqueue_hook_times += 1
458
+ end
459
+ end
460
+ assert_equal 0, buf.instance_eval{ @enqueue_hook_times }
461
+
462
+ out = DummyOutput.new
463
+ assert_equal nil, out.written
464
+
465
+ buf.before_shutdown(out)
466
+
467
+ assert_equal 0, buf.instance_eval{ @enqueue_hook_times } # k0, k1, k2
468
+ assert_nil out.written
469
+ end
470
+
471
+ def test_before_shutdown_with_flush_at_shutdown
472
+ buf = Fluent::FileBuffer.new
473
+ buf.configure({'buffer_path' => bufpath('before_shutdown2'), 'flush_at_shutdown' => 'true'})
474
+ buf.start
475
+
476
+ # before_shutdown flushes all chunks in @map and @queue
477
+
478
+ c1 = [ buf.new_chunk('k0'), buf.new_chunk('k1'), buf.new_chunk('k2'), buf.new_chunk('k3') ]
479
+ c2 = [ buf.new_chunk('q0'), buf.new_chunk('q1') ]
480
+
481
+ buf.instance_eval do
482
+ @map = {
483
+ 'k0' => c1[0], 'k1' => c1[1], 'k2' => c1[2], 'k3' => c1[3],
484
+ 'q0' => c2[0], 'q1' => c2[1]
485
+ }
486
+ end
487
+ c1[0] << "data1\ndata2\n"
488
+ c1[1] << "data1\ndata2\n"
489
+ c1[2] << "data1\ndata2\n"
490
+ # k3 chunk is empty!
491
+
492
+ c2[0] << "data1\ndata2\n"
493
+ c2[1] << "data1\ndata2\n"
494
+ buf.push('q0')
495
+ buf.push('q1')
496
+
497
+ buf.instance_eval do
498
+ @enqueue_hook_times = 0
499
+ def enqueue(chunk)
500
+ @enqueue_hook_times += 1
501
+ end
502
+ end
503
+ assert_equal 0, buf.instance_eval{ @enqueue_hook_times }
504
+
505
+ out = DummyOutput.new
506
+ assert_equal nil, out.written
507
+
508
+ buf.before_shutdown(out)
509
+
510
+ assert_equal 3, buf.instance_eval{ @enqueue_hook_times } # k0, k1, k2
511
+ assert_equal 5, out.written.size
512
+ assert_equal [c2[0], c2[1], c1[0], c1[1], c1[2]], out.written
513
+ end
514
+
515
+ def test_resume
516
+ buffer_path_for_resume_test = bufpath('resume')
517
+
518
+ buf1 = Fluent::FileBuffer.new
519
+ buf1.configure({'buffer_path' => buffer_path_for_resume_test})
520
+ prefix = buf1.instance_eval{ @buffer_path_prefix }
521
+ suffix = buf1.instance_eval{ @buffer_path_suffix }
522
+
523
+ buf1.start
524
+
525
+ chunk1 = buf1.new_chunk('key1')
526
+ chunk1 << "data1\ndata2\n"
527
+
528
+ chunk2 = buf1.new_chunk('key2')
529
+ chunk2 << "data3\ndata4\n"
530
+
531
+ assert chunk1
532
+ assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
533
+
534
+ buf1.enqueue(chunk1)
535
+
536
+ assert chunk1
537
+ assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "chunk1 must be enqueued"
538
+ assert chunk2
539
+ assert chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "chunk2 is not enqueued yet"
540
+
541
+ buf1.shutdown
542
+
543
+ buf2 = Fluent::FileBuffer.new
544
+ Fluent::FileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
545
+ buf2.configure({'buffer_path' => buffer_path_for_resume_test})
546
+ prefix = buf2.instance_eval{ @buffer_path_prefix }
547
+ suffix = buf2.instance_eval{ @buffer_path_suffix }
548
+
549
+ # buf1.start -> resume is normal operation, but now, we cannot it.
550
+ queue, map = buf2.resume
551
+
552
+ assert_equal 1, queue.size
553
+ assert_equal 1, map.size
554
+
555
+ resumed_chunk1 = queue.first
556
+ assert_equal chunk1.path, resumed_chunk1.path
557
+ resumed_chunk2 = map['key2']
558
+ assert_equal chunk2.path, resumed_chunk2.path
559
+
560
+ assert_equal "data1\ndata2\n", resumed_chunk1.read
561
+ assert_equal "data3\ndata4\n", resumed_chunk2.read
562
+ end
563
+
564
+ class DummyChain
565
+ def next
566
+ true
567
+ end
568
+ end
569
+
570
+ def test_resume_only_for_my_buffer_path
571
+ chain = DummyChain.new
572
+
573
+ buffer_path_for_resume_test_1 = bufpath('resume_fix.1.*.log')
574
+ buffer_path_for_resume_test_2 = bufpath('resume_fix.*.log')
575
+
576
+ buf1 = Fluent::FileBuffer.new
577
+ buf1.configure({'buffer_path' => buffer_path_for_resume_test_1})
578
+ buf1.start
579
+
580
+ buf1.emit('key1', "x1\ty1\tz1\n", chain)
581
+ buf1.emit('key1', "x2\ty2\tz2\n", chain)
582
+
583
+ assert buf1.instance_eval{ @map['key1'] }
584
+
585
+ buf1.shutdown
586
+
587
+ buf2 = Fluent::FileBuffer.new
588
+ buf2.configure({'buffer_path' => buffer_path_for_resume_test_2}) # other buffer_path
589
+
590
+ queue, map = buf2.resume
591
+
592
+ assert_equal 0, queue.size
593
+
594
+ ### TODO: This map size MUST be 0, but actually, 1
595
+ # This is because 1.XXXXX is misunderstood like chunk key of resume_fix.*.log.
596
+ # This may be a kind of bug, but we cannot decide whether 1. is a part of chunk key or not,
597
+ # because current version of buffer plugin uses '.'(dot) as a one of chars for chunk encoding.
598
+ # I think that this is a mistake of design, but we cannot fix it because updated plugin become
599
+ # not to be able to resume existing file buffer chunk.
600
+ # We will fix it in next API version of buffer plugin.
601
+ assert_equal 1, map.size
602
+ end
603
+ end
604
+ end