fluentd 0.14.5-x64-mingw32 → 0.14.7-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +55 -0
  3. data/bin/fluent-binlog-reader +7 -0
  4. data/example/in_dummy_with_compression.conf +23 -0
  5. data/lib/fluent/agent.rb +8 -12
  6. data/lib/fluent/command/binlog_reader.rb +234 -0
  7. data/lib/fluent/command/fluentd.rb +17 -1
  8. data/lib/fluent/compat/file_util.rb +1 -1
  9. data/lib/fluent/compat/output.rb +5 -1
  10. data/lib/fluent/config/configure_proxy.rb +18 -3
  11. data/lib/fluent/config/element.rb +1 -1
  12. data/lib/fluent/config/section.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/env.rb +1 -0
  15. data/lib/fluent/event.rb +49 -2
  16. data/lib/fluent/event_router.rb +6 -2
  17. data/lib/fluent/label.rb +8 -0
  18. data/lib/fluent/log.rb +30 -1
  19. data/lib/fluent/plugin.rb +1 -1
  20. data/lib/fluent/plugin/base.rb +3 -0
  21. data/lib/fluent/plugin/buf_file.rb +2 -2
  22. data/lib/fluent/plugin/buf_memory.rb +1 -1
  23. data/lib/fluent/plugin/buffer.rb +12 -2
  24. data/lib/fluent/plugin/buffer/chunk.rb +68 -7
  25. data/lib/fluent/plugin/buffer/file_chunk.rb +4 -4
  26. data/lib/fluent/plugin/buffer/memory_chunk.rb +4 -4
  27. data/lib/fluent/plugin/compressable.rb +91 -0
  28. data/lib/fluent/plugin/filter_grep.rb +4 -4
  29. data/lib/fluent/plugin/formatter.rb +2 -2
  30. data/lib/fluent/plugin/formatter_json.rb +2 -1
  31. data/lib/fluent/plugin/formatter_out_file.rb +3 -30
  32. data/lib/fluent/plugin/in_forward.rb +6 -5
  33. data/lib/fluent/plugin/in_monitor_agent.rb +7 -21
  34. data/lib/fluent/plugin/in_syslog.rb +1 -1
  35. data/lib/fluent/plugin/in_tail.rb +11 -2
  36. data/lib/fluent/plugin/multi_output.rb +63 -3
  37. data/lib/fluent/plugin/out_exec.rb +1 -1
  38. data/lib/fluent/plugin/out_file.rb +5 -1
  39. data/lib/fluent/plugin/out_forward.rb +17 -5
  40. data/lib/fluent/plugin/out_stdout.rb +2 -1
  41. data/lib/fluent/plugin/output.rb +205 -19
  42. data/lib/fluent/plugin/parser.rb +5 -49
  43. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  44. data/lib/fluent/plugin/parser_json.rb +4 -4
  45. data/lib/fluent/plugin/parser_multiline.rb +5 -5
  46. data/lib/fluent/plugin/parser_regexp.rb +1 -2
  47. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  48. data/lib/fluent/plugin/storage_local.rb +2 -1
  49. data/lib/fluent/plugin_helper.rb +1 -0
  50. data/lib/fluent/plugin_helper/compat_parameters.rb +39 -21
  51. data/lib/fluent/plugin_helper/extract.rb +92 -0
  52. data/lib/fluent/plugin_helper/inject.rb +10 -12
  53. data/lib/fluent/plugin_helper/thread.rb +23 -3
  54. data/lib/fluent/registry.rb +1 -1
  55. data/lib/fluent/root_agent.rb +2 -1
  56. data/lib/fluent/supervisor.rb +28 -8
  57. data/lib/fluent/test/base.rb +0 -7
  58. data/lib/fluent/test/driver/base.rb +1 -0
  59. data/lib/fluent/test/driver/output.rb +3 -0
  60. data/lib/fluent/test/helpers.rb +18 -0
  61. data/lib/fluent/test/input_test.rb +4 -2
  62. data/lib/fluent/test/log.rb +3 -1
  63. data/lib/fluent/time.rb +232 -1
  64. data/lib/fluent/timezone.rb +1 -1
  65. data/lib/fluent/version.rb +1 -1
  66. data/test/command/test_binlog_reader.rb +351 -0
  67. data/test/config/test_config_parser.rb +6 -0
  68. data/test/config/test_configurable.rb +47 -1
  69. data/test/helper.rb +0 -1
  70. data/test/plugin/test_buffer.rb +22 -2
  71. data/test/plugin/test_buffer_chunk.rb +34 -4
  72. data/test/plugin/test_buffer_file_chunk.rb +73 -0
  73. data/test/plugin/test_buffer_memory_chunk.rb +73 -0
  74. data/test/plugin/test_compressable.rb +81 -0
  75. data/test/plugin/test_formatter_json.rb +14 -1
  76. data/test/plugin/test_in_forward.rb +67 -3
  77. data/test/plugin/test_in_monitor_agent.rb +17 -1
  78. data/test/plugin/test_in_tail.rb +8 -8
  79. data/test/plugin/test_out_file.rb +0 -8
  80. data/test/plugin/test_out_forward.rb +85 -0
  81. data/test/plugin/test_out_secondary_file.rb +20 -12
  82. data/test/plugin/test_out_stdout.rb +11 -10
  83. data/test/plugin/test_output.rb +234 -0
  84. data/test/plugin/test_output_as_buffered.rb +223 -0
  85. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  86. data/test/plugin/test_parser_json.rb +8 -0
  87. data/test/plugin/test_parser_regexp.rb +1 -1
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_extract.rb +195 -0
  90. data/test/plugin_helper/test_inject.rb +0 -7
  91. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  92. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  93. data/test/test_event.rb +186 -0
  94. data/test/test_event_router.rb +1 -1
  95. data/test/test_formatter.rb +0 -7
  96. data/test/test_log.rb +121 -0
  97. data/test/test_plugin_classes.rb +62 -0
  98. data/test/test_root_agent.rb +125 -0
  99. data/test/test_supervisor.rb +25 -2
  100. data/test/test_time_formatter.rb +103 -7
  101. data/test/test_time_parser.rb +211 -0
  102. metadata +22 -4
  103. data/test/plugin/test_parser_time.rb +0 -46
@@ -44,7 +44,6 @@ require 'fluent/log'
44
44
  require 'fluent/test'
45
45
  require 'fluent/test/helpers'
46
46
  require 'fluent/plugin/base'
47
- require 'fluent/log'
48
47
  require 'fluent/plugin_id'
49
48
  require 'fluent/plugin_helper'
50
49
  require 'fluent/msgpack_factory'
@@ -1,6 +1,8 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin/buffer'
3
3
  require 'fluent/plugin/buffer/memory_chunk'
4
+ require 'fluent/plugin/compressable'
5
+ require 'fluent/plugin/buffer/chunk'
4
6
  require 'fluent/event'
5
7
  require 'flexmock/test_unit'
6
8
 
@@ -18,7 +20,7 @@ module FluentPluginBufferTest
18
20
  class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk
19
21
  attr_reader :append_count, :rollbacked, :closed, :purged
20
22
  attr_accessor :failing
21
- def initialize(metadata)
23
+ def initialize(metadata, compress: :text)
22
24
  super
23
25
  @append_count = 0
24
26
  @rollbacked = false
@@ -77,7 +79,7 @@ module FluentPluginBufferTest
77
79
  return staged, queued
78
80
  end
79
81
  def generate_chunk(metadata)
80
- DummyMemoryChunk.new(metadata)
82
+ DummyMemoryChunk.new(metadata, compress: @compress)
81
83
  end
82
84
  end
83
85
  end
@@ -865,6 +867,10 @@ class BufferTest < Test::Unit::TestCase
865
867
  assert{ @p.stage[@dm1].size == 2 }
866
868
  assert @p.stage[@dm1].rollbacked
867
869
  end
870
+
871
+ test '#compress returns :text' do
872
+ assert_equal :text, @p.compress
873
+ end
868
874
  end
869
875
 
870
876
  sub_test_case 'standard format with configuration for test with lower chunk limit size' do
@@ -1197,4 +1203,18 @@ class BufferTest < Test::Unit::TestCase
1197
1203
  end
1198
1204
  end
1199
1205
 
1206
+ sub_test_case 'when compress is gzip' do
1207
+ setup do
1208
+ @p = create_buffer({'compress' => 'gzip'})
1209
+ end
1210
+
1211
+ test '#compress returns :gzip' do
1212
+ assert_equal :gzip, @p.compress
1213
+ end
1214
+
1215
+ test 'create decompressable chunk' do
1216
+ chunk = @p.generate_chunk(create_metadata)
1217
+ assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
1218
+ end
1219
+ end
1200
1220
  end
@@ -31,7 +31,6 @@ class BufferChunkTest < Test::Unit::TestCase
31
31
  assert chunk.respond_to?(:read)
32
32
  assert chunk.respond_to?(:open)
33
33
  assert chunk.respond_to?(:write_to)
34
- assert chunk.respond_to?(:msgpack_each)
35
34
  assert_raise(NotImplementedError){ chunk.append([]) }
36
35
  assert_raise(NotImplementedError){ chunk.concat(nil, 0) }
37
36
  assert_raise(NotImplementedError){ chunk.commit }
@@ -43,7 +42,29 @@ class BufferChunkTest < Test::Unit::TestCase
43
42
  assert_raise(NotImplementedError){ chunk.read }
44
43
  assert_raise(NotImplementedError){ chunk.open(){} }
45
44
  assert_raise(NotImplementedError){ chunk.write_to(nil) }
46
- assert_raise(NotImplementedError){ chunk.msgpack_each(){|v| v} }
45
+ assert !chunk.respond_to?(:msgpack_each)
46
+ end
47
+
48
+ test 'has method #each and #msgpack_each only when extended by ChunkMessagePackEventStreamer' do
49
+ meta = Object.new
50
+ chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
51
+
52
+ assert !chunk.respond_to?(:each)
53
+ assert !chunk.respond_to?(:msgpack_each)
54
+
55
+ chunk.extend Fluent::ChunkMessagePackEventStreamer
56
+ assert chunk.respond_to?(:each)
57
+ assert chunk.respond_to?(:msgpack_each)
58
+ end
59
+
60
+ test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do
61
+ meta = Object.new
62
+ chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
63
+
64
+ assert_raise(ArgumentError){ chunk.read(compressed: :gzip) }
65
+ assert_raise(ArgumentError){ chunk.open(compressed: :gzip){} }
66
+ assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :gzip) }
67
+ assert_raise(ArgumentError){ chunk.append(nil, compress: :gzip) }
47
68
  end
48
69
  end
49
70
 
@@ -56,7 +77,7 @@ class BufferChunkTest < Test::Unit::TestCase
56
77
  def size
57
78
  @data.size
58
79
  end
59
- def open
80
+ def open(**kwargs)
60
81
  require 'stringio'
61
82
  io = StringIO.new(@data)
62
83
  yield io
@@ -152,9 +173,10 @@ class BufferChunkTest < Test::Unit::TestCase
152
173
  assert "my data\nyour data\n", io.to_s
153
174
  end
154
175
 
155
- test 'can feed objects into blocks with unpacking msgpack' do
176
+ test 'can feed objects into blocks with unpacking msgpack if ChunkMessagePackEventStreamer is included' do
156
177
  require 'msgpack'
157
178
  c = TestChunk.new(Object.new)
179
+ c.extend Fluent::ChunkMessagePackEventStreamer
158
180
  c.data << MessagePack.pack(['my data', 1])
159
181
  c.data << MessagePack.pack(['your data', 2])
160
182
  ary = []
@@ -165,4 +187,12 @@ class BufferChunkTest < Test::Unit::TestCase
165
187
  assert_equal ['your data', 2], ary[1]
166
188
  end
167
189
  end
190
+
191
+ sub_test_case 'when compress is gzip' do
192
+ test 'create decompressable chunk' do
193
+ meta = Object.new
194
+ chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :gzip)
195
+ assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
196
+ end
197
+ end
168
198
  end
@@ -1,5 +1,6 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin/buffer/file_chunk'
3
+ require 'fluent/plugin/compressable'
3
4
  require 'fluent/unique_id'
4
5
 
5
6
  require 'fileutils'
@@ -8,6 +9,8 @@ require 'time'
8
9
  require 'timecop'
9
10
 
10
11
  class BufferFileChunkTest < Test::Unit::TestCase
12
+ include Fluent::Plugin::Compressable
13
+
11
14
  setup do
12
15
  @klass = Fluent::Plugin::Buffer::FileChunk
13
16
  @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__)
@@ -768,4 +771,74 @@ class BufferFileChunkTest < Test::Unit::TestCase
768
771
  end
769
772
  end
770
773
  end
774
+
775
+ sub_test_case 'compressed buffer' do
776
+ setup do
777
+ @src = 'text data for compressing' * 5
778
+ @gzipped_src = compress(@src)
779
+ end
780
+
781
+ test '#append with compress option writes compressed data to chunk when compress is gzip' do
782
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
783
+ c.append([@src, @src], compress: :gzip)
784
+ c.commit
785
+
786
+ # check chunk is compressed
787
+ assert c.read(compressed: :gzip).size < [@src, @src].join("").size
788
+
789
+ assert_equal @src + @src, c.read
790
+ end
791
+
792
+ test '#open passes io object having decompressed data to a block when compress is gzip' do
793
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
794
+ c.concat(@gzipped_src, @src.size)
795
+ c.commit
796
+
797
+ decomressed_data = c.open do |io|
798
+ v = io.read
799
+ assert_equal @src, v
800
+ v
801
+ end
802
+ assert_equal @src, decomressed_data
803
+ end
804
+
805
+ test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
806
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
807
+ c.concat(@gzipped_src, @src.size)
808
+ c.commit
809
+
810
+ comressed_data = c.open(compressed: :gzip) do |io|
811
+ v = io.read
812
+ assert_equal @gzipped_src, v
813
+ v
814
+ end
815
+ assert_equal @gzipped_src, comressed_data
816
+ end
817
+
818
+ test '#write_to writes decompressed data when compress is gzip' do
819
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
820
+ c.concat(@gzipped_src, @src.size)
821
+ c.commit
822
+
823
+ assert_equal @src, c.read
824
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
825
+
826
+ io = StringIO.new
827
+ c.write_to(io)
828
+ assert_equal @src, io.string
829
+ end
830
+
831
+ test '#write_to with compressed option writes compressed data when compress is gzip' do
832
+ c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
833
+ c.concat(@gzipped_src, @src.size)
834
+ c.commit
835
+
836
+ assert_equal @src, c.read
837
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
838
+
839
+ io = StringIO.new
840
+ c.write_to(io, compressed: :gzip)
841
+ assert_equal @gzipped_src, io.string
842
+ end
843
+ end
771
844
  end
@@ -1,9 +1,12 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin/buffer/memory_chunk'
3
+ require 'fluent/plugin/compressable'
3
4
 
4
5
  require 'json'
5
6
 
6
7
  class BufferMemoryChunkTest < Test::Unit::TestCase
8
+ include Fluent::Plugin::Compressable
9
+
7
10
  setup do
8
11
  @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new)
9
12
  end
@@ -262,4 +265,74 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
262
265
  assert_equal d3.to_json + "\n", lines[2]
263
266
  assert_equal d4.to_json + "\n", lines[3]
264
267
  end
268
+
269
+ sub_test_case 'compressed buffer' do
270
+ setup do
271
+ @src = 'text data for compressing' * 5
272
+ @gzipped_src = compress(@src)
273
+ end
274
+
275
+ test '#append with compress option writes compressed data to chunk when compress is gzip' do
276
+ c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
277
+ c.append([@src, @src], compress: :gzip)
278
+ c.commit
279
+
280
+ # check chunk is compressed
281
+ assert c.read(compressed: :gzip).size < [@src, @src].join("").size
282
+
283
+ assert_equal @src + @src, c.read
284
+ end
285
+
286
+ test '#open passes io object having decompressed data to a block when compress is gzip' do
287
+ c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
288
+ c.concat(@gzipped_src, @src.size)
289
+ c.commit
290
+
291
+ decomressed_data = c.open do |io|
292
+ v = io.read
293
+ assert_equal @src, v
294
+ v
295
+ end
296
+ assert_equal @src, decomressed_data
297
+ end
298
+
299
+ test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
300
+ c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
301
+ c.concat(@gzipped_src, @src.size)
302
+ c.commit
303
+
304
+ comressed_data = c.open(compressed: :gzip) do |io|
305
+ v = io.read
306
+ assert_equal @gzipped_src, v
307
+ v
308
+ end
309
+ assert_equal @gzipped_src, comressed_data
310
+ end
311
+
312
+ test '#write_to writes decompressed data when compress is gzip' do
313
+ c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
314
+ c.concat(@gzipped_src, @src.size)
315
+ c.commit
316
+
317
+ assert_equal @src, c.read
318
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
319
+
320
+ io = StringIO.new
321
+ c.write_to(io)
322
+ assert_equal @src, io.string
323
+ end
324
+
325
+ test '#write_to with compressed option writes compressed data when compress is gzip' do
326
+ c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
327
+ c.concat(@gzipped_src, @src.size)
328
+ c.commit
329
+
330
+ assert_equal @src, c.read
331
+ assert_equal @gzipped_src, c.read(compressed: :gzip)
332
+
333
+ io = StringIO.new
334
+ c.write_to(io, compressed: :gzip)
335
+ assert_equal @gzipped_src, io.string
336
+ end
337
+ end
265
338
  end
@@ -0,0 +1,81 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/compressable'
3
+
4
+ class CompressableTest < Test::Unit::TestCase
5
+ include Fluent::Plugin::Compressable
6
+
7
+ sub_test_case '#compress' do
8
+ setup do
9
+ @src = 'text data for compressing' * 5
10
+ @gzipped_src = compress(@src)
11
+ end
12
+
13
+ test 'compress data' do
14
+ assert compress(@src).size < @src.size
15
+ assert_not_equal @gzipped_src, @src
16
+ end
17
+
18
+ test 'write compressed data to IO with output_io option' do
19
+ io = StringIO.new
20
+ compress(@src, output_io: io)
21
+ assert_equal @gzipped_src, io.string
22
+ end
23
+ end
24
+
25
+ sub_test_case '#decompress' do
26
+ setup do
27
+ @src = 'text data for compressing' * 5
28
+ @gzipped_src = compress(@src)
29
+ end
30
+
31
+ test 'decompress compressed data' do
32
+ assert_equal @src, decompress(@gzipped_src)
33
+ end
34
+
35
+ test 'write decompressed data to IO with output_io option' do
36
+ io = StringIO.new
37
+ decompress(@gzipped_src, output_io: io)
38
+ assert_equal @src, io.string
39
+ end
40
+
41
+ test 'return decompressed string with output_io option' do
42
+ io = StringIO.new(@gzipped_src)
43
+ assert_equal @src, decompress(input_io: io)
44
+ end
45
+
46
+ test 'decompress multiple compressed data' do
47
+ src1 = 'text data'
48
+ src2 = 'text data2'
49
+ gzipped_src = compress(src1) + compress(src2)
50
+
51
+ assert_equal src1 + src2, decompress(gzipped_src)
52
+ end
53
+
54
+ test 'decompress with input_io and output_io' do
55
+ input_io = StringIO.new(@gzipped_src)
56
+ output_io = StringIO.new
57
+
58
+ decompress(input_io: input_io, output_io: output_io)
59
+ assert_equal @src, output_io.string
60
+ end
61
+
62
+ test 'decompress multiple compressed data with input_io and output_io' do
63
+ src1 = 'text data'
64
+ src2 = 'text data2'
65
+ gzipped_src = compress(src1) + compress(src2)
66
+
67
+ input_io = StringIO.new(gzipped_src)
68
+ output_io = StringIO.new
69
+
70
+ decompress(input_io: input_io, output_io: output_io)
71
+ assert_equal src1 + src2, output_io.string
72
+ end
73
+
74
+ test 'return the received value as it is with empty string or nil' do
75
+ assert_equal nil, decompress
76
+ assert_equal nil, decompress(nil)
77
+ assert_equal '', decompress('')
78
+ assert_equal '', decompress('', output_io: StringIO.new)
79
+ end
80
+ end
81
+ end
@@ -1,4 +1,5 @@
1
1
  require_relative '../helper'
2
+ require 'json'
2
3
  require 'fluent/test/driver/formatter'
3
4
  require 'fluent/plugin/formatter_json'
4
5
 
@@ -20,11 +21,23 @@ class JsonFormatterTest < ::Test::Unit::TestCase
20
21
  {'message' => 'awesome'}
21
22
  end
22
23
 
24
+ def symbolic_record
25
+ {:message => :awesome}
26
+ end
27
+
23
28
  data('oj' => 'oj', 'yajl' => 'yajl')
24
29
  def test_format(data)
25
30
  d = create_driver('json_parser' => data)
26
31
  formatted = d.instance.format(tag, @time, record)
27
32
 
28
- assert_equal("#{Yajl.dump(record)}\n", formatted)
33
+ assert_equal("#{JSON.generate(record)}\n", formatted)
34
+ end
35
+
36
+ data('oj' => 'oj', 'yajl' => 'yajl')
37
+ def test_format_with_symbolic_record(data)
38
+ d = create_driver('json_parser' => data)
39
+ formatted = d.instance.format(tag, @time, symbolic_record)
40
+
41
+ assert_equal("#{JSON.generate(record)}\n", formatted)
29
42
  end
30
43
  end
@@ -5,9 +5,13 @@ require 'fluent/test/startup_shutdown'
5
5
  require 'base64'
6
6
 
7
7
  require 'fluent/env'
8
+ require 'fluent/event'
8
9
  require 'fluent/plugin/in_forward'
10
+ require 'fluent/plugin/compressable'
9
11
 
10
12
  class ForwardInputTest < Test::Unit::TestCase
13
+ include Fluent::Plugin::Compressable
14
+
11
15
  def setup
12
16
  Fluent::Test.setup
13
17
  @responses = [] # for testing responses after sending data
@@ -465,6 +469,66 @@ class ForwardInputTest < Test::Unit::TestCase
465
469
  end
466
470
  end
467
471
 
472
+ class CompressedPackedForward < self
473
+ extend Fluent::Test::StartupShutdown
474
+
475
+ def test_set_compress_to_option
476
+ d = create_driver
477
+
478
+ time = event_time("2011-01-02 13:14:15 UTC").to_i
479
+ events = [
480
+ ["tag1", time, {"a"=>1}],
481
+ ["tag1", time, {"a"=>2}]
482
+ ]
483
+
484
+ # create compressed entries
485
+ entries = ''
486
+ events.each do |_tag, _time, record|
487
+ v = [_time, record].to_msgpack
488
+ entries << compress(v)
489
+ end
490
+ chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
491
+
492
+ d.run do
493
+ Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
494
+ option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
495
+ assert_equal 'gzip', option['compressed']
496
+ end
497
+ end
498
+
499
+ assert_equal events, d.emits
500
+ end
501
+
502
+ def test_create_CompressedMessagePackEventStream_with_gzip_compress_option
503
+ d = create_driver
504
+
505
+ time = event_time("2011-01-02 13:14:15 UTC").to_i
506
+ events = [
507
+ ["tag1", time, {"a"=>1}],
508
+ ["tag1", time, {"a"=>2}]
509
+ ]
510
+
511
+ # create compressed entries
512
+ entries = ''
513
+ events.each do |_tag, _time, record|
514
+ v = [_time, record].to_msgpack
515
+ entries << compress(v)
516
+ end
517
+ chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
518
+
519
+ # check CompressedMessagePackEventStream is created
520
+ mock(Fluent::CompressedMessagePackEventStream).new(entries, nil, 0)
521
+
522
+ d.run do
523
+ Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
524
+ option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
525
+ assert_equal 'gzip', option['compressed']
526
+ end
527
+ end
528
+ d.emits
529
+ end
530
+ end
531
+
468
532
  class Warning < self
469
533
  extend Fluent::Test::StartupShutdown
470
534
 
@@ -688,7 +752,7 @@ class ForwardInputTest < Test::Unit::TestCase
688
752
  auth: true
689
753
  }
690
754
  })
691
- def test_packed_forward
755
+ def test_packed_forward(data)
692
756
  config = data[:config]
693
757
  options = data[:options]
694
758
  d = create_driver(config)
@@ -783,7 +847,7 @@ class ForwardInputTest < Test::Unit::TestCase
783
847
  auth: true
784
848
  }
785
849
  })
786
- def test_message
850
+ def test_message(data)
787
851
  config = data[:config]
788
852
  options = data[:options]
789
853
  d = create_driver(config)
@@ -805,7 +869,7 @@ class ForwardInputTest < Test::Unit::TestCase
805
869
  end
806
870
 
807
871
  assert_equal events, d.emits
808
- assert_equal ["", ""], @responses
872
+ assert_equal [nil, nil], @responses
809
873
 
810
874
  sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
811
875
  end