fluentd 0.14.4 → 0.14.5

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +18 -0
  3. data/example/in_forward.conf +3 -0
  4. data/example/in_forward_client.conf +37 -0
  5. data/example/in_forward_shared_key.conf +15 -0
  6. data/example/in_forward_users.conf +24 -0
  7. data/example/out_forward.conf +13 -13
  8. data/example/out_forward_client.conf +109 -0
  9. data/example/out_forward_shared_key.conf +36 -0
  10. data/example/out_forward_users.conf +65 -0
  11. data/example/{out_buffered_null.conf → out_null.conf} +10 -6
  12. data/example/secondary_file.conf +41 -0
  13. data/lib/fluent/agent.rb +3 -1
  14. data/lib/fluent/plugin/buffer.rb +5 -1
  15. data/lib/fluent/plugin/in_forward.rb +300 -50
  16. data/lib/fluent/plugin/in_tail.rb +41 -85
  17. data/lib/fluent/plugin/multi_output.rb +4 -0
  18. data/lib/fluent/plugin/out_forward.rb +326 -209
  19. data/lib/fluent/plugin/out_null.rb +37 -0
  20. data/lib/fluent/plugin/out_secondary_file.rb +128 -0
  21. data/lib/fluent/plugin/out_stdout.rb +38 -2
  22. data/lib/fluent/plugin/output.rb +13 -5
  23. data/lib/fluent/root_agent.rb +1 -1
  24. data/lib/fluent/test/startup_shutdown.rb +33 -0
  25. data/lib/fluent/version.rb +1 -1
  26. data/test/plugin/test_in_forward.rb +906 -441
  27. data/test/plugin/test_in_monitor_agent.rb +4 -0
  28. data/test/plugin/test_in_tail.rb +681 -663
  29. data/test/plugin/test_out_forward.rb +150 -208
  30. data/test/plugin/test_out_null.rb +85 -9
  31. data/test/plugin/test_out_secondary_file.rb +432 -0
  32. data/test/plugin/test_out_stdout.rb +143 -45
  33. data/test/test_root_agent.rb +42 -0
  34. metadata +14 -9
  35. data/lib/fluent/plugin/out_buffered_null.rb +0 -59
  36. data/lib/fluent/plugin/out_buffered_stdout.rb +0 -70
  37. data/test/plugin/test_out_buffered_null.rb +0 -79
  38. data/test/plugin/test_out_buffered_stdout.rb +0 -122
@@ -18,10 +18,47 @@ require 'fluent/plugin/output'
18
18
 
19
19
  module Fluent::Plugin
20
20
  class NullOutput < Output
21
+ # This plugin is for tests of non-buffered/buffered plugins
21
22
  Fluent::Plugin.register_output('null', self)
22
23
 
24
+ config_section :buffer do
25
+ config_set_default :chunk_keys, ['tag']
26
+ config_set_default :flush_at_shutdown, true
27
+ config_set_default :chunk_limit_size, 10 * 1024
28
+ end
29
+
30
+ def prefer_buffered_processing
31
+ false
32
+ end
33
+
34
+ def prefer_delayed_commit
35
+ @delayed
36
+ end
37
+
38
+ attr_accessor :feed_proc, :delayed
39
+
40
+ def initialize
41
+ super
42
+ @delayed = false
43
+ @feed_proc = nil
44
+ end
45
+
23
46
  def process(tag, es)
24
47
  # Do nothing
25
48
  end
49
+
50
+ def write(chunk)
51
+ if @feed_proc
52
+ @feed_proc.call(chunk)
53
+ end
54
+ end
55
+
56
+ def try_write(chunk)
57
+ if @feed_proc
58
+ @feed_proc.call(chunk)
59
+ end
60
+ # not to commit chunks for testing
61
+ # commit_write(chunk.unique_id)
62
+ end
26
63
  end
27
64
  end
@@ -0,0 +1,128 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "fileutils"
18
+ require "fluent/plugin/file_util"
19
+ require "fluent/plugin/output"
20
+ require "fluent/config/error"
21
+
22
+ module Fluent::Plugin
23
+ class SecondaryFileOutput < Output
24
+ Fluent::Plugin.register_output("secondary_file", self)
25
+
26
+ FILE_PERMISSION = 0644
27
+ DIR_PERMISSION = 0755
28
+ PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
29
+
30
+ desc "The directory path of the output file."
31
+ config_param :directory, :string
32
+ desc "The basename of the output file."
33
+ config_param :basename, :string, default: "dump.bin"
34
+ desc "The flushed chunk is appended to existence file or not."
35
+ config_param :append, :bool, default: false
36
+ config_param :compress, :enum, list: [:text, :gzip], default: :text
37
+
38
+ def configure(conf)
39
+ super
40
+
41
+ unless @as_secondary
42
+ raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
43
+ end
44
+
45
+ if @basename.include?("/")
46
+ raise Fluent::ConfigError, "basename should not include `/`"
47
+ end
48
+
49
+ @path_without_suffix = File.join(@directory, @basename)
50
+ validate_compatible_with_primary_buffer!(@path_without_suffix)
51
+
52
+ @suffix = case @compress
53
+ when :text
54
+ ""
55
+ when :gzip
56
+ ".gz"
57
+ end
58
+
59
+ test_path = @path_without_suffix
60
+ unless Fluent::FileUtil.writable_p?(test_path)
61
+ raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
62
+ end
63
+
64
+ @dir_perm = system_config.dir_permission || DIR_PERMISSION
65
+ @file_perm = system_config.file_permission || FILE_PERMISSION
66
+ end
67
+
68
+ def write(chunk)
69
+ path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
70
+ path = generate_path(path_without_suffix)
71
+ FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
72
+
73
+ case @compress
74
+ when :text
75
+ File.open(path, "ab", @file_perm) {|f|
76
+ f.flock(File::LOCK_EX)
77
+ chunk.write_to(f)
78
+ }
79
+ when :gzip
80
+ File.open(path, "ab", @file_perm) {|f|
81
+ f.flock(File::LOCK_EX)
82
+ gz = Zlib::GzipWriter.new(f)
83
+ chunk.write_to(gz)
84
+ gz.close
85
+ }
86
+ end
87
+
88
+ path
89
+ end
90
+
91
+ private
92
+
93
+ def validate_compatible_with_primary_buffer!(path_without_suffix)
94
+ placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\d+]
95
+
96
+ if !@chunk_key_time && has_time_format?(path_without_suffix)
97
+ raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
98
+ end
99
+
100
+ if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
101
+ raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
102
+ end
103
+
104
+ vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) }
105
+
106
+ if ph = vars.find { |v| !@chunk_keys.include?(v) }
107
+ raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
108
+ end
109
+ end
110
+
111
+ def has_time_format?(str)
112
+ str != Time.now.strftime(str)
113
+ end
114
+
115
+ def generate_path(path_without_suffix)
116
+ if @append
117
+ "#{path_without_suffix}#{@suffix}"
118
+ else
119
+ i = 0
120
+ loop do
121
+ path = "#{path_without_suffix}.#{i}#{@suffix}"
122
+ return path unless File.exist?(path)
123
+ i += 1
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
@@ -24,25 +24,61 @@ module Fluent::Plugin
24
24
 
25
25
  DEFAULT_FORMAT_TYPE = 'json'
26
26
 
27
+ config_section :buffer do
28
+ config_set_default :chunk_keys, ['tag']
29
+ config_set_default :flush_at_shutdown, true
30
+ config_set_default :chunk_limit_size, 10 * 1024
31
+ end
32
+
27
33
  config_section :format do
28
34
  config_set_default :@type, DEFAULT_FORMAT_TYPE
29
35
  end
30
36
 
37
+ def prefer_buffered_processing
38
+ false
39
+ end
40
+
41
+ def prefer_delayed_commit
42
+ @delayed
43
+ end
44
+
45
+ attr_accessor :delayed
46
+
47
+ def initialize
48
+ super
49
+ @delayed = false
50
+ end
51
+
31
52
  def configure(conf)
32
53
  if conf['output_type'] && !conf['format']
33
54
  conf['format'] = conf['output_type']
34
55
  end
35
56
  compat_parameters_convert(conf, :inject, :formatter)
57
+
36
58
  super
59
+
37
60
  @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
38
61
  end
39
62
 
40
63
  def process(tag, es)
41
64
  es.each {|time,record|
42
- r = inject_values_to_record(tag, time, record)
43
- $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, r).chomp}\n"
65
+ $log.write(format(tag, time, record))
44
66
  }
45
67
  $log.flush
46
68
  end
69
+
70
+ def format(tag, time, record)
71
+ record = inject_values_to_record(tag, time, record)
72
+ "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
73
+ end
74
+
75
+ def write(chunk)
76
+ chunk.write_to($log)
77
+ end
78
+
79
+ def try_write(chunk)
80
+ chunk.write_to($log)
81
+ commit_write(chunk.unique_id)
82
+ end
47
83
  end
48
84
  end
@@ -138,7 +138,7 @@ module Fluent
138
138
  end
139
139
  end
140
140
 
141
- attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout
141
+ attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
142
142
  attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
143
143
 
144
144
  # for tests
@@ -185,13 +185,21 @@ module Fluent
185
185
  @simple_chunking = nil
186
186
  @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
187
187
  @flush_mode = nil
188
+ @timekey_zone = nil
188
189
  end
189
190
 
190
191
  def acts_as_secondary(primary)
191
192
  @as_secondary = true
192
193
  @primary_instance = primary
194
+ @chunk_keys = @primary_instance.chunk_keys || []
195
+ @chunk_key_tag = @primary_instance.chunk_key_tag || false
196
+ if @primary_instance.chunk_key_time
197
+ @chunk_key_time = @primary_instance.chunk_key_time
198
+ @timekey_zone = @primary_instance.timekey_zone
199
+ @output_time_formatter_cache = {}
200
+ end
201
+
193
202
  (class << self; self; end).module_eval do
194
- define_method(:extract_placeholders){ |str, metadata| @primary_instance.extract_placeholders(str, metadata) }
195
203
  define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
196
204
  define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) }
197
205
  end
@@ -251,7 +259,7 @@ module Fluent
251
259
  if @chunk_key_time
252
260
  raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
253
261
  Fluent::Timezone.validate!(@buffer_config.timekey_zone)
254
- @buffer_config.timekey_zone = '+0000' if @buffer_config.timekey_use_utc
262
+ @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
255
263
  @output_time_formatter_cache = {}
256
264
  end
257
265
 
@@ -477,13 +485,13 @@ module Fluent
477
485
 
478
486
  # TODO: optimize this code
479
487
  def extract_placeholders(str, metadata)
480
- if metadata.timekey.nil? && metadata.tag.nil? && metadata.variables.nil?
488
+ if metadata.empty?
481
489
  str
482
490
  else
483
491
  rvalue = str
484
492
  # strftime formatting
485
493
  if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
486
- @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@buffer_config.timekey_zone, str)
494
+ @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
487
495
  rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
488
496
  end
489
497
  # ${tag}, ${tag[0]}, ${tag[1]}, ...
@@ -91,7 +91,7 @@ module Fluent
91
91
  else
92
92
  conf.elements(name: 'source').each { |e|
93
93
  type = e['@type']
94
- raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
94
+ raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
95
95
  add_source(type, e)
96
96
  }
97
97
  end
@@ -0,0 +1,33 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'serverengine'
18
+
19
+ module Fluent
20
+ module Test
21
+ module StartupShutdown
22
+ def startup
23
+ socket_manager_path = ServerEngine::SocketManager::Server.generate_path
24
+ @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
25
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
26
+ end
27
+
28
+ def shutdown
29
+ @server.close
30
+ end
31
+ end
32
+ end
33
+ end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '0.14.4'
19
+ VERSION = '0.14.5'
20
20
 
21
21
  end
@@ -1,47 +1,73 @@
1
1
  require_relative '../helper'
2
2
 
3
3
  require 'fluent/test'
4
+ require 'fluent/test/startup_shutdown'
4
5
  require 'base64'
5
6
 
6
7
  require 'fluent/env'
7
8
  require 'fluent/plugin/in_forward'
8
9
 
9
10
  class ForwardInputTest < Test::Unit::TestCase
10
- class << self
11
- def startup
12
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
13
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
14
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
15
- end
16
-
17
- def shutdown
18
- @server.close
19
- end
20
- end
21
-
22
11
  def setup
23
12
  Fluent::Test.setup
24
13
  @responses = [] # for testing responses after sending data
25
14
  end
26
15
 
27
16
  PORT = unused_port
17
+
18
+ SHARED_KEY = 'foobar1'
19
+ USER_NAME = 'tagomoris'
20
+ USER_PASSWORD = 'fluentd'
21
+
28
22
  CONFIG = %[
29
23
  port #{PORT}
30
24
  bind 127.0.0.1
31
25
  ]
32
26
  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
27
+ CONFIG_AUTH = %[
28
+ port #{PORT}
29
+ bind 127.0.0.1
30
+ <security>
31
+ self_hostname localhost
32
+ shared_key foobar1
33
+ user_auth true
34
+ <user>
35
+ username #{USER_NAME}
36
+ password #{USER_PASSWORD}
37
+ </user>
38
+ <client>
39
+ network 127.0.0.0/8
40
+ shared_key #{SHARED_KEY}
41
+ users ["#{USER_NAME}"]
42
+ </client>
43
+ </security>
44
+ ]
33
45
 
34
46
  def create_driver(conf=CONFIG)
35
47
  Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
36
48
  end
37
49
 
38
- def test_configure
39
- d = create_driver
40
- assert_equal PORT, d.instance.port
41
- assert_equal '127.0.0.1', d.instance.bind
42
- assert_equal 0, d.instance.linger_timeout
43
- assert_equal 0.5, d.instance.blocking_timeout
44
- assert !d.instance.backlog
50
+ class Configure < self
51
+ def test_simple
52
+ d = create_driver
53
+ assert_equal PORT, d.instance.port
54
+ assert_equal '127.0.0.1', d.instance.bind
55
+ assert_equal 0, d.instance.linger_timeout
56
+ assert_equal 0.5, d.instance.blocking_timeout
57
+ assert !d.instance.backlog
58
+ end
59
+
60
+ def test_auth
61
+ d = create_driver(CONFIG_AUTH)
62
+ assert_equal PORT, d.instance.port
63
+ assert_equal '127.0.0.1', d.instance.bind
64
+ assert_equal 0, d.instance.linger_timeout
65
+ assert !d.instance.backlog
66
+
67
+ assert d.instance.security
68
+ assert_equal 1, d.instance.security.users.size
69
+ assert_equal 1, d.instance.security.clients.size
70
+ end
45
71
  end
46
72
 
47
73
  # TODO: Will add Loop::run arity check with stub/mock library
@@ -50,566 +76,1005 @@ class ForwardInputTest < Test::Unit::TestCase
50
76
  TCPSocket.new('127.0.0.1', PORT)
51
77
  end
52
78
 
53
- def test_time
54
- d = create_driver
79
+ class Message < self
80
+ extend Fluent::Test::StartupShutdown
55
81
 
56
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
57
- Fluent::Engine.now = time
58
-
59
- d.expect_emit "tag1", time, {"a"=>1}
60
- d.expect_emit "tag2", time, {"a"=>2}
61
-
62
- d.run do
63
- d.expected_emits.each {|tag, _time, record|
64
- send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
65
- }
66
- end
67
- end
82
+ def test_time
83
+ d = create_driver
68
84
 
69
- def test_message
70
- d = create_driver
85
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
86
+ Fluent::Engine.now = time
71
87
 
72
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
73
-
74
- d.expect_emit "tag1", time, {"a"=>1}
75
- d.expect_emit "tag2", time, {"a"=>2}
88
+ records = [
89
+ ["tag1", time, {"a"=>1}],
90
+ ["tag2", time, {"a"=>2}],
91
+ ]
76
92
 
77
- d.run do
78
- d.expected_emits.each {|tag, _time, record|
79
- send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
80
- }
93
+ d.expected_emits_length = records.length
94
+ d.run_timeout = 2
95
+ d.run do
96
+ records.each {|tag, _time, record|
97
+ send_data packer.write([tag, 0, record]).to_s
98
+ }
99
+ end
100
+ assert_equal(records, d.emits.sort_by {|a| a[0] })
81
101
  end
82
- end
83
102
 
84
- def test_message_with_time_as_integer
85
- d = create_driver
103
+ def test_plain
104
+ d = create_driver
86
105
 
87
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
106
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
88
107
 
89
- d.expect_emit "tag1", time, {"a"=>1}
90
- d.expect_emit "tag2", time, {"a"=>2}
108
+ records = [
109
+ ["tag1", time, {"a"=>1}],
110
+ ["tag2", time, {"a"=>2}],
111
+ ]
91
112
 
92
- d.run do
93
- d.expected_emits.each {|tag, _time, record|
94
- send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
95
- }
113
+ d.expected_emits_length = records.length
114
+ d.run_timeout = 2
115
+ d.run do
116
+ records.each {|tag, _time, record|
117
+ send_data packer.write([tag, _time, record]).to_s
118
+ }
119
+ end
120
+ assert_equal(records, d.emits)
96
121
  end
97
- end
98
122
 
99
- def test_message_with_skip_invalid_event
100
- d = create_driver(CONFIG + "skip_invalid_event true")
123
+ def test_time_as_integer
124
+ d = create_driver
101
125
 
102
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
126
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
103
127
 
104
- d.expect_emit "tag1", time, {"a" => 1}
105
- d.expect_emit "tag2", time, {"a" => 2}
128
+ records = [
129
+ ["tag1", time, {"a"=>1}],
130
+ ["tag2", time, {"a"=>2}],
131
+ ]
106
132
 
107
- d.run do
108
- entries = d.expected_emits.map {|tag, _time, record| [tag, _time, record] }
109
- # These entries are skipped
110
- entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
133
+ d.expected_emits_length = records.length
134
+ d.run_timeout = 2
135
+ d.run do
136
+ records.each {|tag, _time, record|
137
+ send_data packer.write([tag, _time, record]).to_s
138
+ }
139
+ end
111
140
 
112
- entries.each {|tag, _time, record|
113
- # Without ack, logs are sometimes not saved to logs during test.
114
- send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s, true
115
- }
141
+ assert_equal(records, d.emits)
116
142
  end
117
143
 
118
- assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
119
- end
120
-
121
- def test_forward
122
- d = create_driver
123
-
124
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
144
+ def test_skip_invalid_event
145
+ d = create_driver(CONFIG + "skip_invalid_event true")
125
146
 
126
- d.expect_emit "tag1", time, {"a"=>1}
127
- d.expect_emit "tag1", time, {"a"=>2}
128
-
129
- d.run do
130
- entries = []
131
- d.expected_emits.each {|tag, _time,record|
132
- entries << [time, record]
133
- }
134
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
135
- end
136
- end
147
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
137
148
 
138
- def test_forward_with_time_as_integer
139
- d = create_driver
149
+ records = [
150
+ ["tag1", time, {"a" => 1}],
151
+ ["tag2", time, {"a" => 2}],
152
+ ]
140
153
 
141
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
154
+ d.run do
155
+ entries = records.map { |tag, _time, record| [tag, _time, record] }
156
+ # These entries are skipped
157
+ entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
142
158
 
143
- d.expect_emit "tag1", time, {"a"=>1}
144
- d.expect_emit "tag1", time, {"a"=>2}
159
+ entries.each {|tag, _time, record|
160
+ # Without ack, logs are sometimes not saved to logs during test.
161
+ send_data packer.write([tag, _time, record]).to_s, try_to_receive_response: true
162
+ }
163
+ end
145
164
 
146
- d.run do
147
- entries = []
148
- d.expected_emits.each {|_tag, _time, record|
149
- entries << [_time, record]
150
- }
151
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
165
+ assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
166
+ assert_equal records[0], d.emits[0]
167
+ assert_equal records[1], d.emits[1]
152
168
  end
153
- end
154
169
 
155
- def test_forward_with_skip_invalid_event
156
- d = create_driver(CONFIG + "skip_invalid_event true")
170
+ def test_json
171
+ d = create_driver
157
172
 
158
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
173
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
159
174
 
160
- d.expect_emit "tag1", time, {"a" => 1}
161
- d.expect_emit "tag1", time, {"a" => 2}
175
+ records = [
176
+ ["tag1", time, {"a"=>1}],
177
+ ["tag2", time, {"a"=>2}],
178
+ ]
162
179
 
163
- d.run do
164
- entries = d.expected_emits.map {|_tag, _time, record| [_time, record] }
165
- # These entries are skipped
166
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
180
+ d.expected_emits_length = records.length
181
+ d.run_timeout = 2
182
+ d.run do
183
+ records.each {|tag, _time, record|
184
+ send_data [tag, _time, record].to_json
185
+ }
186
+ end
167
187
 
168
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
188
+ assert_equal(records, d.emits.sort_by {|a| a[1] })
169
189
  end
170
190
 
171
- assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
172
- end
191
+ def test_json_with_newline
192
+ d = create_driver
173
193
 
174
- def test_packed_forward
175
- d = create_driver
194
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
176
195
 
177
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
196
+ records = [
197
+ ["tag1", time, {"a"=>1}],
198
+ ["tag2", time, {"a"=>2}],
199
+ ]
178
200
 
179
- d.expect_emit "tag1", time, {"a"=>1}
180
- d.expect_emit "tag1", time, {"a"=>2}
201
+ d.expected_emits_length = records.length
202
+ d.run_timeout = 2
203
+ d.run do
204
+ records.each {|tag, _time, record|
205
+ send_data [tag, _time, record].to_json + "\n"
206
+ }
207
+ end
181
208
 
182
- d.run do
183
- entries = ''
184
- d.expected_emits.each {|_tag, _time, record|
185
- Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
186
- }
187
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
209
+ assert_equal(records, d.emits.sort_by {|a| a[1] })
188
210
  end
189
211
  end
190
212
 
191
- def test_packed_forward_with_time_as_integer
192
- d = create_driver
193
-
194
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
195
-
196
- d.expect_emit "tag1", time, {"a"=>1}
197
- d.expect_emit "tag1", time, {"a"=>2}
213
+ class Forward < self
214
+ extend Fluent::Test::StartupShutdown
215
+
216
+ data(tcp: {
217
+ config: CONFIG,
218
+ options: {
219
+ auth: false
220
+ }
221
+ },
222
+ auth: {
223
+ config: CONFIG_AUTH,
224
+ options: {
225
+ auth: true
226
+ }
227
+ })
228
+ def test_plain(data)
229
+ config = data[:config]
230
+ options = data[:options]
231
+ d = create_driver(config)
232
+
233
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
234
+
235
+ records = [
236
+ ["tag1", time, {"a"=>1}],
237
+ ["tag1", time, {"a"=>2}]
238
+ ]
239
+
240
+ d.expected_emits_length = records.length
241
+ d.run_timeout = 2
242
+ d.run do
243
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
244
+
245
+ entries = []
246
+ records.each {|tag, _time, record|
247
+ entries << [_time, record]
248
+ }
249
+ send_data packer.write(["tag1", entries]).to_s, **options
250
+ end
251
+ assert_equal(records, d.emits)
198
252
 
199
- d.run do
200
- entries = ''
201
- d.expected_emits.each {|_tag, _time, record|
202
- Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
203
- }
204
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
253
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
205
254
  end
206
- end
207
-
208
- def test_packed_forward_with_skip_invalid_event
209
- d = create_driver(CONFIG + "skip_invalid_event true")
210
255
 
211
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
212
-
213
- d.expect_emit "tag1", time, {"a" => 1}
214
- d.expect_emit "tag1", time, {"a" => 2}
256
+ data(tcp: {
257
+ config: CONFIG,
258
+ options: {
259
+ auth: false
260
+ }
261
+ },
262
+ auth: {
263
+ config: CONFIG_AUTH,
264
+ options: {
265
+ auth: true
266
+ }
267
+ })
268
+ def test_time_as_integer(data)
269
+ config = data[:config]
270
+ options = data[:options]
271
+ d = create_driver(config)
272
+
273
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
274
+
275
+ records = [
276
+ ["tag1", time, {"a"=>1}],
277
+ ["tag1", time, {"a"=>2}]
278
+ ]
279
+
280
+ d.expected_emits_length = records.length
281
+ d.run_timeout = 2
282
+ d.run do
283
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
284
+
285
+ entries = []
286
+ records.each {|tag, _time, record|
287
+ entries << [_time, record]
288
+ }
289
+ send_data packer.write(["tag1", entries]).to_s, **options
290
+ end
215
291
 
216
- d.run do
217
- entries = d.expected_emits.map {|_tag , _time, record| [_time, record] }
218
- # These entries are skipped
219
- entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
292
+ assert_equal(records, d.emits)
220
293
 
221
- packed_entries = ''
222
- entries.each {|_time, record|
223
- Fluent::Engine.msgpack_factory.packer(packed_entries).write([_time, record]).flush
224
- }
225
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
294
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
226
295
  end
227
296
 
228
- assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
229
- end
230
-
231
- def test_message_json
232
- d = create_driver
233
-
234
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
297
+ data(tcp: {
298
+ config: CONFIG,
299
+ options: {
300
+ auth: false
301
+ }
302
+ },
303
+ auth: {
304
+ config: CONFIG_AUTH,
305
+ options: {
306
+ auth: true
307
+ }
308
+ })
309
+ def test_skip_invalid_event(data)
310
+ config = data[:config]
311
+ options = data[:options]
312
+ d = create_driver(config + "skip_invalid_event true")
313
+
314
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
315
+
316
+ records = [
317
+ ["tag1", time, {"a" => 1}],
318
+ ["tag1", time, {"a" => 2}],
319
+ ]
320
+
321
+ d.expected_emits_length = records.length
322
+ d.run_timeout = 2
323
+ d.run do
324
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
325
+
326
+ entries = records.map { |tag, _time, record| [_time, record] }
327
+ # These entries are skipped
328
+ entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
329
+
330
+ send_data packer.write(["tag1", entries]).to_s, **options
331
+ end
235
332
 
236
- d.expect_emit "tag1", time, {"a"=>1}
237
- d.expect_emit "tag2", time, {"a"=>2}
333
+ assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
238
334
 
239
- d.run do
240
- d.expected_emits.each {|tag, _time, record|
241
- send_data [tag, _time, record].to_json
242
- }
335
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
243
336
  end
244
337
  end
245
338
 
246
- def test_set_size_to_option
247
- d = create_driver
339
+ class PackedForward < self
340
+ extend Fluent::Test::StartupShutdown
341
+
342
+ data(tcp: {
343
+ config: CONFIG,
344
+ options: {
345
+ auth: false
346
+ }
347
+ },
348
+ auth: {
349
+ config: CONFIG_AUTH,
350
+ options: {
351
+ auth: true
352
+ }
353
+ })
354
+ def test_plain(data)
355
+ config = data[:config]
356
+ options = data[:options]
357
+ d = create_driver(config)
358
+
359
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
360
+
361
+ records = [
362
+ ["tag1", time, {"a"=>1}],
363
+ ["tag1", time, {"a"=>2}],
364
+ ]
365
+
366
+ d.expected_emits_length = records.length
367
+ d.run_timeout = 2
368
+ d.run do
369
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
370
+
371
+ entries = ''
372
+ records.each {|_tag, _time, record|
373
+ packer(entries).write([_time, record]).flush
374
+ }
375
+ send_data packer.write(["tag1", entries]).to_s, **options
376
+ end
377
+ assert_equal(records, d.emits)
248
378
 
249
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
250
- events = [
251
- ["tag1", time, {"a"=>1}],
252
- ["tag1", time, {"a"=>2}]
253
- ]
379
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
380
+ end
254
381
 
255
- entries = ''
256
- events.each {|_tag, _time, record|
257
- [_time, record].to_msgpack(entries)
258
- }
382
+ data(tcp: {
383
+ config: CONFIG,
384
+ options: {
385
+ auth: false
386
+ }
387
+ },
388
+ auth: {
389
+ config: CONFIG_AUTH,
390
+ options: {
391
+ auth: true
392
+ }
393
+ })
394
+ def test_time_as_integer(data)
395
+ config = data[:config]
396
+ options = data[:options]
397
+ d = create_driver(config)
398
+
399
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
400
+
401
+ records = [
402
+ ["tag1", time, {"a"=>1}],
403
+ ["tag1", time, {"a"=>2}],
404
+ ]
405
+
406
+ d.expected_emits_length = records.length
407
+ d.run_timeout = 2
408
+ d.run do
409
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
410
+
411
+ entries = ''
412
+ records.each {|tag, _time, record|
413
+ packer(entries).write([_time, record]).flush
414
+ }
415
+ send_data packer.write(["tag1", entries]).to_s, **options
416
+ end
417
+ assert_equal(records, d.emits)
259
418
 
260
- chunk = ["tag1", entries, { 'size' => events.length }].to_msgpack
419
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
420
+ end
261
421
 
262
- d.run do
263
- Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
264
- option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
265
- assert_equal option['size'], events.length
422
+ data(tcp: {
423
+ config: CONFIG,
424
+ options: {
425
+ auth: false
426
+ }
427
+ },
428
+ auth: {
429
+ config: CONFIG_AUTH,
430
+ options: {
431
+ auth: true
432
+ }
433
+ })
434
+ def test_skip_invalid_event(data)
435
+ config = data[:config]
436
+ options = data[:options]
437
+ d = create_driver(config + "skip_invalid_event true")
438
+
439
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
440
+
441
+ records = [
442
+ ["tag1", time, {"a" => 1}],
443
+ ["tag1", time, {"a" => 2}],
444
+ ]
445
+
446
+ d.expected_emits_length = records.length
447
+ d.run_timeout = 2
448
+ d.run do
449
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
450
+
451
+ entries = records.map { |tag, _time, record| [_time, record] }
452
+ # These entries are skipped
453
+ entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
454
+
455
+ packed_entries = ''
456
+ entries.each { |_time, record|
457
+ packer(packed_entries).write([_time, record]).flush
458
+ }
459
+ send_data packer.write(["tag1", packed_entries]).to_s, **options
266
460
  end
461
+
462
+ assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
463
+
464
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
267
465
  end
268
466
  end
269
467
 
270
- def test_send_large_chunk_warning
271
- d = create_driver(CONFIG + %[
468
+ class Warning < self
469
+ extend Fluent::Test::StartupShutdown
470
+
471
+ def test_send_large_chunk_warning
472
+ d = create_driver(CONFIG + %[
272
473
  chunk_size_warn_limit 16M
273
474
  chunk_size_limit 32M
274
475
  ])
275
476
 
276
- time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
477
+ time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
277
478
 
278
- # generate over 16M chunk
279
- str = "X" * 1024 * 1024
280
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
281
- assert chunk.size > (16 * 1024 * 1024)
282
- assert chunk.size < (32 * 1024 * 1024)
479
+ # generate over 16M chunk
480
+ str = "X" * 1024 * 1024
481
+ chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
482
+ assert chunk.size > (16 * 1024 * 1024)
483
+ assert chunk.size < (32 * 1024 * 1024)
283
484
 
284
- d.run do
285
- Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
286
- d.instance.send(:on_message, obj, chunk.size, PEERADDR)
287
- end
288
- end
485
+ d.run do
486
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
289
487
 
290
- # check emitted data
291
- emits = d.emits
292
- assert_equal 16, emits.size
293
- assert emits.map(&:first).all?{|t| t == "test.tag" }
294
- assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
488
+ Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
489
+ d.instance.send(:on_message, obj, chunk.size, PEERADDR)
490
+ end
491
+ end
295
492
 
296
- # check log
297
- assert d.instance.log.logs.select{|line|
298
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
299
- line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
300
- }.size == 1, "large chunk warning is not logged"
301
- end
493
+ # check emitted data
494
+ emits = d.emits
495
+ assert_equal 16, emits.size
496
+ assert emits.map(&:first).all?{|t| t == "test.tag" }
497
+ assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
498
+
499
+ # check log
500
+ assert d.instance.log.logs.select{|line|
501
+ line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
502
+ line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
503
+ }.size == 1, "large chunk warning is not logged"
504
+ end
302
505
 
303
- def test_send_large_chunk_only_warning
304
- d = create_driver(CONFIG + %[
506
+ def test_send_large_chunk_only_warning
507
+ d = create_driver(CONFIG + %[
305
508
  chunk_size_warn_limit 16M
306
509
  ])
307
- time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
510
+ time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
308
511
 
309
- # generate over 16M chunk
310
- str = "X" * 1024 * 1024
311
- chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
512
+ # generate over 16M chunk
513
+ str = "X" * 1024 * 1024
514
+ chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
312
515
 
313
- d.run do
314
- Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
315
- d.instance.send(:on_message, obj, chunk.size, PEERADDR)
516
+ d.run do
517
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
518
+
519
+ Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
520
+ d.instance.send(:on_message, obj, chunk.size, PEERADDR)
521
+ end
316
522
  end
317
- end
318
523
 
319
- # check log
320
- assert d.instance.log.logs.select{ |line|
321
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
322
- line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
323
- }.size == 1, "large chunk warning is not logged"
324
- end
524
+ # check log
525
+ assert d.instance.log.logs.select{ |line|
526
+ line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
527
+ line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
528
+ }.size == 1, "large chunk warning is not logged"
529
+ end
325
530
 
326
- def test_send_large_chunk_limit
327
- d = create_driver(CONFIG + %[
531
+ def test_send_large_chunk_limit
532
+ d = create_driver(CONFIG + %[
328
533
  chunk_size_warn_limit 16M
329
534
  chunk_size_limit 32M
330
535
  ])
331
536
 
332
- time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
537
+ time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
333
538
 
334
- # generate over 32M chunk
335
- str = "X" * 1024 * 1024
336
- chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
337
- assert chunk.size > (32 * 1024 * 1024)
539
+ # generate over 32M chunk
540
+ str = "X" * 1024 * 1024
541
+ chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
542
+ assert chunk.size > (32 * 1024 * 1024)
338
543
 
339
- # d.run => send_data
340
- d.run do
341
- Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
342
- d.instance.send(:on_message, obj, chunk.size, PEERADDR)
343
- end
344
- end
345
-
346
- # check emitted data
347
- emits = d.emits
348
- assert_equal 0, emits.size
544
+ # d.run => send_data
545
+ d.run do
546
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
349
547
 
350
- # check log
351
- assert d.instance.log.logs.select{|line|
352
- line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
353
- line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
354
- }.size == 1, "large chunk warning is not logged"
355
- end
548
+ Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
549
+ d.instance.send(:on_message, obj, chunk.size, PEERADDR)
550
+ end
551
+ end
356
552
 
357
- data('string chunk' => 'broken string',
358
- 'integer chunk' => 10)
359
- def test_send_broken_chunk(data)
360
- d = create_driver
553
+ # check emitted data
554
+ emits = d.emits
555
+ assert_equal 0, emits.size
361
556
 
362
- # d.run => send_data
363
- d.run do
364
- d.instance.send(:on_message, data, 1000000000, PEERADDR)
557
+ # check log
558
+ assert d.instance.log.logs.select{|line|
559
+ line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
560
+ line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
561
+ }.size == 1, "large chunk warning is not logged"
365
562
  end
366
563
 
367
- # check emitted data
368
- emits = d.emits
369
- assert_equal 0, emits.size
370
-
371
- # check log
372
- assert d.instance.log.logs.select{|line|
373
- line =~ / \[warn\]: incoming chunk is broken: source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" msg=#{data.inspect}/
374
- }.size == 1, "should not accept broken chunk"
375
- end
376
-
377
- def test_respond_to_message_requiring_ack
378
- d = create_driver
564
+ data('string chunk' => 'broken string',
565
+ 'integer chunk' => 10)
566
+ def test_send_broken_chunk(data)
567
+ d = create_driver
379
568
 
380
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
569
+ # d.run => send_data
570
+ d.run do
571
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
381
572
 
382
- events = [
383
- ["tag1", time, {"a"=>1}],
384
- ["tag2", time, {"a"=>2}]
385
- ]
386
- d.expected_emits_length = events.length
573
+ d.instance.send(:on_message, data, 1000000000, PEERADDR)
574
+ end
387
575
 
388
- expected_acks = []
576
+ # check emitted data
577
+ emits = d.emits
578
+ assert_equal 0, emits.size
389
579
 
390
- d.run do
391
- events.each {|tag, _time, record|
392
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
393
- expected_acks << op['chunk']
394
- send_data [tag, _time, record, op].to_msgpack, true
395
- }
580
+ # check log
581
+ assert d.instance.log.logs.select{|line|
582
+ line =~ / \[warn\]: incoming chunk is broken: source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" msg=#{data.inspect}/
583
+ }.size == 1, "should not accept broken chunk"
396
584
  end
397
-
398
- assert_equal events, d.emits
399
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
400
585
  end
401
586
 
402
- # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
403
- def test_respond_to_forward_requiring_ack
404
- d = create_driver
405
-
406
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
407
-
408
- events = [
409
- ["tag1", time, {"a"=>1}],
410
- ["tag1", time, {"a"=>2}]
411
- ]
412
- d.expected_emits_length = events.length
587
+ class RespondToRequiringAck < self
588
+ extend Fluent::Test::StartupShutdown
589
+
590
+ data(tcp: {
591
+ config: CONFIG,
592
+ options: {
593
+ auth: false
594
+ }
595
+ },
596
+ auth: {
597
+ config: CONFIG_AUTH,
598
+ options: {
599
+ auth: true
600
+ }
601
+ })
602
+ def test_message(data)
603
+ config = data[:config]
604
+ options = data[:options]
605
+ d = create_driver(config)
606
+
607
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
608
+
609
+ events = [
610
+ ["tag1", time, {"a"=>1}],
611
+ ["tag2", time, {"a"=>2}]
612
+ ]
613
+ d.expected_emits_length = events.length
614
+
615
+ expected_acks = []
616
+
617
+ d.run do
618
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
619
+
620
+ events.each {|tag, _time, record|
621
+ op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
622
+ expected_acks << op['chunk']
623
+ send_data [tag, _time, record, op].to_msgpack, try_to_receive_response: true, **options
624
+ }
625
+ end
413
626
 
414
- expected_acks = []
627
+ assert_equal events, d.emits
628
+ assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
415
629
 
416
- d.run do
417
- entries = []
418
- events.each {|_tag, _time, record|
419
- entries << [time, record]
420
- }
421
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
422
- expected_acks << op['chunk']
423
- send_data ["tag1", entries, op].to_msgpack, true
630
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
424
631
  end
425
632
 
426
- assert_equal events, d.emits
427
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
428
- end
429
-
430
- def test_respond_to_packed_forward_requiring_ack
431
- d = create_driver
432
-
433
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
434
-
435
- events = [
436
- ["tag1", time, {"a"=>1}],
437
- ["tag1", time, {"a"=>2}]
438
- ]
439
- d.expected_emits_length = events.length
633
+ # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
634
+ data(tcp: {
635
+ config: CONFIG,
636
+ options: {
637
+ auth: false
638
+ }
639
+ },
640
+ auth: {
641
+ config: CONFIG_AUTH,
642
+ options: {
643
+ auth: true
644
+ }
645
+ })
646
+ def test_forward(data)
647
+ config = data[:config]
648
+ options = data[:options]
649
+ d = create_driver(config)
650
+
651
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
652
+
653
+ events = [
654
+ ["tag1", time, {"a"=>1}],
655
+ ["tag1", time, {"a"=>2}]
656
+ ]
657
+ d.expected_emits_length = events.length
658
+
659
+ expected_acks = []
660
+
661
+ d.run do
662
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
663
+
664
+ entries = []
665
+ events.each {|_tag, _time, record|
666
+ entries << [_time, record]
667
+ }
668
+ op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
669
+ expected_acks << op['chunk']
670
+ send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
671
+ end
440
672
 
441
- expected_acks = []
673
+ assert_equal events, d.emits
674
+ assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
442
675
 
443
- d.run do
444
- entries = ''
445
- events.each {|_tag, _time, record|
446
- [_time, record].to_msgpack(entries)
447
- }
448
- op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
449
- expected_acks << op['chunk']
450
- send_data ["tag1", entries, op].to_msgpack, true
676
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
451
677
  end
452
678
 
453
- assert_equal events, d.emits
454
- assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
455
- end
679
+ data(tcp: {
680
+ config: CONFIG,
681
+ options: {
682
+ auth: false
683
+ }
684
+ },
685
+ auth: {
686
+ config: CONFIG_AUTH,
687
+ options: {
688
+ auth: true
689
+ }
690
+ })
691
+ def test_packed_forward
692
+ config = data[:config]
693
+ options = data[:options]
694
+ d = create_driver(config)
695
+
696
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
697
+
698
+ events = [
699
+ ["tag1", time, {"a"=>1}],
700
+ ["tag1", time, {"a"=>2}]
701
+ ]
702
+ d.expected_emits_length = events.length
703
+
704
+ expected_acks = []
705
+
706
+ d.run do
707
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
708
+
709
+ entries = ''
710
+ events.each {|_tag, _time,record|
711
+ [time, record].to_msgpack(entries)
712
+ }
713
+ op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
714
+ expected_acks << op['chunk']
715
+ send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
716
+ end
456
717
 
457
- def test_respond_to_message_json_requiring_ack
458
- d = create_driver
718
+ assert_equal events, d.emits
719
+ assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
459
720
 
460
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
721
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
722
+ end
461
723
 
462
- events = [
463
- ["tag1", time, {"a"=>1}],
464
- ["tag2", time, {"a"=>2}]
465
- ]
466
- d.expected_emits_length = events.length
724
+ data(tcp: {
725
+ config: CONFIG,
726
+ options: {
727
+ auth: false
728
+ }
729
+ },
730
+ auth: {
731
+ config: CONFIG_AUTH,
732
+ options: {
733
+ auth: true
734
+ }
735
+ })
736
+ def test_message_json(data)
737
+ config = data[:config]
738
+ options = data[:options]
739
+ omit "with json, auth doen NOT work" if options[:auth]
740
+ d = create_driver(config)
741
+
742
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
743
+
744
+ events = [
745
+ ["tag1", time, {"a"=>1}],
746
+ ["tag2", time, {"a"=>2}]
747
+ ]
748
+ d.expected_emits_length = events.length
749
+
750
+ expected_acks = []
751
+
752
+ d.expected_emits_length = events.length
753
+ d.run_timeout = 2
754
+ d.run do
755
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
756
+
757
+ events.each {|tag, _time, record|
758
+ op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
759
+ expected_acks << op['chunk']
760
+ send_data [tag, _time, record, op].to_json, try_to_receive_response: true, **options
761
+ }
762
+ end
467
763
 
468
- expected_acks = []
764
+ assert_equal events, d.emits
765
+ assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
469
766
 
470
- d.run do
471
- events.each {|tag, _time, record|
472
- op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
473
- expected_acks << op['chunk']
474
- send_data [tag, _time, record, op].to_json, true
475
- }
767
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
476
768
  end
477
-
478
- assert_equal events, d.emits
479
- assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
480
769
  end
481
770
 
482
- def test_not_respond_to_message_not_requiring_ack
483
- d = create_driver
484
-
485
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
771
+ class NotRespondToNotRequiringAck < self
772
+ extend Fluent::Test::StartupShutdown
773
+
774
+ data(tcp: {
775
+ config: CONFIG,
776
+ options: {
777
+ auth: false
778
+ }
779
+ },
780
+ auth: {
781
+ config: CONFIG_AUTH,
782
+ options: {
783
+ auth: true
784
+ }
785
+ })
786
+ def test_message
787
+ config = data[:config]
788
+ options = data[:options]
789
+ d = create_driver(config)
790
+
791
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
792
+
793
+ events = [
794
+ ["tag1", time, {"a"=>1}],
795
+ ["tag2", time, {"a"=>2}]
796
+ ]
797
+ d.expected_emits_length = events.length
798
+
799
+ d.run do
800
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
801
+
802
+ events.each {|tag, _time, record|
803
+ send_data [tag, _time, record].to_msgpack, try_to_receive_response: true, **options
804
+ }
805
+ end
486
806
 
487
- events = [
488
- ["tag1", time, {"a"=>1}],
489
- ["tag2", time, {"a"=>2}]
490
- ]
491
- d.expected_emits_length = events.length
807
+ assert_equal events, d.emits
808
+ assert_equal ["", ""], @responses
492
809
 
493
- d.run do
494
- events.each {|tag, _time, record|
495
- send_data [tag, _time, record].to_msgpack, true
496
- }
810
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
497
811
  end
498
812
 
499
- assert_equal events, d.emits
500
- assert_equal [nil, nil], @responses
501
- end
502
-
503
- def test_not_respond_to_forward_not_requiring_ack
504
- d = create_driver
505
-
506
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
813
+ data(tcp: {
814
+ config: CONFIG,
815
+ options: {
816
+ auth: false
817
+ }
818
+ },
819
+ auth: {
820
+ config: CONFIG_AUTH,
821
+ options: {
822
+ auth: true
823
+ }
824
+ })
825
+ def test_forward(data)
826
+ config = data[:config]
827
+ options = data[:options]
828
+ d = create_driver(config)
829
+
830
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
831
+
832
+ events = [
833
+ ["tag1", time, {"a"=>1}],
834
+ ["tag1", time, {"a"=>2}]
835
+ ]
836
+ d.expected_emits_length = events.length
837
+
838
+ d.run do
839
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
840
+
841
+ entries = []
842
+ events.each {|tag, _time, record|
843
+ entries << [_time, record]
844
+ }
845
+ send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
846
+ end
507
847
 
508
- events = [
509
- ["tag1", time, {"a"=>1}],
510
- ["tag1", time, {"a"=>2}]
511
- ]
512
- d.expected_emits_length = events.length
848
+ assert_equal events, d.emits
849
+ assert_equal [nil], @responses
513
850
 
514
- d.run do
515
- entries = []
516
- events.each {|_tag, _time, record|
517
- entries << [_time, record]
518
- }
519
- send_data ["tag1", entries].to_msgpack, true
851
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
520
852
  end
521
853
 
522
- assert_equal events, d.emits
523
- assert_equal [nil], @responses
524
- end
854
+ data(tcp: {
855
+ config: CONFIG,
856
+ options: {
857
+ auth: false
858
+ }
859
+ },
860
+ auth: {
861
+ config: CONFIG_AUTH,
862
+ options: {
863
+ auth: true
864
+ }
865
+ })
866
+ def test_packed_forward(data)
867
+ config = data[:config]
868
+ options = data[:options]
869
+ d = create_driver(config)
870
+
871
+ time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
872
+
873
+ events = [
874
+ ["tag1", time, {"a"=>1}],
875
+ ["tag1", time, {"a"=>2}]
876
+ ]
877
+ d.expected_emits_length = events.length
878
+
879
+ d.run do
880
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
881
+
882
+ entries = ''
883
+ events.each {|tag, _time, record|
884
+ [_time, record].to_msgpack(entries)
885
+ }
886
+ send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
887
+ end
525
888
 
526
- def test_not_respond_to_packed_forward_not_requiring_ack
527
- d = create_driver
889
+ assert_equal events, d.emits
890
+ assert_equal [nil], @responses
528
891
 
529
- time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
892
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
893
+ end
530
894
 
531
- events = [
532
- ["tag1", time, {"a"=>1}],
533
- ["tag1", time, {"a"=>2}]
534
- ]
535
- d.expected_emits_length = events.length
895
+ data(tcp: {
896
+ config: CONFIG,
897
+ options: {
898
+ auth: false
899
+ }
900
+ },
901
+ auth: {
902
+ config: CONFIG_AUTH,
903
+ options: {
904
+ auth: true
905
+ }
906
+ })
907
+ def test_message_json(data)
908
+ config = data[:config]
909
+ options = data[:options]
910
+ omit "with json, auth doen NOT work" if options[:auth]
911
+ d = create_driver(config)
912
+
913
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
914
+
915
+ events = [
916
+ ["tag1", time, {"a"=>1}],
917
+ ["tag2", time, {"a"=>2}]
918
+ ]
919
+ d.expected_emits_length = events.length
920
+
921
+ d.run do
922
+ sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
923
+
924
+ events.each {|tag, _time, record|
925
+ send_data [tag, _time, record].to_json, try_to_receive_response: true, **options
926
+ }
927
+ end
536
928
 
537
- d.run do
538
- entries = ''
539
- events.each {|_tag, _time, record|
540
- [_time, record].to_msgpack(entries)
541
- }
542
- send_data ["tag1", entries].to_msgpack, true
929
+ assert_equal events, d.emits
930
+ assert_equal [nil, nil], @responses
931
+
932
+ sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
543
933
  end
934
+ end
544
935
 
545
- assert_equal events, d.emits
546
- assert_equal [nil], @responses
936
+ def packer(*args)
937
+ Fluent::Engine.msgpack_factory.packer(*args)
547
938
  end
548
939
 
549
- def test_not_respond_to_message_json_not_requiring_ack
550
- d = create_driver
940
+ def unpacker
941
+ Fluent::Engine.msgpack_factory.unpacker
942
+ end
551
943
 
552
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
944
+ # res
945
+ # '' : socket is disconnected without any data
946
+ # nil: socket read timeout
947
+ def read_data(io, timeout, &block)
948
+ res = ''
949
+ select_timeout = 2
950
+ timeout_at = Time.now + timeout
951
+ begin
952
+ buf = ''
953
+ io_activated = false
954
+ while Time.now < timeout_at
955
+ if IO.select([io], nil, nil, select_timeout)
956
+ io_activated = true
957
+ buf = io.readpartial(2048)
958
+ res ||= ''
959
+ res << buf
960
+
961
+ break if block.call(res)
962
+ end
963
+ end
964
+ res = nil unless io_activated # timeout without no data arrival
965
+ rescue Errno::EAGAIN
966
+ sleep 0.01
967
+ retry if res == ''
968
+ # if res is not empty, all data in socket buffer are read, so do not retry
969
+ rescue IOError, EOFError, Errno::ECONNRESET
970
+ # socket disconnected
971
+ end
972
+ res
973
+ end
553
974
 
554
- events = [
555
- ["tag1", time, {"a"=>1}],
556
- ["tag2", time, {"a"=>2}]
975
+ def simulate_auth_sequence(io, shared_key=SHARED_KEY, username=USER_NAME, password=USER_PASSWORD)
976
+ auth_response_timeout = 30
977
+ shared_key_salt = 'salt'
978
+
979
+ # reading helo
980
+ helo_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
981
+ raise "Authentication packet timeout" unless helo_data
982
+ raise "Authentication connection closed" if helo_data == ''
983
+ # ['HELO', options(hash)]
984
+ helo = MessagePack.unpack(helo_data)
985
+ raise "Invalid HELO header" unless helo[0] == 'HELO'
986
+ raise "Invalid HELO option object" unless helo[1].is_a?(Hash)
987
+ @options = helo[1]
988
+
989
+ # sending ping
990
+ ping = [
991
+ 'PING',
992
+ 'selfhostname',
993
+ shared_key_salt,
994
+ Digest::SHA512.new
995
+ .update(shared_key_salt)
996
+ .update('selfhostname')
997
+ .update(@options['nonce'])
998
+ .update(shared_key).hexdigest,
557
999
  ]
558
- d.expected_emits_length = events.length
559
-
560
- d.run do
561
- events.each {|tag, _time, record|
562
- send_data [tag, _time, record].to_json, true
563
- }
1000
+ if @options['auth'] # auth enabled -> value is auth salt
1001
+ pass_digest = Digest::SHA512.new.update(@options['auth']).update(username).update(password).hexdigest
1002
+ ping.push(username, pass_digest)
1003
+ else
1004
+ ping.push('', '')
564
1005
  end
565
-
566
- assert_equal events, d.emits
567
- assert_equal [nil, nil], @responses
1006
+ io.write ping.to_msgpack
1007
+ io.flush
1008
+
1009
+ # reading pong
1010
+ pong_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
1011
+ raise "PONG packet timeout" unless pong_data
1012
+ raise "PONG connection closed" if pong_data == ''
1013
+ # ['PING', bool(auth_result), string(reason_if_failed), self_hostname, shared_key_digest]
1014
+ pong = MessagePack.unpack(pong_data)
1015
+ raise "Invalid PONG header" unless pong[0] == 'PONG'
1016
+ raise "Authentication Failure: #{pong[2]}" unless pong[1]
1017
+ clientside_calculated = Digest::SHA512.new
1018
+ .update(shared_key_salt)
1019
+ .update(pong[3])
1020
+ .update(@options['nonce'])
1021
+ .update(shared_key).hexdigest
1022
+ raise "Shared key digest mismatch" unless clientside_calculated == pong[4]
1023
+
1024
+ # authentication success
1025
+ true
568
1026
  end
569
1027
 
570
- def send_data(data, try_to_receive_response=false, response_timeout=1)
1028
+ # Data ordering is not assured:
1029
+ # Records in different sockets are processed on different thread, so its scheduling make effect
1030
+ # on order of emitted records.
1031
+ # So, we MUST sort emitted records in different `send_data` before assertion.
1032
+ def send_data(data, try_to_receive_response: false, response_timeout: 5, auth: false)
571
1033
  io = connect
572
- begin
573
- io.write data
574
- if try_to_receive_response
575
- if IO.select([io], nil, nil, response_timeout)
576
- res = io.recv(1024)
577
- end
578
- # timeout means no response, so push nil to @responses
579
- end
580
- ensure
581
- io.close
1034
+
1035
+ if auth
1036
+ simulate_auth_sequence(io)
1037
+ end
1038
+
1039
+ io.write data
1040
+ io.flush
1041
+ if try_to_receive_response
1042
+ @responses << read_data(io, response_timeout){|d| MessagePack.unpack(d) rescue nil }
582
1043
  end
583
- @responses << res if try_to_receive_response
1044
+ ensure
1045
+ io.close rescue nil # SSL socket requires any writes to close sockets
584
1046
  end
585
1047
 
586
- # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
587
- test 'message protocol with source_hostname_key' do
588
- execute_test { |events|
589
- events.each { |tag, time, record|
590
- send_data [tag, time, record].to_msgpack
1048
+ sub_test_case 'source_hostname_key feature' do
1049
+ extend Fluent::Test::StartupShutdown
1050
+
1051
+ test 'message protocol with source_hostname_key' do
1052
+ execute_test { |events|
1053
+ events.each { |tag, time, record|
1054
+ send_data [tag, time, record].to_msgpack
1055
+ }
591
1056
  }
592
- }
593
- end
1057
+ end
594
1058
 
595
- test 'forward protocol with source_hostname_key' do
596
- execute_test { |events|
597
- entries = []
598
- events.each {|tag,time,record|
599
- entries << [time, record]
1059
+ test 'forward protocol with source_hostname_key' do
1060
+ execute_test { |events|
1061
+ entries = []
1062
+ events.each {|tag,time,record|
1063
+ entries << [time, record]
1064
+ }
1065
+ send_data ['tag1', entries].to_msgpack
600
1066
  }
601
- send_data ['tag1', entries].to_msgpack
602
- }
603
- end
1067
+ end
604
1068
 
605
- test 'packed forward protocol with source_hostname_key' do
606
- execute_test { |events|
607
- entries = ''
608
- events.each { |tag, time, record|
609
- Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
1069
+ test 'packed forward protocol with source_hostname_key' do
1070
+ execute_test { |events|
1071
+ entries = ''
1072
+ events.each { |tag, time, record|
1073
+ Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
1074
+ }
1075
+ send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
610
1076
  }
611
- send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
612
- }
1077
+ end
613
1078
  end
614
1079
 
615
1080
  def execute_test(&block)