fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -16,42 +16,59 @@
16
16
 
17
17
  require 'fileutils'
18
18
  require 'zlib'
19
+ require 'time'
19
20
 
20
- require 'fluent/output'
21
+ require 'fluent/plugin/output'
21
22
  require 'fluent/config/error'
22
- require 'fluent/system_config'
23
+ # TODO remove ...
24
+ require 'fluent/plugin/file_util'
23
25
 
24
- module Fluent
25
- class FileOutput < TimeSlicedOutput
26
- include SystemConfig::Mixin
26
+ module Fluent::Plugin
27
+ class FileOutput < Output
28
+ Fluent::Plugin.register_output('file', self)
27
29
 
28
- Plugin.register_output('file', self)
30
+ helpers :formatter, :inject, :compat_parameters
29
31
 
30
- SUPPORTED_COMPRESS = {
31
- 'gz' => :gz,
32
- 'gzip' => :gz,
32
+ SUPPORTED_COMPRESS = [:text, :gz, :gzip]
33
+ SUPPORTED_COMPRESS_MAP = {
34
+ text: nil,
35
+ gz: :gzip,
36
+ gzip: :gzip,
33
37
  }
34
38
 
35
39
  FILE_PERMISSION = 0644
36
40
  DIR_PERMISSION = 0755
37
41
 
42
+ DEFAULT_TIMEKEY = 60 * 60 * 24
43
+
38
44
  desc "The Path of the file."
39
45
  config_param :path, :string
40
- desc "The format of the file content. The default is out_file."
41
- config_param :format, :string, default: 'out_file', skip_accessor: true
46
+
47
+ desc "Specify to add file suffix for bare file path or not."
48
+ config_param :add_path_suffix, :bool, default: true
49
+ desc "The file suffix added to bare file path."
50
+ config_param :path_suffix, :string, default: '.log'
42
51
  desc "The flushed chunk is appended to existence file or not."
43
52
  config_param :append, :bool, default: false
44
53
  desc "Compress flushed file."
45
- config_param :compress, default: nil do |val|
46
- c = SUPPORTED_COMPRESS[val]
47
- unless c
48
- raise ConfigError, "Unsupported compression algorithm '#{val}'"
49
- end
50
- c
51
- end
52
- desc "Create symlink to temporary buffered file when buffer_type is file."
54
+ config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
55
+ desc "Execute compression again even when buffer chunk is already compressed."
56
+ config_param :recompress, :bool, default: false
57
+ desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
53
58
  config_param :symlink_path, :string, default: nil
54
59
 
60
+ config_section :format do
61
+ config_set_default :@type, 'out_file'
62
+ end
63
+
64
+ config_section :buffer do
65
+ config_set_default :@type, 'file'
66
+ config_set_default :chunk_keys, ['time']
67
+ config_set_default :timekey, DEFAULT_TIMEKEY
68
+ end
69
+
70
+ attr_accessor :last_written_path # for tests
71
+
55
72
  module SymlinkBufferMixin
56
73
  def symlink_path=(path)
57
74
  @_symlink_path = path
@@ -63,52 +80,81 @@ module Fluent
63
80
  # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
64
81
  # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
65
82
  # These chunks will be enqueued immediately, and will be flushed soon.
66
- latest_chunk = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
67
- if chunk.metadata == latest_chunk
83
+ latest_metadata = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
84
+ if chunk.metadata == latest_metadata
68
85
  FileUtils.ln_sf(chunk.path, @_symlink_path)
69
86
  end
70
87
  chunk
71
88
  end
72
89
  end
73
90
 
74
- def initialize
75
- require 'zlib'
76
- require 'time'
77
- require 'fluent/plugin/file_util'
78
- super
79
- end
80
-
81
91
  def configure(conf)
82
- if path = conf['path']
83
- @path = path
92
+ compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: "time")
93
+
94
+ configured_time_slice_format = conf['time_slice_format']
95
+
96
+ # v0.14 file buffer handles path as directory if '*' is missing
97
+ # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
98
+ # but raise it in this plugin.
99
+ if conf.elements(name: 'buffer').empty?
100
+ conf.add_element('buffer', 'time')
84
101
  end
85
- unless @path
86
- raise ConfigError, "'path' parameter is required on file output"
102
+ buffer_conf = conf.elements(name: 'buffer').first
103
+ unless buffer_conf.has_key?('path')
104
+ buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
87
105
  end
88
106
 
89
- if pos = @path.index('*')
90
- @path_prefix = @path[0,pos]
91
- @path_suffix = @path[pos+1..-1]
92
- conf['buffer_path'] ||= "#{@path}"
93
- else
94
- @path_prefix = @path+"."
95
- @path_suffix = ".log"
96
- conf['buffer_path'] ||= "#{@path}.*"
97
- end
107
+ super
108
+
109
+ @compress_method = SUPPORTED_COMPRESS_MAP[@compress]
98
110
 
99
- test_path = generate_path(Time.now.strftime(@time_slice_format))
100
- unless ::Fluent::FileUtil.writable_p?(test_path)
101
- raise ConfigError, "out_file: `#{test_path}` is not writable"
111
+ if @path.include?('*') && !@buffer_config.timekey
112
+ raise Fluent::ConfigError, "path including '*' must be used with buffer chunk key 'time'"
102
113
  end
103
114
 
104
- super
115
+ path_suffix = @add_path_suffix ? @path_suffix : ''
116
+ path_timekey = if @chunk_key_time
117
+ @as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey
118
+ else
119
+ nil
120
+ end
121
+ @path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
105
122
 
106
- @formatter = Plugin.new_formatter(@format)
107
- @formatter.configure(conf)
123
+ if @as_secondary
124
+ # When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.
125
+ # Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.
126
+ placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|
127
+ v.validate!
128
+ end
129
+ else
130
+ placeholder_validate!(:path, @path_template)
131
+
132
+ max_tag_index = get_placeholders_tag(@path_template).max || 1
133
+ max_tag_index = 1 if max_tag_index < 1
134
+ dummy_tag = (['a'] * max_tag_index).join('.')
135
+ dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
136
+ dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
137
+
138
+ test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
139
+ test_path = extract_placeholders(@path_template, test_meta1)
140
+ unless ::Fluent::FileUtil.writable_p?(test_path)
141
+ raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
142
+ end
143
+ end
144
+
145
+ @formatter = formatter_create
108
146
 
109
147
  if @symlink_path && @buffer.respond_to?(:path)
110
- @buffer.extend SymlinkBufferMixin
111
- @buffer.symlink_path = @symlink_path
148
+ if @as_secondary
149
+ raise Fluent::ConfigError, "symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin"
150
+ end
151
+ if Fluent.windows?
152
+ log.warn "symlink_path is unavailable on Windows platform. disabled."
153
+ @symlink_path = nil
154
+ else
155
+ @buffer.extend SymlinkBufferMixin
156
+ @buffer.symlink_path = @symlink_path
157
+ end
112
158
  end
113
159
 
114
160
  @dir_perm = system_config.dir_permission || DIR_PERMISSION
@@ -116,56 +162,101 @@ module Fluent
116
162
  end
117
163
 
118
164
  def format(tag, time, record)
119
- @formatter.format(tag, time, record)
165
+ r = inject_values_to_record(tag, time, record)
166
+ @formatter.format(tag, time, r)
120
167
  end
121
168
 
122
169
  def write(chunk)
123
- path = generate_path(chunk.key)
170
+ path = extract_placeholders(@path_template, chunk.metadata)
124
171
  FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
125
172
 
126
- case @compress
173
+ unless @append
174
+ path = find_filepath_available(path)
175
+ end
176
+
177
+ case @compress_method
127
178
  when nil
128
- File.open(path, "ab", @file_perm) {|f|
179
+ File.open(path, "ab", @file_perm) do |f|
129
180
  chunk.write_to(f)
130
- }
131
- when :gz
132
- File.open(path, "ab", @file_perm) {|f|
133
- gz = Zlib::GzipWriter.new(f)
134
- chunk.write_to(gz)
135
- gz.close
136
- }
181
+ end
182
+ when :gzip
183
+ if @buffer.compress != :gzip || @recompress
184
+ File.open(path, "ab", @file_perm) do |f|
185
+ gz = Zlib::GzipWriter.new(f)
186
+ chunk.write_to(gz, compressed: :text)
187
+ gz.close
188
+ end
189
+ else
190
+ File.open(path, "ab", @file_perm) do |f|
191
+ chunk.write_to(f, compressed: :gzip)
192
+ end
193
+ end
194
+ else
195
+ raise "BUG: unknown compression method #{@compress_method}"
137
196
  end
138
197
 
139
- return path # for test
198
+ @last_written_path = path
140
199
  end
141
200
 
142
- def secondary_init(primary)
143
- # don't warn even if primary.class is not FileOutput
201
+ def timekey_to_timeformat(timekey)
202
+ case timekey
203
+ when nil then ''
204
+ when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
205
+ when 60...3600 then '%Y%m%d%H%M'
206
+ when 3600...86400 then '%Y%m%d%H'
207
+ else '%Y%m%d'
208
+ end
144
209
  end
145
210
 
146
- private
147
-
148
- def suffix
149
- case @compress
150
- when nil
151
- ''
152
- when :gz
153
- ".gz"
211
+ def compression_suffix(compress)
212
+ case compress
213
+ when :gzip then '.gz'
214
+ when nil then ''
215
+ else
216
+ raise ArgumentError, "unknown compression type #{compress}"
154
217
  end
155
218
  end
156
219
 
157
- def generate_path(time_string)
158
- if @append
159
- "#{@path_prefix}#{time_string}#{@path_suffix}#{suffix}"
220
+ # /path/to/dir/file.* -> /path/to/dir/file.%Y%m%d
221
+ # /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data
222
+ # /path/to/dir/file -> /path/to/dir/file.%Y%m%d.log
223
+ # %Y%m%d -> %Y%m%d_** (non append)
224
+ # + .gz (gzipped)
225
+ ## TODO: remove time_slice_format when end of support of compat_parameters
226
+ def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)
227
+ comp_suffix = compression_suffix(compress)
228
+ index_placeholder = append ? '' : '_**'
229
+ if original.index('*')
230
+ raise "BUG: configuration error must be raised for path including '*' without timekey" unless timekey
231
+ time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)
232
+ original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix
160
233
  else
161
- path = nil
162
- i = 0
163
- begin
164
- path = "#{@path_prefix}#{time_string}_#{i}#{@path_suffix}#{suffix}"
165
- i += 1
166
- end while File.exist?(path)
167
- path
234
+ if timekey
235
+ if time_slice_format
236
+ "#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}"
237
+ else
238
+ time_placeholders = timekey_to_timeformat(timekey)
239
+ if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }
240
+ raise Fluent::ConfigError, "insufficient timestamp placeholders in path" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }
241
+ "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
242
+ else
243
+ "#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}"
244
+ end
245
+ end
246
+ else
247
+ "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
248
+ end
249
+ end
250
+ end
251
+
252
+ def find_filepath_available(path_with_placeholder) # for non-append
253
+ raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
254
+ i = 0
255
+ while path = path_with_placeholder.sub('_**', "_#{i}")
256
+ break unless File.exist?(path)
257
+ i += 1
168
258
  end
259
+ path
169
260
  end
170
261
  end
171
262
  end
@@ -14,44 +14,30 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'base64'
18
- require 'socket'
19
- require 'fileutils'
20
-
21
- require 'cool.io'
22
-
23
17
  require 'fluent/output'
24
18
  require 'fluent/config/error'
19
+ require 'base64'
25
20
 
26
- module Fluent
27
- class ForwardOutputError < StandardError
28
- end
29
-
30
- class ForwardOutputResponseError < ForwardOutputError
31
- end
21
+ require 'fluent/compat/socket_util'
32
22
 
33
- class ForwardOutputConnectionClosedError < ForwardOutputError
34
- end
23
+ module Fluent::Plugin
24
+ class ForwardOutput < Output
25
+ class Error < StandardError; end
26
+ class NoNodesAvailable < Error; end
27
+ class ConnectionClosedError < Error; end
35
28
 
36
- class ForwardOutputACKTimeoutError < ForwardOutputResponseError
37
- end
29
+ Fluent::Plugin.register_output('forward', self)
38
30
 
39
- class ForwardOutput < ObjectBufferedOutput
40
- Plugin.register_output('forward', self)
31
+ helpers :socket, :server, :timer, :thread, :compat_parameters
41
32
 
42
33
  LISTEN_PORT = 24224
43
34
 
44
- def initialize
45
- super
46
- require 'fluent/plugin/socket_util'
47
- @nodes = [] #=> [Node]
48
- @loop = nil
49
- @thread = nil
50
- @finished = false
51
- end
35
+ PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
52
36
 
53
37
  desc 'The timeout time when sending event logs.'
54
38
  config_param :send_timeout, :time, default: 60
39
+ # TODO: add linger_timeout, recv_timeout
40
+
55
41
  desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
56
42
  config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
57
43
  desc 'The interval of the heartbeat packer.'
@@ -60,8 +46,6 @@ module Fluent
60
46
  config_param :recover_wait, :time, default: 10
61
47
  desc 'The hard timeout used to detect server failure.'
62
48
  config_param :hard_timeout, :time, default: 60
63
- desc 'Set TTL to expire DNS cache in seconds.'
64
- config_param :expire_dns_cache, :time, default: nil # 0 means disable cache
65
49
  desc 'The threshold parameter used to detect server faults.'
66
50
  config_param :phi_threshold, :integer, default: 16
67
51
  desc 'Use the "Phi accrual failure detector" to detect server failure.'
@@ -69,14 +53,20 @@ module Fluent
69
53
 
70
54
  desc 'Change the protocol to at-least-once.'
71
55
  config_param :require_ack_response, :bool, default: false # require in_forward to respond with ack
56
+
57
+ ## The reason of default value of :ack_response_timeout:
58
+ # Linux default tcp_syn_retries is 5 (in many environment)
59
+ # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
72
60
  desc 'This option is used when require_ack_response is true.'
73
61
  config_param :ack_response_timeout, :time, default: 190
74
- desc 'Reading data size from server'
75
- config_param :read_length, :size, default: 512 # 512bytes
62
+
76
63
  desc 'The interval while reading data from server'
77
64
  config_param :read_interval_msec, :integer, default: 50 # 50ms
78
- # Linux default tcp_syn_retries is 5 (in many environment)
79
- # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
65
+ desc 'Reading data size from server'
66
+ config_param :read_length, :size, default: 512 # 512bytes
67
+
68
+ desc 'Set TTL to expire DNS cache in seconds.'
69
+ config_param :expire_dns_cache, :time, default: nil # 0 means disable cache
80
70
  desc 'Enable client-side DNS round robin.'
81
71
  config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
82
72
 
@@ -114,17 +104,39 @@ module Fluent
114
104
  config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
115
105
  config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
116
106
 
107
+ config_section :buffer do
108
+ config_set_default :chunk_keys, ["tag"]
109
+ end
110
+
117
111
  attr_reader :read_interval, :recover_sample_size
118
112
 
113
+ def initialize
114
+ super
115
+
116
+ @nodes = [] #=> [Node]
117
+ @loop = nil
118
+ @thread = nil
119
+
120
+ @usock = nil
121
+ @sock_ack_waiting = nil
122
+ @sock_ack_waiting_mutex = nil
123
+ end
124
+
119
125
  def configure(conf)
126
+ compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
127
+
120
128
  super
121
129
 
130
+ unless @chunk_key_tag
131
+ raise Fluent::ConfigError, "buffer chunk key must include 'tag' for forward output"
132
+ end
133
+
122
134
  @read_interval = @read_interval_msec / 1000.0
123
135
  @recover_sample_size = @recover_wait / @heartbeat_interval
124
136
 
125
137
  if @dns_round_robin
126
138
  if @heartbeat_type == :udp
127
- raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
139
+ raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
128
140
  end
129
141
  end
130
142
 
@@ -140,90 +152,119 @@ module Fluent
140
152
  end
141
153
  end
142
154
 
143
- if @compress == :gzip && @buffer.compress == :text
144
- @buffer.compress = :gzip
145
- elsif @compress == :text && @buffer.compress == :gzip
146
- log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
155
+ unless @as_secondary
156
+ if @compress == :gzip && @buffer.compress == :text
157
+ @buffer.compress = :gzip
158
+ elsif @compress == :text && @buffer.compress == :gzip
159
+ log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
160
+ end
147
161
  end
148
162
 
149
163
  if @nodes.empty?
150
- raise ConfigError, "forward output plugin requires at least one <server> is required"
164
+ raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
151
165
  end
152
166
 
153
167
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
154
168
  end
155
169
 
170
+ def prefer_delayed_commit
171
+ @require_ack_response
172
+ end
173
+
156
174
  def start
157
175
  super
158
176
 
177
+ # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
178
+ # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
179
+ if @ack_response_timeout && @delayed_commit_timeout != @ack_response_timeout
180
+ log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
181
+ @delayed_commit_timeout = @ack_response_timeout
182
+ end
183
+
159
184
  @rand_seed = Random.new.seed
160
185
  rebuild_weight_array
161
186
  @rr = 0
162
- @usock = nil
163
187
 
164
188
  unless @heartbeat_type == :none
165
- @loop = Coolio::Loop.new
166
-
167
189
  if @heartbeat_type == :udp
168
- # assuming all hosts use udp
169
- @usock = SocketUtil.create_udp_socket(@nodes.first.host)
170
- @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
171
- @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
172
- @loop.attach(@hb)
190
+ @usock = socket_create_udp(@nodes.first.host, @nodes.first.port, nonblock: true)
191
+ server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length) do |data, sock|
192
+ sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
193
+ on_heartbeat(sockaddr, data)
194
+ end
173
195
  end
196
+ timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_timer))
197
+ end
174
198
 
175
- @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
176
- @loop.attach(@timer)
177
-
178
- @thread = Thread.new(&method(:run))
199
+ if @require_ack_response
200
+ @sock_ack_waiting_mutex = Mutex.new
201
+ @sock_ack_waiting = []
202
+ thread_create(:out_forward_receiving_ack, &method(:ack_reader))
179
203
  end
180
204
  end
181
205
 
182
- def shutdown
183
- @finished = true
184
- if @loop
185
- @loop.watchers.each {|w| w.detach }
186
- @loop.stop
187
- end
188
- @thread.join if @thread
206
+ def close
189
207
  @usock.close if @usock
190
-
191
208
  super
192
209
  end
193
210
 
194
- def run
195
- @loop.run if @loop
196
- rescue
197
- log.error "unexpected error", error: $!.to_s
198
- log.error_backtrace
211
+ def write(chunk)
212
+ return if chunk.empty?
213
+ tag = chunk.metadata.tag
214
+ select_a_healthy_node{|node| node.send_data(tag, chunk) }
199
215
  end
200
216
 
201
- def write_objects(tag, chunk)
202
- return if chunk.empty?
217
+ ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :node, :time, :timeout) do
218
+ def expired?(now)
219
+ time + timeout < now
220
+ end
221
+ end
222
+
223
+ def try_write(chunk)
224
+ if chunk.empty?
225
+ commit_write(chunk.unique_id)
226
+ return
227
+ end
228
+ tag = chunk.metadata.tag
229
+ sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
230
+ chunk_id = Base64.encode64(chunk.unique_id)
231
+ current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
232
+ info = ACKWaitingSockInfo.new(sock, chunk_id, node, current_time, @ack_response_timeout)
233
+ @sock_ack_waiting_mutex.synchronize do
234
+ @sock_ack_waiting << info
235
+ end
236
+ end
203
237
 
238
+ def select_a_healthy_node
204
239
  error = nil
205
240
 
206
241
  wlen = @weight_array.length
207
242
  wlen.times do
208
243
  @rr = (@rr + 1) % wlen
209
244
  node = @weight_array[@rr]
245
+ next unless node.available?
210
246
 
211
- if node.available?
212
- begin
213
- node.send_data(tag, chunk)
214
- return
215
- rescue
216
- # for load balancing during detecting crashed servers
217
- error = $! # use the latest error
218
- end
247
+ begin
248
+ ret = yield node
249
+ return ret, node
250
+ rescue
251
+ # for load balancing during detecting crashed servers
252
+ error = $! # use the latest error
219
253
  end
220
254
  end
221
255
 
222
- if error
223
- raise error
224
- else
225
- raise "no nodes are available" # TODO message
226
- end
256
+ raise error if error
257
+ raise NoNodesAvailable, "no nodes are available"
258
+ end
259
+
260
+ def create_transfer_socket(host, port, &block)
261
+ socket_create_tcp(
262
+ host, port,
263
+ linger_timeout: @send_timeout,
264
+ send_timeout: @send_timeout,
265
+ recv_timeout: @ack_response_timeout,
266
+ &block
267
+ )
227
268
  end
228
269
 
229
270
  # MessagePack FixArray length is 3
@@ -276,21 +317,7 @@ module Fluent
276
317
  @weight_array = weight_array
277
318
  end
278
319
 
279
- class HeartbeatRequestTimer < Coolio::TimerWatcher
280
- def initialize(interval, callback)
281
- super(interval, true)
282
- @callback = callback
283
- end
284
-
285
- def on_timer
286
- @callback.call
287
- rescue
288
- # TODO log?
289
- end
290
- end
291
-
292
320
  def on_timer
293
- return if @finished
294
321
  @nodes.each {|n|
295
322
  if n.tick
296
323
  rebuild_weight_array
@@ -305,33 +332,86 @@ module Fluent
305
332
  }
306
333
  end
307
334
 
308
- class HeartbeatHandler < Coolio::IO
309
- def initialize(io, callback)
310
- super(io)
311
- @io = io
312
- @callback = callback
335
+ def on_heartbeat(sockaddr, msg)
336
+ if node = @nodes.find {|n| n.sockaddr == sockaddr }
337
+ # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
338
+ if node.heartbeat
339
+ rebuild_weight_array
340
+ end
313
341
  end
342
+ end
314
343
 
315
- def on_readable
316
- begin
317
- msg, addr = @io.recvfrom(1024)
318
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
319
- return
344
+ # return chunk id when succeeded for tests
345
+ def read_ack_from_sock(sock, unpacker)
346
+ begin
347
+ raw_data = sock.recv(@read_length)
348
+ rescue Errno::ECONNRESET
349
+ raw_data = ""
350
+ end
351
+ info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
352
+
353
+ # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
354
+ # If this happens we assume the data wasn't delivered and retry it.
355
+ if raw_data.empty?
356
+ log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
357
+ info.node.disable!
358
+ return nil
359
+ else
360
+ unpacker.feed(raw_data)
361
+ res = unpacker.read
362
+ if res['ack'] != info.chunk_id
363
+ # Some errors may have occured when ack and chunk id is different, so send the chunk again.
364
+ log.warn "ack in response and chunk id in sent data are different", chunk_id: info.chunk_id, ack: res['ack']
365
+ rollback_write(info.chunk_id)
366
+ return nil
320
367
  end
321
- host = addr[3]
322
- port = addr[1]
323
- sockaddr = Socket.pack_sockaddr_in(port, host)
324
- @callback.call(sockaddr, msg)
325
- rescue
326
- # TODO log?
368
+ return info.chunk_id
369
+ end
370
+ rescue => e
371
+ log.error "unexpected error while receiving ack message", error: e
372
+ log.error_backtrace
373
+ ensure
374
+ @sock_ack_waiting_mutex.synchronize do
375
+ @sock_ack_waiting.delete(info)
327
376
  end
328
377
  end
329
378
 
330
- def on_heartbeat(sockaddr, msg)
331
- if node = @nodes.find {|n| n.sockaddr == sockaddr }
332
- #log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
333
- if node.heartbeat
334
- rebuild_weight_array
379
+ def ack_reader
380
+ select_interval = if @delayed_commit_timeout > 3
381
+ 2
382
+ else
383
+ @delayed_commit_timeout / 2.0
384
+ end
385
+
386
+ unpacker = Fluent::Engine.msgpack_unpacker
387
+
388
+ while thread_current_running?
389
+ now = Process.clock_gettime(PROCESS_CLOCK_ID)
390
+ sockets = []
391
+ @sock_ack_waiting_mutex.synchronize do
392
+ new_list = []
393
+ @sock_ack_waiting.each do |info|
394
+ if info.expired?(now)
395
+ # There are 2 types of cases when no response has been received from socket:
396
+ # (1) the node does not support sending responses
397
+ # (2) the node does support sending response but responses have not arrived for some reasons.
398
+ log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
399
+ info.node.disable!
400
+ info.sock.close rescue nil
401
+ rollback_write(info.chunk_id)
402
+ else
403
+ sockets << info.sock
404
+ new_list << info
405
+ end
406
+ end
407
+ @sock_ack_waiting = new_list
408
+ end
409
+
410
+ readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
411
+ next unless readable_sockets
412
+
413
+ readable_sockets.each do |sock|
414
+ read_ack_from_sock(sock, unpacker)
335
415
  end
336
416
  end
337
417
  end
@@ -384,20 +464,6 @@ module Fluent
384
464
  @standby
385
465
  end
386
466
 
387
- def connect
388
- TCPSocket.new(resolved_host, port)
389
- end
390
-
391
- def set_socket_options(sock)
392
- opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
393
- sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
394
-
395
- opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval
396
- sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
397
-
398
- sock
399
- end
400
-
401
467
  def establish_connection(sock)
402
468
  while available? && @state != :established
403
469
  begin
@@ -428,98 +494,60 @@ module Fluent
428
494
  end
429
495
  end
430
496
 
431
- def send_data(tag, chunk)
432
- sock = connect
497
+ def send_data_actual(sock, tag, chunk)
433
498
  @state = @sender.security ? :helo : :established
434
- begin
435
- set_socket_options(sock)
436
-
437
- if @state != :established
438
- establish_connection(sock)
439
- end
499
+ if @state != :established
500
+ establish_connection(sock)
501
+ end
440
502
 
441
- unless available?
442
- raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
443
- end
503
+ unless available?
504
+ raise ConnectionClosedError, "failed to establish connection with node #{@name}"
505
+ end
444
506
 
445
- option = { 'size' => chunk.size_of_events, 'compressed' => @compress }
446
- option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
507
+ option = { 'size' => chunk.size, 'compressed' => @compress }
508
+ option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
447
509
 
448
- # out_forward always uses Raw32 type for content.
449
- # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
510
+ # out_forward always uses Raw32 type for content.
511
+ # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
450
512
 
451
- sock.write @sender.forward_header # beginArray(3)
452
- sock.write tag.to_msgpack # 1. writeRaw(tag)
453
- chunk.open(compressed: @compress) do |chunk_io|
454
- sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
455
- IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es)
456
- end
457
- sock.write option.to_msgpack # 3. writeOption(option)
458
-
459
- if @sender.require_ack_response
460
- # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
461
- # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
462
- # and process them asynchronously.
463
- if IO.select([sock], nil, nil, @sender.ack_response_timeout)
464
- raw_data = begin
465
- sock.recv(1024)
466
- rescue Errno::ECONNRESET
467
- ""
468
- end
469
-
470
- # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
471
- # If this happens we assume the data wasn't delivered and retry it.
472
- if raw_data.empty?
473
- @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
474
- disable!
475
- raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection"
476
- else
477
- @unpacker.feed(raw_data)
478
- res = @unpacker.read
479
- if res['ack'] != option['chunk']
480
- # Some errors may have occured when ack and chunk id is different, so send the chunk again.
481
- raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
482
- end
483
- end
513
+ sock.write @sender.forward_header # beginArray(3)
514
+ sock.write tag.to_msgpack # 1. writeRaw(tag)
515
+ chunk.open(compressed: @compress) do |chunk_io|
516
+ sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
517
+ IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es)
518
+ end
519
+ sock.write option.to_msgpack # 3. writeOption(option)
520
+ end
484
521
 
485
- else
486
- # IO.select returns nil on timeout.
487
- # There are 2 types of cases when no response has been received:
488
- # (1) the node does not support sending responses
489
- # (2) the node does support sending response but responses have not arrived for some reasons.
490
- @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
491
- disable!
492
- raise ForwardOutputACKTimeoutError, "node #{host}:#{port} does not return ACK"
493
- end
494
- end
522
+ def send_data(tag, chunk)
523
+ sock = @sender.create_transfer_socket(resolved_host, port)
524
+ begin
525
+ send_data_actual(sock, tag, chunk)
526
+ rescue
527
+ sock.close rescue nil
528
+ raise
529
+ end
495
530
 
496
- heartbeat(false)
497
- res # for test
498
- ensure
499
- sock.close_write
500
- sock.close
531
+ if @sender.require_ack_response
532
+ return sock # to read ACK from socket
501
533
  end
534
+
535
+ sock.close_write rescue nil
536
+ sock.close rescue nil
537
+ heartbeat(false)
538
+ nil
502
539
  end
503
540
 
504
541
  # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
505
542
  def send_heartbeat
506
543
  case @sender.heartbeat_type
507
544
  when :tcp
508
- sock = connect
509
- begin
510
- opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
511
- sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
512
- # opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval
513
- # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
514
-
545
+ @sender.create_transfer_socket(resolved_host, port) do |sock|
515
546
  ## don't send any data to not cause a compatibility problem
516
547
  # sock.write FORWARD_TCP_HEARTBEAT_DATA
517
548
 
518
549
  # successful tcp connection establishment is considered as valid heartbeat
519
550
  heartbeat(true)
520
- ensure
521
- sock.close_write
522
- sock.close
523
551
  end
524
552
  when :udp
525
553
  @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
@@ -541,7 +569,7 @@ module Fluent
541
569
  @resolved_host ||= resolve_dns!
542
570
 
543
571
  else
544
- now = Engine.now
572
+ now = Fluent::Engine.now
545
573
  rh = @resolved_host
546
574
  if !rh || now - @resolved_time >= @sender.expire_dns_cache
547
575
  rh = @resolved_host = resolve_dns!
@@ -601,11 +629,6 @@ module Fluent
601
629
  end
602
630
  end
603
631
 
604
- # TODO: #to_msgpack(string) is deprecated
605
- def to_msgpack(out = '')
606
- [@host, @port, @weight, @available].to_msgpack(out)
607
- end
608
-
609
632
  def generate_salt
610
633
  SecureRandom.hex(16)
611
634
  end