fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +2 -0
- data/CONTRIBUTING.md +6 -1
- data/ChangeLog +95 -0
- data/Rakefile +21 -0
- data/appveyor.yml +1 -0
- data/code-of-conduct.md +3 -0
- data/example/out_exec_filter.conf +42 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/agent.rb +2 -2
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cat.rb +15 -4
- data/lib/fluent/compat/output.rb +14 -9
- data/lib/fluent/compat/parser.rb +141 -11
- data/lib/fluent/config/configure_proxy.rb +2 -11
- data/lib/fluent/config/section.rb +8 -1
- data/lib/fluent/configurable.rb +1 -3
- data/lib/fluent/env.rb +1 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/plugin/base.rb +17 -0
- data/lib/fluent/plugin/filter_parser.rb +108 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
- data/lib/fluent/plugin/filter_stdout.rb +1 -1
- data/lib/fluent/plugin/formatter.rb +5 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_exec.rb +48 -93
- data/lib/fluent/plugin/in_forward.rb +66 -265
- data/lib/fluent/plugin/in_http.rb +68 -65
- data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
- data/lib/fluent/plugin/in_syslog.rb +42 -58
- data/lib/fluent/plugin/in_tail.rb +29 -14
- data/lib/fluent/plugin/in_tcp.rb +54 -14
- data/lib/fluent/plugin/in_udp.rb +49 -13
- data/lib/fluent/plugin/multi_output.rb +1 -3
- data/lib/fluent/plugin/out_exec.rb +58 -71
- data/lib/fluent/plugin/out_exec_filter.rb +199 -279
- data/lib/fluent/plugin/out_file.rb +172 -81
- data/lib/fluent/plugin/out_forward.rb +229 -206
- data/lib/fluent/plugin/out_stdout.rb +6 -21
- data/lib/fluent/plugin/output.rb +90 -59
- data/lib/fluent/plugin/parser.rb +121 -61
- data/lib/fluent/plugin/parser_csv.rb +9 -3
- data/lib/fluent/plugin/parser_json.rb +37 -35
- data/lib/fluent/plugin/parser_ltsv.rb +11 -19
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -42
- data/lib/fluent/plugin/parser_tsv.rb +8 -3
- data/lib/fluent/plugin_helper.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +139 -73
- data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
- data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
- data/lib/fluent/plugin_helper/event_loop.rb +24 -6
- data/lib/fluent/plugin_helper/extract.rb +16 -4
- data/lib/fluent/plugin_helper/formatter.rb +9 -11
- data/lib/fluent/plugin_helper/inject.rb +16 -1
- data/lib/fluent/plugin_helper/parser.rb +3 -3
- data/lib/fluent/plugin_helper/server.rb +494 -0
- data/lib/fluent/plugin_helper/socket.rb +101 -0
- data/lib/fluent/plugin_helper/socket_option.rb +84 -0
- data/lib/fluent/plugin_helper/timer.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/driver/base.rb +95 -49
- data/lib/fluent/test/driver/base_owner.rb +18 -8
- data/lib/fluent/test/driver/multi_output.rb +2 -1
- data/lib/fluent/test/driver/output.rb +29 -6
- data/lib/fluent/test/helpers.rb +3 -1
- data/lib/fluent/test/log.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +13 -0
- data/lib/fluent/time.rb +14 -8
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +1 -1
- data/test/command/test_binlog_reader.rb +5 -1
- data/test/compat/test_parser.rb +10 -0
- data/test/config/test_configurable.rb +193 -0
- data/test/config/test_configure_proxy.rb +0 -43
- data/test/helper.rb +36 -1
- data/test/plugin/test_base.rb +16 -0
- data/test/plugin/test_filter_parser.rb +665 -0
- data/test/plugin/test_filter_record_transformer.rb +36 -100
- data/test/plugin/test_filter_stdout.rb +18 -27
- data/test/plugin/test_in_dummy.rb +1 -1
- data/test/plugin/test_in_exec.rb +206 -94
- data/test/plugin/test_in_forward.rb +268 -347
- data/test/plugin/test_in_http.rb +310 -186
- data/test/plugin/test_in_monitor_agent.rb +65 -35
- data/test/plugin/test_in_syslog.rb +39 -3
- data/test/plugin/test_in_tcp.rb +78 -62
- data/test/plugin/test_in_udp.rb +101 -80
- data/test/plugin/test_out_exec.rb +223 -68
- data/test/plugin/test_out_exec_filter.rb +520 -169
- data/test/plugin/test_out_file.rb +637 -177
- data/test/plugin/test_out_forward.rb +242 -234
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_out_secondary_file.rb +4 -2
- data/test/plugin/test_out_stdout.rb +14 -35
- data/test/plugin/test_output_as_buffered.rb +60 -2
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_csv.rb +1 -2
- data/test/plugin/test_parser_json.rb +3 -4
- data/test/plugin/test_parser_labeled_tsv.rb +1 -2
- data/test/plugin/test_parser_none.rb +1 -2
- data/test/plugin/test_parser_regexp.rb +8 -4
- data/test/plugin/test_parser_tsv.rb +4 -3
- data/test/plugin_helper/test_child_process.rb +184 -0
- data/test/plugin_helper/test_compat_parameters.rb +88 -1
- data/test/plugin_helper/test_extract.rb +0 -1
- data/test/plugin_helper/test_formatter.rb +5 -2
- data/test/plugin_helper/test_inject.rb +21 -0
- data/test/plugin_helper/test_parser.rb +6 -5
- data/test/plugin_helper/test_server.rb +905 -0
- data/test/test_event_time.rb +3 -1
- data/test/test_output.rb +53 -2
- data/test/test_plugin_classes.rb +20 -0
- data/test/test_root_agent.rb +139 -0
- data/test/test_test_drivers.rb +135 -0
- metadata +28 -8
- data/test/plugin/test_parser_base.rb +0 -32
@@ -16,42 +16,59 @@
|
|
16
16
|
|
17
17
|
require 'fileutils'
|
18
18
|
require 'zlib'
|
19
|
+
require 'time'
|
19
20
|
|
20
|
-
require 'fluent/output'
|
21
|
+
require 'fluent/plugin/output'
|
21
22
|
require 'fluent/config/error'
|
22
|
-
|
23
|
+
# TODO remove ...
|
24
|
+
require 'fluent/plugin/file_util'
|
23
25
|
|
24
|
-
module Fluent
|
25
|
-
class FileOutput <
|
26
|
-
|
26
|
+
module Fluent::Plugin
|
27
|
+
class FileOutput < Output
|
28
|
+
Fluent::Plugin.register_output('file', self)
|
27
29
|
|
28
|
-
|
30
|
+
helpers :formatter, :inject, :compat_parameters
|
29
31
|
|
30
|
-
SUPPORTED_COMPRESS =
|
31
|
-
|
32
|
-
|
32
|
+
SUPPORTED_COMPRESS = [:text, :gz, :gzip]
|
33
|
+
SUPPORTED_COMPRESS_MAP = {
|
34
|
+
text: nil,
|
35
|
+
gz: :gzip,
|
36
|
+
gzip: :gzip,
|
33
37
|
}
|
34
38
|
|
35
39
|
FILE_PERMISSION = 0644
|
36
40
|
DIR_PERMISSION = 0755
|
37
41
|
|
42
|
+
DEFAULT_TIMEKEY = 60 * 60 * 24
|
43
|
+
|
38
44
|
desc "The Path of the file."
|
39
45
|
config_param :path, :string
|
40
|
-
|
41
|
-
|
46
|
+
|
47
|
+
desc "Specify to add file suffix for bare file path or not."
|
48
|
+
config_param :add_path_suffix, :bool, default: true
|
49
|
+
desc "The file suffix added to bare file path."
|
50
|
+
config_param :path_suffix, :string, default: '.log'
|
42
51
|
desc "The flushed chunk is appended to existence file or not."
|
43
52
|
config_param :append, :bool, default: false
|
44
53
|
desc "Compress flushed file."
|
45
|
-
config_param :compress,
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
end
|
50
|
-
c
|
51
|
-
end
|
52
|
-
desc "Create symlink to temporary buffered file when buffer_type is file."
|
54
|
+
config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
|
55
|
+
desc "Execute compression again even when buffer chunk is already compressed."
|
56
|
+
config_param :recompress, :bool, default: false
|
57
|
+
desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
|
53
58
|
config_param :symlink_path, :string, default: nil
|
54
59
|
|
60
|
+
config_section :format do
|
61
|
+
config_set_default :@type, 'out_file'
|
62
|
+
end
|
63
|
+
|
64
|
+
config_section :buffer do
|
65
|
+
config_set_default :@type, 'file'
|
66
|
+
config_set_default :chunk_keys, ['time']
|
67
|
+
config_set_default :timekey, DEFAULT_TIMEKEY
|
68
|
+
end
|
69
|
+
|
70
|
+
attr_accessor :last_written_path # for tests
|
71
|
+
|
55
72
|
module SymlinkBufferMixin
|
56
73
|
def symlink_path=(path)
|
57
74
|
@_symlink_path = path
|
@@ -63,52 +80,81 @@ module Fluent
|
|
63
80
|
# timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
|
64
81
|
# have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
|
65
82
|
# These chunks will be enqueued immediately, and will be flushed soon.
|
66
|
-
|
67
|
-
if chunk.metadata ==
|
83
|
+
latest_metadata = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
|
84
|
+
if chunk.metadata == latest_metadata
|
68
85
|
FileUtils.ln_sf(chunk.path, @_symlink_path)
|
69
86
|
end
|
70
87
|
chunk
|
71
88
|
end
|
72
89
|
end
|
73
90
|
|
74
|
-
def initialize
|
75
|
-
require 'zlib'
|
76
|
-
require 'time'
|
77
|
-
require 'fluent/plugin/file_util'
|
78
|
-
super
|
79
|
-
end
|
80
|
-
|
81
91
|
def configure(conf)
|
82
|
-
|
83
|
-
|
92
|
+
compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: "time")
|
93
|
+
|
94
|
+
configured_time_slice_format = conf['time_slice_format']
|
95
|
+
|
96
|
+
# v0.14 file buffer handles path as directory if '*' is missing
|
97
|
+
# 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
|
98
|
+
# but raise it in this plugin.
|
99
|
+
if conf.elements(name: 'buffer').empty?
|
100
|
+
conf.add_element('buffer', 'time')
|
84
101
|
end
|
85
|
-
|
86
|
-
|
102
|
+
buffer_conf = conf.elements(name: 'buffer').first
|
103
|
+
unless buffer_conf.has_key?('path')
|
104
|
+
buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
|
87
105
|
end
|
88
106
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
conf['buffer_path'] ||= "#{@path}"
|
93
|
-
else
|
94
|
-
@path_prefix = @path+"."
|
95
|
-
@path_suffix = ".log"
|
96
|
-
conf['buffer_path'] ||= "#{@path}.*"
|
97
|
-
end
|
107
|
+
super
|
108
|
+
|
109
|
+
@compress_method = SUPPORTED_COMPRESS_MAP[@compress]
|
98
110
|
|
99
|
-
|
100
|
-
|
101
|
-
raise ConfigError, "out_file: `#{test_path}` is not writable"
|
111
|
+
if @path.include?('*') && !@buffer_config.timekey
|
112
|
+
raise Fluent::ConfigError, "path including '*' must be used with buffer chunk key 'time'"
|
102
113
|
end
|
103
114
|
|
104
|
-
|
115
|
+
path_suffix = @add_path_suffix ? @path_suffix : ''
|
116
|
+
path_timekey = if @chunk_key_time
|
117
|
+
@as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey
|
118
|
+
else
|
119
|
+
nil
|
120
|
+
end
|
121
|
+
@path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
|
105
122
|
|
106
|
-
|
107
|
-
|
123
|
+
if @as_secondary
|
124
|
+
# When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.
|
125
|
+
# Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.
|
126
|
+
placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|
|
127
|
+
v.validate!
|
128
|
+
end
|
129
|
+
else
|
130
|
+
placeholder_validate!(:path, @path_template)
|
131
|
+
|
132
|
+
max_tag_index = get_placeholders_tag(@path_template).max || 1
|
133
|
+
max_tag_index = 1 if max_tag_index < 1
|
134
|
+
dummy_tag = (['a'] * max_tag_index).join('.')
|
135
|
+
dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
|
136
|
+
dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
|
137
|
+
|
138
|
+
test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
|
139
|
+
test_path = extract_placeholders(@path_template, test_meta1)
|
140
|
+
unless ::Fluent::FileUtil.writable_p?(test_path)
|
141
|
+
raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
@formatter = formatter_create
|
108
146
|
|
109
147
|
if @symlink_path && @buffer.respond_to?(:path)
|
110
|
-
@
|
111
|
-
|
148
|
+
if @as_secondary
|
149
|
+
raise Fluent::ConfigError, "symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin"
|
150
|
+
end
|
151
|
+
if Fluent.windows?
|
152
|
+
log.warn "symlink_path is unavailable on Windows platform. disabled."
|
153
|
+
@symlink_path = nil
|
154
|
+
else
|
155
|
+
@buffer.extend SymlinkBufferMixin
|
156
|
+
@buffer.symlink_path = @symlink_path
|
157
|
+
end
|
112
158
|
end
|
113
159
|
|
114
160
|
@dir_perm = system_config.dir_permission || DIR_PERMISSION
|
@@ -116,56 +162,101 @@ module Fluent
|
|
116
162
|
end
|
117
163
|
|
118
164
|
def format(tag, time, record)
|
119
|
-
|
165
|
+
r = inject_values_to_record(tag, time, record)
|
166
|
+
@formatter.format(tag, time, r)
|
120
167
|
end
|
121
168
|
|
122
169
|
def write(chunk)
|
123
|
-
path =
|
170
|
+
path = extract_placeholders(@path_template, chunk.metadata)
|
124
171
|
FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
|
125
172
|
|
126
|
-
|
173
|
+
unless @append
|
174
|
+
path = find_filepath_available(path)
|
175
|
+
end
|
176
|
+
|
177
|
+
case @compress_method
|
127
178
|
when nil
|
128
|
-
File.open(path, "ab", @file_perm)
|
179
|
+
File.open(path, "ab", @file_perm) do |f|
|
129
180
|
chunk.write_to(f)
|
130
|
-
|
131
|
-
when :
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
181
|
+
end
|
182
|
+
when :gzip
|
183
|
+
if @buffer.compress != :gzip || @recompress
|
184
|
+
File.open(path, "ab", @file_perm) do |f|
|
185
|
+
gz = Zlib::GzipWriter.new(f)
|
186
|
+
chunk.write_to(gz, compressed: :text)
|
187
|
+
gz.close
|
188
|
+
end
|
189
|
+
else
|
190
|
+
File.open(path, "ab", @file_perm) do |f|
|
191
|
+
chunk.write_to(f, compressed: :gzip)
|
192
|
+
end
|
193
|
+
end
|
194
|
+
else
|
195
|
+
raise "BUG: unknown compression method #{@compress_method}"
|
137
196
|
end
|
138
197
|
|
139
|
-
|
198
|
+
@last_written_path = path
|
140
199
|
end
|
141
200
|
|
142
|
-
def
|
143
|
-
|
201
|
+
def timekey_to_timeformat(timekey)
|
202
|
+
case timekey
|
203
|
+
when nil then ''
|
204
|
+
when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
|
205
|
+
when 60...3600 then '%Y%m%d%H%M'
|
206
|
+
when 3600...86400 then '%Y%m%d%H'
|
207
|
+
else '%Y%m%d'
|
208
|
+
end
|
144
209
|
end
|
145
210
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
when :gz
|
153
|
-
".gz"
|
211
|
+
def compression_suffix(compress)
|
212
|
+
case compress
|
213
|
+
when :gzip then '.gz'
|
214
|
+
when nil then ''
|
215
|
+
else
|
216
|
+
raise ArgumentError, "unknown compression type #{compress}"
|
154
217
|
end
|
155
218
|
end
|
156
219
|
|
157
|
-
|
158
|
-
|
159
|
-
|
220
|
+
# /path/to/dir/file.* -> /path/to/dir/file.%Y%m%d
|
221
|
+
# /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data
|
222
|
+
# /path/to/dir/file -> /path/to/dir/file.%Y%m%d.log
|
223
|
+
# %Y%m%d -> %Y%m%d_** (non append)
|
224
|
+
# + .gz (gzipped)
|
225
|
+
## TODO: remove time_slice_format when end of support of compat_parameters
|
226
|
+
def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)
|
227
|
+
comp_suffix = compression_suffix(compress)
|
228
|
+
index_placeholder = append ? '' : '_**'
|
229
|
+
if original.index('*')
|
230
|
+
raise "BUG: configuration error must be raised for path including '*' without timekey" unless timekey
|
231
|
+
time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)
|
232
|
+
original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix
|
160
233
|
else
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
234
|
+
if timekey
|
235
|
+
if time_slice_format
|
236
|
+
"#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
237
|
+
else
|
238
|
+
time_placeholders = timekey_to_timeformat(timekey)
|
239
|
+
if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }
|
240
|
+
raise Fluent::ConfigError, "insufficient timestamp placeholders in path" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }
|
241
|
+
"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
242
|
+
else
|
243
|
+
"#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
244
|
+
end
|
245
|
+
end
|
246
|
+
else
|
247
|
+
"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def find_filepath_available(path_with_placeholder) # for non-append
|
253
|
+
raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
|
254
|
+
i = 0
|
255
|
+
while path = path_with_placeholder.sub('_**', "_#{i}")
|
256
|
+
break unless File.exist?(path)
|
257
|
+
i += 1
|
168
258
|
end
|
259
|
+
path
|
169
260
|
end
|
170
261
|
end
|
171
262
|
end
|
@@ -14,44 +14,30 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'base64'
|
18
|
-
require 'socket'
|
19
|
-
require 'fileutils'
|
20
|
-
|
21
|
-
require 'cool.io'
|
22
|
-
|
23
17
|
require 'fluent/output'
|
24
18
|
require 'fluent/config/error'
|
19
|
+
require 'base64'
|
25
20
|
|
26
|
-
|
27
|
-
class ForwardOutputError < StandardError
|
28
|
-
end
|
29
|
-
|
30
|
-
class ForwardOutputResponseError < ForwardOutputError
|
31
|
-
end
|
21
|
+
require 'fluent/compat/socket_util'
|
32
22
|
|
33
|
-
|
34
|
-
|
23
|
+
module Fluent::Plugin
|
24
|
+
class ForwardOutput < Output
|
25
|
+
class Error < StandardError; end
|
26
|
+
class NoNodesAvailable < Error; end
|
27
|
+
class ConnectionClosedError < Error; end
|
35
28
|
|
36
|
-
|
37
|
-
end
|
29
|
+
Fluent::Plugin.register_output('forward', self)
|
38
30
|
|
39
|
-
|
40
|
-
Plugin.register_output('forward', self)
|
31
|
+
helpers :socket, :server, :timer, :thread, :compat_parameters
|
41
32
|
|
42
33
|
LISTEN_PORT = 24224
|
43
34
|
|
44
|
-
|
45
|
-
super
|
46
|
-
require 'fluent/plugin/socket_util'
|
47
|
-
@nodes = [] #=> [Node]
|
48
|
-
@loop = nil
|
49
|
-
@thread = nil
|
50
|
-
@finished = false
|
51
|
-
end
|
35
|
+
PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
|
52
36
|
|
53
37
|
desc 'The timeout time when sending event logs.'
|
54
38
|
config_param :send_timeout, :time, default: 60
|
39
|
+
# TODO: add linger_timeout, recv_timeout
|
40
|
+
|
55
41
|
desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
|
56
42
|
config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
|
57
43
|
desc 'The interval of the heartbeat packer.'
|
@@ -60,8 +46,6 @@ module Fluent
|
|
60
46
|
config_param :recover_wait, :time, default: 10
|
61
47
|
desc 'The hard timeout used to detect server failure.'
|
62
48
|
config_param :hard_timeout, :time, default: 60
|
63
|
-
desc 'Set TTL to expire DNS cache in seconds.'
|
64
|
-
config_param :expire_dns_cache, :time, default: nil # 0 means disable cache
|
65
49
|
desc 'The threshold parameter used to detect server faults.'
|
66
50
|
config_param :phi_threshold, :integer, default: 16
|
67
51
|
desc 'Use the "Phi accrual failure detector" to detect server failure.'
|
@@ -69,14 +53,20 @@ module Fluent
|
|
69
53
|
|
70
54
|
desc 'Change the protocol to at-least-once.'
|
71
55
|
config_param :require_ack_response, :bool, default: false # require in_forward to respond with ack
|
56
|
+
|
57
|
+
## The reason of default value of :ack_response_timeout:
|
58
|
+
# Linux default tcp_syn_retries is 5 (in many environment)
|
59
|
+
# 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
|
72
60
|
desc 'This option is used when require_ack_response is true.'
|
73
61
|
config_param :ack_response_timeout, :time, default: 190
|
74
|
-
|
75
|
-
config_param :read_length, :size, default: 512 # 512bytes
|
62
|
+
|
76
63
|
desc 'The interval while reading data from server'
|
77
64
|
config_param :read_interval_msec, :integer, default: 50 # 50ms
|
78
|
-
|
79
|
-
|
65
|
+
desc 'Reading data size from server'
|
66
|
+
config_param :read_length, :size, default: 512 # 512bytes
|
67
|
+
|
68
|
+
desc 'Set TTL to expire DNS cache in seconds.'
|
69
|
+
config_param :expire_dns_cache, :time, default: nil # 0 means disable cache
|
80
70
|
desc 'Enable client-side DNS round robin.'
|
81
71
|
config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
|
82
72
|
|
@@ -114,17 +104,39 @@ module Fluent
|
|
114
104
|
config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
|
115
105
|
config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
|
116
106
|
|
107
|
+
config_section :buffer do
|
108
|
+
config_set_default :chunk_keys, ["tag"]
|
109
|
+
end
|
110
|
+
|
117
111
|
attr_reader :read_interval, :recover_sample_size
|
118
112
|
|
113
|
+
def initialize
|
114
|
+
super
|
115
|
+
|
116
|
+
@nodes = [] #=> [Node]
|
117
|
+
@loop = nil
|
118
|
+
@thread = nil
|
119
|
+
|
120
|
+
@usock = nil
|
121
|
+
@sock_ack_waiting = nil
|
122
|
+
@sock_ack_waiting_mutex = nil
|
123
|
+
end
|
124
|
+
|
119
125
|
def configure(conf)
|
126
|
+
compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
|
127
|
+
|
120
128
|
super
|
121
129
|
|
130
|
+
unless @chunk_key_tag
|
131
|
+
raise Fluent::ConfigError, "buffer chunk key must include 'tag' for forward output"
|
132
|
+
end
|
133
|
+
|
122
134
|
@read_interval = @read_interval_msec / 1000.0
|
123
135
|
@recover_sample_size = @recover_wait / @heartbeat_interval
|
124
136
|
|
125
137
|
if @dns_round_robin
|
126
138
|
if @heartbeat_type == :udp
|
127
|
-
raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
|
139
|
+
raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
|
128
140
|
end
|
129
141
|
end
|
130
142
|
|
@@ -140,90 +152,119 @@ module Fluent
|
|
140
152
|
end
|
141
153
|
end
|
142
154
|
|
143
|
-
|
144
|
-
@buffer.compress
|
145
|
-
|
146
|
-
|
155
|
+
unless @as_secondary
|
156
|
+
if @compress == :gzip && @buffer.compress == :text
|
157
|
+
@buffer.compress = :gzip
|
158
|
+
elsif @compress == :text && @buffer.compress == :gzip
|
159
|
+
log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
|
160
|
+
end
|
147
161
|
end
|
148
162
|
|
149
163
|
if @nodes.empty?
|
150
|
-
raise ConfigError, "forward output plugin requires at least one <server> is required"
|
164
|
+
raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
|
151
165
|
end
|
152
166
|
|
153
167
|
raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
|
154
168
|
end
|
155
169
|
|
170
|
+
def prefer_delayed_commit
|
171
|
+
@require_ack_response
|
172
|
+
end
|
173
|
+
|
156
174
|
def start
|
157
175
|
super
|
158
176
|
|
177
|
+
# Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
|
178
|
+
# But it should be overwritten by ack_response_timeout to rollback chunks after timeout
|
179
|
+
if @ack_response_timeout && @delayed_commit_timeout != @ack_response_timeout
|
180
|
+
log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
|
181
|
+
@delayed_commit_timeout = @ack_response_timeout
|
182
|
+
end
|
183
|
+
|
159
184
|
@rand_seed = Random.new.seed
|
160
185
|
rebuild_weight_array
|
161
186
|
@rr = 0
|
162
|
-
@usock = nil
|
163
187
|
|
164
188
|
unless @heartbeat_type == :none
|
165
|
-
@loop = Coolio::Loop.new
|
166
|
-
|
167
189
|
if @heartbeat_type == :udp
|
168
|
-
|
169
|
-
@usock
|
170
|
-
|
171
|
-
|
172
|
-
|
190
|
+
@usock = socket_create_udp(@nodes.first.host, @nodes.first.port, nonblock: true)
|
191
|
+
server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length) do |data, sock|
|
192
|
+
sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
|
193
|
+
on_heartbeat(sockaddr, data)
|
194
|
+
end
|
173
195
|
end
|
196
|
+
timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_timer))
|
197
|
+
end
|
174
198
|
|
175
|
-
|
176
|
-
@
|
177
|
-
|
178
|
-
|
199
|
+
if @require_ack_response
|
200
|
+
@sock_ack_waiting_mutex = Mutex.new
|
201
|
+
@sock_ack_waiting = []
|
202
|
+
thread_create(:out_forward_receiving_ack, &method(:ack_reader))
|
179
203
|
end
|
180
204
|
end
|
181
205
|
|
182
|
-
def
|
183
|
-
@finished = true
|
184
|
-
if @loop
|
185
|
-
@loop.watchers.each {|w| w.detach }
|
186
|
-
@loop.stop
|
187
|
-
end
|
188
|
-
@thread.join if @thread
|
206
|
+
def close
|
189
207
|
@usock.close if @usock
|
190
|
-
|
191
208
|
super
|
192
209
|
end
|
193
210
|
|
194
|
-
def
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
log.error_backtrace
|
211
|
+
def write(chunk)
|
212
|
+
return if chunk.empty?
|
213
|
+
tag = chunk.metadata.tag
|
214
|
+
select_a_healthy_node{|node| node.send_data(tag, chunk) }
|
199
215
|
end
|
200
216
|
|
201
|
-
|
202
|
-
|
217
|
+
ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :node, :time, :timeout) do
|
218
|
+
def expired?(now)
|
219
|
+
time + timeout < now
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
def try_write(chunk)
|
224
|
+
if chunk.empty?
|
225
|
+
commit_write(chunk.unique_id)
|
226
|
+
return
|
227
|
+
end
|
228
|
+
tag = chunk.metadata.tag
|
229
|
+
sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
|
230
|
+
chunk_id = Base64.encode64(chunk.unique_id)
|
231
|
+
current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
|
232
|
+
info = ACKWaitingSockInfo.new(sock, chunk_id, node, current_time, @ack_response_timeout)
|
233
|
+
@sock_ack_waiting_mutex.synchronize do
|
234
|
+
@sock_ack_waiting << info
|
235
|
+
end
|
236
|
+
end
|
203
237
|
|
238
|
+
def select_a_healthy_node
|
204
239
|
error = nil
|
205
240
|
|
206
241
|
wlen = @weight_array.length
|
207
242
|
wlen.times do
|
208
243
|
@rr = (@rr + 1) % wlen
|
209
244
|
node = @weight_array[@rr]
|
245
|
+
next unless node.available?
|
210
246
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
error = $! # use the latest error
|
218
|
-
end
|
247
|
+
begin
|
248
|
+
ret = yield node
|
249
|
+
return ret, node
|
250
|
+
rescue
|
251
|
+
# for load balancing during detecting crashed servers
|
252
|
+
error = $! # use the latest error
|
219
253
|
end
|
220
254
|
end
|
221
255
|
|
222
|
-
if error
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
256
|
+
raise error if error
|
257
|
+
raise NoNodesAvailable, "no nodes are available"
|
258
|
+
end
|
259
|
+
|
260
|
+
def create_transfer_socket(host, port, &block)
|
261
|
+
socket_create_tcp(
|
262
|
+
host, port,
|
263
|
+
linger_timeout: @send_timeout,
|
264
|
+
send_timeout: @send_timeout,
|
265
|
+
recv_timeout: @ack_response_timeout,
|
266
|
+
&block
|
267
|
+
)
|
227
268
|
end
|
228
269
|
|
229
270
|
# MessagePack FixArray length is 3
|
@@ -276,21 +317,7 @@ module Fluent
|
|
276
317
|
@weight_array = weight_array
|
277
318
|
end
|
278
319
|
|
279
|
-
class HeartbeatRequestTimer < Coolio::TimerWatcher
|
280
|
-
def initialize(interval, callback)
|
281
|
-
super(interval, true)
|
282
|
-
@callback = callback
|
283
|
-
end
|
284
|
-
|
285
|
-
def on_timer
|
286
|
-
@callback.call
|
287
|
-
rescue
|
288
|
-
# TODO log?
|
289
|
-
end
|
290
|
-
end
|
291
|
-
|
292
320
|
def on_timer
|
293
|
-
return if @finished
|
294
321
|
@nodes.each {|n|
|
295
322
|
if n.tick
|
296
323
|
rebuild_weight_array
|
@@ -305,33 +332,86 @@ module Fluent
|
|
305
332
|
}
|
306
333
|
end
|
307
334
|
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
335
|
+
def on_heartbeat(sockaddr, msg)
|
336
|
+
if node = @nodes.find {|n| n.sockaddr == sockaddr }
|
337
|
+
# log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
|
338
|
+
if node.heartbeat
|
339
|
+
rebuild_weight_array
|
340
|
+
end
|
313
341
|
end
|
342
|
+
end
|
314
343
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
344
|
+
# return chunk id when succeeded for tests
|
345
|
+
def read_ack_from_sock(sock, unpacker)
|
346
|
+
begin
|
347
|
+
raw_data = sock.recv(@read_length)
|
348
|
+
rescue Errno::ECONNRESET
|
349
|
+
raw_data = ""
|
350
|
+
end
|
351
|
+
info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
|
352
|
+
|
353
|
+
# When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
|
354
|
+
# If this happens we assume the data wasn't delivered and retry it.
|
355
|
+
if raw_data.empty?
|
356
|
+
log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
|
357
|
+
info.node.disable!
|
358
|
+
return nil
|
359
|
+
else
|
360
|
+
unpacker.feed(raw_data)
|
361
|
+
res = unpacker.read
|
362
|
+
if res['ack'] != info.chunk_id
|
363
|
+
# Some errors may have occured when ack and chunk id is different, so send the chunk again.
|
364
|
+
log.warn "ack in response and chunk id in sent data are different", chunk_id: info.chunk_id, ack: res['ack']
|
365
|
+
rollback_write(info.chunk_id)
|
366
|
+
return nil
|
320
367
|
end
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
368
|
+
return info.chunk_id
|
369
|
+
end
|
370
|
+
rescue => e
|
371
|
+
log.error "unexpected error while receiving ack message", error: e
|
372
|
+
log.error_backtrace
|
373
|
+
ensure
|
374
|
+
@sock_ack_waiting_mutex.synchronize do
|
375
|
+
@sock_ack_waiting.delete(info)
|
327
376
|
end
|
328
377
|
end
|
329
378
|
|
330
|
-
def
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
379
|
+
def ack_reader
|
380
|
+
select_interval = if @delayed_commit_timeout > 3
|
381
|
+
2
|
382
|
+
else
|
383
|
+
@delayed_commit_timeout / 2.0
|
384
|
+
end
|
385
|
+
|
386
|
+
unpacker = Fluent::Engine.msgpack_unpacker
|
387
|
+
|
388
|
+
while thread_current_running?
|
389
|
+
now = Process.clock_gettime(PROCESS_CLOCK_ID)
|
390
|
+
sockets = []
|
391
|
+
@sock_ack_waiting_mutex.synchronize do
|
392
|
+
new_list = []
|
393
|
+
@sock_ack_waiting.each do |info|
|
394
|
+
if info.expired?(now)
|
395
|
+
# There are 2 types of cases when no response has been received from socket:
|
396
|
+
# (1) the node does not support sending responses
|
397
|
+
# (2) the node does support sending response but responses have not arrived for some reasons.
|
398
|
+
log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
|
399
|
+
info.node.disable!
|
400
|
+
info.sock.close rescue nil
|
401
|
+
rollback_write(info.chunk_id)
|
402
|
+
else
|
403
|
+
sockets << info.sock
|
404
|
+
new_list << info
|
405
|
+
end
|
406
|
+
end
|
407
|
+
@sock_ack_waiting = new_list
|
408
|
+
end
|
409
|
+
|
410
|
+
readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
|
411
|
+
next unless readable_sockets
|
412
|
+
|
413
|
+
readable_sockets.each do |sock|
|
414
|
+
read_ack_from_sock(sock, unpacker)
|
335
415
|
end
|
336
416
|
end
|
337
417
|
end
|
@@ -384,20 +464,6 @@ module Fluent
|
|
384
464
|
@standby
|
385
465
|
end
|
386
466
|
|
387
|
-
def connect
|
388
|
-
TCPSocket.new(resolved_host, port)
|
389
|
-
end
|
390
|
-
|
391
|
-
def set_socket_options(sock)
|
392
|
-
opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
|
393
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
394
|
-
|
395
|
-
opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
396
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
397
|
-
|
398
|
-
sock
|
399
|
-
end
|
400
|
-
|
401
467
|
def establish_connection(sock)
|
402
468
|
while available? && @state != :established
|
403
469
|
begin
|
@@ -428,98 +494,60 @@ module Fluent
|
|
428
494
|
end
|
429
495
|
end
|
430
496
|
|
431
|
-
def
|
432
|
-
sock = connect
|
497
|
+
def send_data_actual(sock, tag, chunk)
|
433
498
|
@state = @sender.security ? :helo : :established
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
if @state != :established
|
438
|
-
establish_connection(sock)
|
439
|
-
end
|
499
|
+
if @state != :established
|
500
|
+
establish_connection(sock)
|
501
|
+
end
|
440
502
|
|
441
|
-
|
442
|
-
|
443
|
-
|
503
|
+
unless available?
|
504
|
+
raise ConnectionClosedError, "failed to establish connection with node #{@name}"
|
505
|
+
end
|
444
506
|
|
445
|
-
|
446
|
-
|
507
|
+
option = { 'size' => chunk.size, 'compressed' => @compress }
|
508
|
+
option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
|
447
509
|
|
448
|
-
|
449
|
-
|
510
|
+
# out_forward always uses Raw32 type for content.
|
511
|
+
# Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
|
450
512
|
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
if @sender.require_ack_response
|
460
|
-
# Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
|
461
|
-
# To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
|
462
|
-
# and process them asynchronously.
|
463
|
-
if IO.select([sock], nil, nil, @sender.ack_response_timeout)
|
464
|
-
raw_data = begin
|
465
|
-
sock.recv(1024)
|
466
|
-
rescue Errno::ECONNRESET
|
467
|
-
""
|
468
|
-
end
|
469
|
-
|
470
|
-
# When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
|
471
|
-
# If this happens we assume the data wasn't delivered and retry it.
|
472
|
-
if raw_data.empty?
|
473
|
-
@log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
|
474
|
-
disable!
|
475
|
-
raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection"
|
476
|
-
else
|
477
|
-
@unpacker.feed(raw_data)
|
478
|
-
res = @unpacker.read
|
479
|
-
if res['ack'] != option['chunk']
|
480
|
-
# Some errors may have occured when ack and chunk id is different, so send the chunk again.
|
481
|
-
raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
|
482
|
-
end
|
483
|
-
end
|
513
|
+
sock.write @sender.forward_header # beginArray(3)
|
514
|
+
sock.write tag.to_msgpack # 1. writeRaw(tag)
|
515
|
+
chunk.open(compressed: @compress) do |chunk_io|
|
516
|
+
sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
|
517
|
+
IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es)
|
518
|
+
end
|
519
|
+
sock.write option.to_msgpack # 3. writeOption(option)
|
520
|
+
end
|
484
521
|
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
end
|
494
|
-
end
|
522
|
+
def send_data(tag, chunk)
|
523
|
+
sock = @sender.create_transfer_socket(resolved_host, port)
|
524
|
+
begin
|
525
|
+
send_data_actual(sock, tag, chunk)
|
526
|
+
rescue
|
527
|
+
sock.close rescue nil
|
528
|
+
raise
|
529
|
+
end
|
495
530
|
|
496
|
-
|
497
|
-
|
498
|
-
ensure
|
499
|
-
sock.close_write
|
500
|
-
sock.close
|
531
|
+
if @sender.require_ack_response
|
532
|
+
return sock # to read ACK from socket
|
501
533
|
end
|
534
|
+
|
535
|
+
sock.close_write rescue nil
|
536
|
+
sock.close rescue nil
|
537
|
+
heartbeat(false)
|
538
|
+
nil
|
502
539
|
end
|
503
540
|
|
504
541
|
# FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
|
505
542
|
def send_heartbeat
|
506
543
|
case @sender.heartbeat_type
|
507
544
|
when :tcp
|
508
|
-
|
509
|
-
begin
|
510
|
-
opt = [1, @sender.send_timeout.to_i].pack('I!I!') # { int l_onoff; int l_linger; }
|
511
|
-
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
512
|
-
# opt = [@sender.send_timeout.to_i, 0].pack('L!L!') # struct timeval
|
513
|
-
# sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
514
|
-
|
545
|
+
@sender.create_transfer_socket(resolved_host, port) do |sock|
|
515
546
|
## don't send any data to not cause a compatibility problem
|
516
547
|
# sock.write FORWARD_TCP_HEARTBEAT_DATA
|
517
548
|
|
518
549
|
# successful tcp connection establishment is considered as valid heartbeat
|
519
550
|
heartbeat(true)
|
520
|
-
ensure
|
521
|
-
sock.close_write
|
522
|
-
sock.close
|
523
551
|
end
|
524
552
|
when :udp
|
525
553
|
@usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
|
@@ -541,7 +569,7 @@ module Fluent
|
|
541
569
|
@resolved_host ||= resolve_dns!
|
542
570
|
|
543
571
|
else
|
544
|
-
now = Engine.now
|
572
|
+
now = Fluent::Engine.now
|
545
573
|
rh = @resolved_host
|
546
574
|
if !rh || now - @resolved_time >= @sender.expire_dns_cache
|
547
575
|
rh = @resolved_host = resolve_dns!
|
@@ -601,11 +629,6 @@ module Fluent
|
|
601
629
|
end
|
602
630
|
end
|
603
631
|
|
604
|
-
# TODO: #to_msgpack(string) is deprecated
|
605
|
-
def to_msgpack(out = '')
|
606
|
-
[@host, @port, @weight, @available].to_msgpack(out)
|
607
|
-
end
|
608
|
-
|
609
632
|
def generate_salt
|
610
633
|
SecureRandom.hex(16)
|
611
634
|
end
|