fluentd 0.14.8 → 0.14.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CONTRIBUTING.md +6 -1
- data/ChangeLog +38 -0
- data/Rakefile +21 -0
- data/example/out_exec_filter.conf +42 -0
- data/lib/fluent/agent.rb +2 -2
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cat.rb +5 -2
- data/lib/fluent/compat/output.rb +7 -8
- data/lib/fluent/compat/parser.rb +139 -11
- data/lib/fluent/config/configure_proxy.rb +2 -11
- data/lib/fluent/config/section.rb +7 -0
- data/lib/fluent/configurable.rb +1 -3
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/plugin/base.rb +17 -0
- data/lib/fluent/plugin/filter_parser.rb +108 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +4 -7
- data/lib/fluent/plugin/filter_stdout.rb +1 -1
- data/lib/fluent/plugin/formatter.rb +5 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_exec.rb +48 -93
- data/lib/fluent/plugin/in_forward.rb +25 -105
- data/lib/fluent/plugin/in_http.rb +68 -65
- data/lib/fluent/plugin/in_syslog.rb +29 -51
- data/lib/fluent/plugin/multi_output.rb +1 -3
- data/lib/fluent/plugin/out_exec.rb +58 -71
- data/lib/fluent/plugin/out_exec_filter.rb +199 -279
- data/lib/fluent/plugin/out_file.rb +155 -80
- data/lib/fluent/plugin/out_forward.rb +44 -47
- data/lib/fluent/plugin/out_stdout.rb +6 -21
- data/lib/fluent/plugin/output.rb +23 -17
- data/lib/fluent/plugin/parser.rb +121 -61
- data/lib/fluent/plugin/parser_csv.rb +9 -3
- data/lib/fluent/plugin/parser_json.rb +37 -35
- data/lib/fluent/plugin/parser_ltsv.rb +11 -19
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -42
- data/lib/fluent/plugin/parser_tsv.rb +8 -3
- data/lib/fluent/plugin_helper.rb +8 -1
- data/lib/fluent/plugin_helper/child_process.rb +139 -73
- data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
- data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
- data/lib/fluent/plugin_helper/extract.rb +16 -4
- data/lib/fluent/plugin_helper/formatter.rb +9 -11
- data/lib/fluent/plugin_helper/inject.rb +4 -0
- data/lib/fluent/plugin_helper/parser.rb +3 -3
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/driver/base.rb +51 -37
- data/lib/fluent/test/driver/base_owner.rb +18 -8
- data/lib/fluent/test/driver/multi_output.rb +2 -1
- data/lib/fluent/test/driver/output.rb +29 -6
- data/lib/fluent/test/helpers.rb +3 -1
- data/lib/fluent/test/log.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +13 -0
- data/lib/fluent/time.rb +14 -8
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_binlog_reader.rb +5 -1
- data/test/config/test_configurable.rb +173 -0
- data/test/config/test_configure_proxy.rb +0 -43
- data/test/plugin/test_base.rb +16 -0
- data/test/plugin/test_filter_parser.rb +665 -0
- data/test/plugin/test_filter_record_transformer.rb +11 -3
- data/test/plugin/test_filter_stdout.rb +18 -27
- data/test/plugin/test_in_dummy.rb +1 -1
- data/test/plugin/test_in_exec.rb +206 -94
- data/test/plugin/test_in_forward.rb +310 -327
- data/test/plugin/test_in_http.rb +310 -186
- data/test/plugin/test_out_exec.rb +223 -68
- data/test/plugin/test_out_exec_filter.rb +520 -169
- data/test/plugin/test_out_file.rb +620 -177
- data/test/plugin/test_out_forward.rb +110 -132
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_out_secondary_file.rb +4 -2
- data/test/plugin/test_out_stdout.rb +14 -35
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_csv.rb +1 -2
- data/test/plugin/test_parser_json.rb +3 -4
- data/test/plugin/test_parser_labeled_tsv.rb +1 -2
- data/test/plugin/test_parser_none.rb +1 -2
- data/test/plugin/test_parser_regexp.rb +8 -4
- data/test/plugin/test_parser_tsv.rb +4 -3
- data/test/plugin_helper/test_child_process.rb +184 -0
- data/test/plugin_helper/test_compat_parameters.rb +88 -1
- data/test/plugin_helper/test_extract.rb +0 -1
- data/test/plugin_helper/test_formatter.rb +5 -2
- data/test/plugin_helper/test_parser.rb +6 -5
- data/test/test_output.rb +24 -2
- data/test/test_plugin_classes.rb +20 -0
- data/test/test_root_agent.rb +139 -0
- data/test/test_test_drivers.rb +132 -0
- metadata +12 -4
- data/test/plugin/test_parser_base.rb +0 -32
@@ -16,42 +16,59 @@
|
|
16
16
|
|
17
17
|
require 'fileutils'
|
18
18
|
require 'zlib'
|
19
|
+
require 'time'
|
19
20
|
|
20
|
-
require 'fluent/output'
|
21
|
+
require 'fluent/plugin/output'
|
21
22
|
require 'fluent/config/error'
|
22
|
-
|
23
|
+
# TODO remove ...
|
24
|
+
require 'fluent/plugin/file_util'
|
23
25
|
|
24
|
-
module Fluent
|
25
|
-
class FileOutput <
|
26
|
-
|
26
|
+
module Fluent::Plugin
|
27
|
+
class FileOutput < Output
|
28
|
+
Fluent::Plugin.register_output('file', self)
|
27
29
|
|
28
|
-
|
30
|
+
helpers :formatter, :inject, :compat_parameters
|
29
31
|
|
30
|
-
SUPPORTED_COMPRESS =
|
31
|
-
|
32
|
-
|
32
|
+
SUPPORTED_COMPRESS = [:text, :gz, :gzip]
|
33
|
+
SUPPORTED_COMPRESS_MAP = {
|
34
|
+
text: nil,
|
35
|
+
gz: :gzip,
|
36
|
+
gzip: :gzip,
|
33
37
|
}
|
34
38
|
|
35
39
|
FILE_PERMISSION = 0644
|
36
40
|
DIR_PERMISSION = 0755
|
37
41
|
|
42
|
+
DEFAULT_TIMEKEY = 60 * 60 * 24
|
43
|
+
|
38
44
|
desc "The Path of the file."
|
39
45
|
config_param :path, :string
|
40
|
-
|
41
|
-
|
46
|
+
|
47
|
+
desc "Specify to add file suffix for bare file path or not."
|
48
|
+
config_param :add_path_suffix, :bool, default: true
|
49
|
+
desc "The file suffix added to bare file path."
|
50
|
+
config_param :path_suffix, :string, default: '.log'
|
42
51
|
desc "The flushed chunk is appended to existence file or not."
|
43
52
|
config_param :append, :bool, default: false
|
44
53
|
desc "Compress flushed file."
|
45
|
-
config_param :compress,
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
end
|
50
|
-
c
|
51
|
-
end
|
52
|
-
desc "Create symlink to temporary buffered file when buffer_type is file."
|
54
|
+
config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
|
55
|
+
desc "Execute compression again even when buffer chunk is already compressed."
|
56
|
+
config_param :recompress, :bool, default: false
|
57
|
+
desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
|
53
58
|
config_param :symlink_path, :string, default: nil
|
54
59
|
|
60
|
+
config_section :format do
|
61
|
+
config_set_default :@type, 'out_file'
|
62
|
+
end
|
63
|
+
|
64
|
+
config_section :buffer do
|
65
|
+
config_set_default :@type, 'file'
|
66
|
+
config_set_default :chunk_keys, ['time']
|
67
|
+
config_set_default :timekey, DEFAULT_TIMEKEY
|
68
|
+
end
|
69
|
+
|
70
|
+
attr_accessor :last_written_path # for tests
|
71
|
+
|
55
72
|
module SymlinkBufferMixin
|
56
73
|
def symlink_path=(path)
|
57
74
|
@_symlink_path = path
|
@@ -63,52 +80,65 @@ module Fluent
|
|
63
80
|
# timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
|
64
81
|
# have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
|
65
82
|
# These chunks will be enqueued immediately, and will be flushed soon.
|
66
|
-
|
67
|
-
if chunk.metadata ==
|
83
|
+
latest_metadata = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
|
84
|
+
if chunk.metadata == latest_metadata
|
68
85
|
FileUtils.ln_sf(chunk.path, @_symlink_path)
|
69
86
|
end
|
70
87
|
chunk
|
71
88
|
end
|
72
89
|
end
|
73
90
|
|
74
|
-
def initialize
|
75
|
-
require 'zlib'
|
76
|
-
require 'time'
|
77
|
-
require 'fluent/plugin/file_util'
|
78
|
-
super
|
79
|
-
end
|
80
|
-
|
81
91
|
def configure(conf)
|
82
|
-
|
83
|
-
|
92
|
+
compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: "time")
|
93
|
+
|
94
|
+
configured_time_slice_format = conf['time_slice_format']
|
95
|
+
|
96
|
+
# v0.14 file buffer handles path as directory if '*' is missing
|
97
|
+
# 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
|
98
|
+
# but raise it in this plugin.
|
99
|
+
if conf.elements(name: 'buffer').empty?
|
100
|
+
conf.add_element('buffer', 'time')
|
84
101
|
end
|
85
|
-
|
86
|
-
|
102
|
+
buffer_conf = conf.elements(name: 'buffer').first
|
103
|
+
unless buffer_conf.has_key?('path')
|
104
|
+
buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
|
87
105
|
end
|
88
106
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
@path_suffix = ".log"
|
96
|
-
conf['buffer_path'] ||= "#{@path}.*"
|
107
|
+
super
|
108
|
+
|
109
|
+
@compress_method = SUPPORTED_COMPRESS_MAP[@compress]
|
110
|
+
|
111
|
+
if @path.include?('*') && !@buffer_config.timekey
|
112
|
+
raise Fluent::ConfigError, "path including '*' must be used with buffer chunk key 'time'"
|
97
113
|
end
|
98
114
|
|
99
|
-
|
115
|
+
path_suffix = @add_path_suffix ? @path_suffix : ''
|
116
|
+
@path_template = generate_path_template(@path, @buffer_config.timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
|
117
|
+
|
118
|
+
placeholder_validate!(:path, @path_template)
|
119
|
+
|
120
|
+
max_tag_index = get_placeholders_tag(@path_template).max || 1
|
121
|
+
max_tag_index = 1 if max_tag_index < 1
|
122
|
+
dummy_tag = (['a'] * max_tag_index).join('.')
|
123
|
+
dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
|
124
|
+
dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
|
125
|
+
|
126
|
+
test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
|
127
|
+
test_path = extract_placeholders(@path_template, test_meta1)
|
100
128
|
unless ::Fluent::FileUtil.writable_p?(test_path)
|
101
|
-
raise ConfigError, "out_file: `#{test_path}` is not writable"
|
129
|
+
raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
|
102
130
|
end
|
103
131
|
|
104
|
-
|
105
|
-
|
106
|
-
@formatter = Plugin.new_formatter(@format)
|
107
|
-
@formatter.configure(conf)
|
132
|
+
@formatter = formatter_create
|
108
133
|
|
109
134
|
if @symlink_path && @buffer.respond_to?(:path)
|
110
|
-
|
111
|
-
|
135
|
+
if Fluent.windows?
|
136
|
+
log.warn "symlink_path is unavailable on Windows platform. disabled."
|
137
|
+
@symlink_path = nil
|
138
|
+
else
|
139
|
+
@buffer.extend SymlinkBufferMixin
|
140
|
+
@buffer.symlink_path = @symlink_path
|
141
|
+
end
|
112
142
|
end
|
113
143
|
|
114
144
|
@dir_perm = system_config.dir_permission || DIR_PERMISSION
|
@@ -116,56 +146,101 @@ module Fluent
|
|
116
146
|
end
|
117
147
|
|
118
148
|
def format(tag, time, record)
|
119
|
-
|
149
|
+
r = inject_values_to_record(tag, time, record)
|
150
|
+
@formatter.format(tag, time, r)
|
120
151
|
end
|
121
152
|
|
122
153
|
def write(chunk)
|
123
|
-
path =
|
154
|
+
path = extract_placeholders(@path_template, chunk.metadata)
|
124
155
|
FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
|
125
156
|
|
126
|
-
|
157
|
+
unless @append
|
158
|
+
path = find_filepath_available(path)
|
159
|
+
end
|
160
|
+
|
161
|
+
case @compress_method
|
127
162
|
when nil
|
128
|
-
File.open(path, "ab", @file_perm)
|
163
|
+
File.open(path, "ab", @file_perm) do |f|
|
129
164
|
chunk.write_to(f)
|
130
|
-
|
131
|
-
when :
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
165
|
+
end
|
166
|
+
when :gzip
|
167
|
+
if @buffer.compress != :gzip || @recompress
|
168
|
+
File.open(path, "ab", @file_perm) do |f|
|
169
|
+
gz = Zlib::GzipWriter.new(f)
|
170
|
+
chunk.write_to(gz, compressed: :text)
|
171
|
+
gz.close
|
172
|
+
end
|
173
|
+
else
|
174
|
+
File.open(path, "ab", @file_perm) do |f|
|
175
|
+
chunk.write_to(f, compressed: :gzip)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
else
|
179
|
+
raise "BUG: unknown compression method #{@compress_method}"
|
137
180
|
end
|
138
181
|
|
139
|
-
|
182
|
+
@last_written_path = path
|
140
183
|
end
|
141
184
|
|
142
|
-
def
|
143
|
-
|
185
|
+
def timekey_to_timeformat(timekey)
|
186
|
+
case timekey
|
187
|
+
when nil then ''
|
188
|
+
when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
|
189
|
+
when 60...3600 then '%Y%m%d%H%M'
|
190
|
+
when 3600...86400 then '%Y%m%d%H'
|
191
|
+
else '%Y%m%d'
|
192
|
+
end
|
144
193
|
end
|
145
194
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
when :gz
|
153
|
-
".gz"
|
195
|
+
def compression_suffix(compress)
|
196
|
+
case compress
|
197
|
+
when :gzip then '.gz'
|
198
|
+
when nil then ''
|
199
|
+
else
|
200
|
+
raise ArgumentError, "unknown compression type #{compress}"
|
154
201
|
end
|
155
202
|
end
|
156
203
|
|
157
|
-
|
158
|
-
|
159
|
-
|
204
|
+
# /path/to/dir/file.* -> /path/to/dir/file.%Y%m%d
|
205
|
+
# /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data
|
206
|
+
# /path/to/dir/file -> /path/to/dir/file.%Y%m%d.log
|
207
|
+
# %Y%m%d -> %Y%m%d_** (non append)
|
208
|
+
# + .gz (gzipped)
|
209
|
+
## TODO: remove time_slice_format when end of support of compat_parameters
|
210
|
+
def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)
|
211
|
+
comp_suffix = compression_suffix(compress)
|
212
|
+
index_placeholder = append ? '' : '_**'
|
213
|
+
if original.index('*')
|
214
|
+
raise "BUG: configuration error must be raised for path including '*' without timekey" unless timekey
|
215
|
+
time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)
|
216
|
+
original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix
|
160
217
|
else
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
218
|
+
if timekey
|
219
|
+
if time_slice_format
|
220
|
+
"#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
221
|
+
else
|
222
|
+
time_placeholders = timekey_to_timeformat(timekey)
|
223
|
+
if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }
|
224
|
+
raise Fluent::ConfigError, "insufficient timestamp placeholders in path" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }
|
225
|
+
"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
226
|
+
else
|
227
|
+
"#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
228
|
+
end
|
229
|
+
end
|
230
|
+
else
|
231
|
+
"#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def find_filepath_available(path_with_placeholder) # for non-append
|
237
|
+
raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
|
238
|
+
i = 0
|
239
|
+
while path = path_with_placeholder.sub('_**', "_#{i}")
|
240
|
+
break unless File.exist?(path)
|
241
|
+
i += 1
|
168
242
|
end
|
243
|
+
path
|
169
244
|
end
|
170
245
|
end
|
171
246
|
end
|
@@ -14,42 +14,23 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'base64'
|
18
|
-
require 'socket'
|
19
|
-
require 'fileutils'
|
20
|
-
|
21
|
-
require 'cool.io'
|
22
|
-
|
23
17
|
require 'fluent/output'
|
24
18
|
require 'fluent/config/error'
|
19
|
+
require 'base64'
|
25
20
|
|
26
|
-
|
27
|
-
class ForwardOutputError < StandardError
|
28
|
-
end
|
29
|
-
|
30
|
-
class ForwardOutputResponseError < ForwardOutputError
|
31
|
-
end
|
32
|
-
|
33
|
-
class ForwardOutputConnectionClosedError < ForwardOutputError
|
34
|
-
end
|
21
|
+
require 'fluent/compat/socket_util'
|
35
22
|
|
36
|
-
|
37
|
-
|
23
|
+
module Fluent::Plugin
|
24
|
+
class ForwardOutput < Output
|
25
|
+
class Error < StandardError; end
|
26
|
+
class ResponseError < Error; end
|
27
|
+
class ConnectionClosedError < Error; end
|
28
|
+
class ACKTimeoutError < Error; end
|
38
29
|
|
39
|
-
|
40
|
-
Plugin.register_output('forward', self)
|
30
|
+
Fluent::Plugin.register_output('forward', self)
|
41
31
|
|
42
32
|
LISTEN_PORT = 24224
|
43
33
|
|
44
|
-
def initialize
|
45
|
-
super
|
46
|
-
require 'fluent/plugin/socket_util'
|
47
|
-
@nodes = [] #=> [Node]
|
48
|
-
@loop = nil
|
49
|
-
@thread = nil
|
50
|
-
@finished = false
|
51
|
-
end
|
52
|
-
|
53
34
|
desc 'The timeout time when sending event logs.'
|
54
35
|
config_param :send_timeout, :time, default: 60
|
55
36
|
desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
|
@@ -114,17 +95,34 @@ module Fluent
|
|
114
95
|
config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
|
115
96
|
config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
|
116
97
|
|
98
|
+
config_section :buffer do
|
99
|
+
config_set_default :chunk_keys, ["tag"]
|
100
|
+
end
|
101
|
+
|
117
102
|
attr_reader :read_interval, :recover_sample_size
|
118
103
|
|
104
|
+
def initialize
|
105
|
+
super
|
106
|
+
|
107
|
+
@nodes = [] #=> [Node]
|
108
|
+
@loop = nil
|
109
|
+
@thread = nil
|
110
|
+
@finished = false
|
111
|
+
end
|
112
|
+
|
119
113
|
def configure(conf)
|
120
114
|
super
|
121
115
|
|
116
|
+
unless @chunk_key_tag
|
117
|
+
raise Fluent::ConfigError, "buffer chunk key must include 'tag' for forward output"
|
118
|
+
end
|
119
|
+
|
122
120
|
@read_interval = @read_interval_msec / 1000.0
|
123
121
|
@recover_sample_size = @recover_wait / @heartbeat_interval
|
124
122
|
|
125
123
|
if @dns_round_robin
|
126
124
|
if @heartbeat_type == :udp
|
127
|
-
raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
|
125
|
+
raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
|
128
126
|
end
|
129
127
|
end
|
130
128
|
|
@@ -140,14 +138,16 @@ module Fluent
|
|
140
138
|
end
|
141
139
|
end
|
142
140
|
|
143
|
-
|
144
|
-
@buffer.compress
|
145
|
-
|
146
|
-
|
141
|
+
unless @as_secondary
|
142
|
+
if @compress == :gzip && @buffer.compress == :text
|
143
|
+
@buffer.compress = :gzip
|
144
|
+
elsif @compress == :text && @buffer.compress == :gzip
|
145
|
+
log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
|
146
|
+
end
|
147
147
|
end
|
148
148
|
|
149
149
|
if @nodes.empty?
|
150
|
-
raise ConfigError, "forward output plugin requires at least one <server> is required"
|
150
|
+
raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
|
151
151
|
end
|
152
152
|
|
153
153
|
raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
|
@@ -166,7 +166,7 @@ module Fluent
|
|
166
166
|
|
167
167
|
if @heartbeat_type == :udp
|
168
168
|
# assuming all hosts use udp
|
169
|
-
@usock = SocketUtil.create_udp_socket(@nodes.first.host)
|
169
|
+
@usock = Fluent::Compat::SocketUtil.create_udp_socket(@nodes.first.host)
|
170
170
|
@usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
|
171
171
|
@hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
|
172
172
|
@loop.attach(@hb)
|
@@ -183,7 +183,8 @@ module Fluent
|
|
183
183
|
@finished = true
|
184
184
|
if @loop
|
185
185
|
@loop.watchers.each {|w| w.detach }
|
186
|
-
@loop.stop
|
186
|
+
# @loop.stop
|
187
|
+
@loop.stop rescue nil
|
187
188
|
end
|
188
189
|
@thread.join if @thread
|
189
190
|
@usock.close if @usock
|
@@ -198,9 +199,10 @@ module Fluent
|
|
198
199
|
log.error_backtrace
|
199
200
|
end
|
200
201
|
|
201
|
-
def
|
202
|
+
def write(chunk)
|
202
203
|
return if chunk.empty?
|
203
204
|
|
205
|
+
tag = chunk.metadata.tag
|
204
206
|
error = nil
|
205
207
|
|
206
208
|
wlen = @weight_array.length
|
@@ -439,10 +441,10 @@ module Fluent
|
|
439
441
|
end
|
440
442
|
|
441
443
|
unless available?
|
442
|
-
raise
|
444
|
+
raise ConnectionClosedError, "failed to establish connection with node #{@name}"
|
443
445
|
end
|
444
446
|
|
445
|
-
option = { 'size' => chunk.
|
447
|
+
option = { 'size' => chunk.size, 'compressed' => @compress }
|
446
448
|
option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
|
447
449
|
|
448
450
|
# out_forward always uses Raw32 type for content.
|
@@ -472,13 +474,13 @@ module Fluent
|
|
472
474
|
if raw_data.empty?
|
473
475
|
@log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
|
474
476
|
disable!
|
475
|
-
raise
|
477
|
+
raise ConnectionClosedError, "node #{@host}:#{@port} closed connection"
|
476
478
|
else
|
477
479
|
@unpacker.feed(raw_data)
|
478
480
|
res = @unpacker.read
|
479
481
|
if res['ack'] != option['chunk']
|
480
482
|
# Some errors may have occured when ack and chunk id is different, so send the chunk again.
|
481
|
-
raise
|
483
|
+
raise ResponseError, "ack in response and chunk id in sent data are different"
|
482
484
|
end
|
483
485
|
end
|
484
486
|
|
@@ -489,7 +491,7 @@ module Fluent
|
|
489
491
|
# (2) the node does support sending response but responses have not arrived for some reasons.
|
490
492
|
@log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
|
491
493
|
disable!
|
492
|
-
raise
|
494
|
+
raise ACKTimeoutError, "node #{host}:#{port} does not return ACK"
|
493
495
|
end
|
494
496
|
end
|
495
497
|
|
@@ -601,11 +603,6 @@ module Fluent
|
|
601
603
|
end
|
602
604
|
end
|
603
605
|
|
604
|
-
# TODO: #to_msgpack(string) is deprecated
|
605
|
-
def to_msgpack(out = '')
|
606
|
-
[@host, @port, @weight, @available].to_msgpack(out)
|
607
|
-
end
|
608
|
-
|
609
606
|
def generate_salt
|
610
607
|
SecureRandom.hex(16)
|
611
608
|
end
|