fluentd 1.6.3 → 1.7.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +35 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
  4. data/CHANGELOG.md +58 -0
  5. data/README.md +5 -1
  6. data/fluentd.gemspec +1 -1
  7. data/lib/fluent/clock.rb +4 -0
  8. data/lib/fluent/compat/output.rb +3 -3
  9. data/lib/fluent/compat/socket_util.rb +1 -1
  10. data/lib/fluent/config/element.rb +3 -3
  11. data/lib/fluent/config/literal_parser.rb +1 -1
  12. data/lib/fluent/config/section.rb +4 -1
  13. data/lib/fluent/error.rb +4 -0
  14. data/lib/fluent/event.rb +28 -24
  15. data/lib/fluent/event_router.rb +2 -1
  16. data/lib/fluent/log.rb +1 -1
  17. data/lib/fluent/msgpack_factory.rb +8 -0
  18. data/lib/fluent/plugin/bare_output.rb +4 -4
  19. data/lib/fluent/plugin/buf_file_single.rb +211 -0
  20. data/lib/fluent/plugin/buffer.rb +62 -63
  21. data/lib/fluent/plugin/buffer/chunk.rb +21 -3
  22. data/lib/fluent/plugin/buffer/file_chunk.rb +37 -12
  23. data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
  24. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
  25. data/lib/fluent/plugin/compressable.rb +10 -6
  26. data/lib/fluent/plugin/filter_grep.rb +2 -2
  27. data/lib/fluent/plugin/formatter_csv.rb +10 -6
  28. data/lib/fluent/plugin/in_syslog.rb +10 -3
  29. data/lib/fluent/plugin/in_tail.rb +7 -2
  30. data/lib/fluent/plugin/in_tcp.rb +34 -7
  31. data/lib/fluent/plugin/multi_output.rb +4 -4
  32. data/lib/fluent/plugin/out_exec_filter.rb +1 -0
  33. data/lib/fluent/plugin/out_file.rb +13 -3
  34. data/lib/fluent/plugin/out_forward.rb +126 -588
  35. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  36. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  37. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  38. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  39. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
  40. data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
  41. data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
  42. data/lib/fluent/plugin/out_http.rb +231 -0
  43. data/lib/fluent/plugin/output.rb +29 -35
  44. data/lib/fluent/plugin/parser.rb +77 -0
  45. data/lib/fluent/plugin/parser_csv.rb +75 -0
  46. data/lib/fluent/plugin_helper/server.rb +1 -1
  47. data/lib/fluent/plugin_helper/thread.rb +1 -0
  48. data/lib/fluent/root_agent.rb +1 -1
  49. data/lib/fluent/time.rb +4 -2
  50. data/lib/fluent/timezone.rb +21 -7
  51. data/lib/fluent/version.rb +1 -1
  52. data/test/command/test_fluentd.rb +1 -1
  53. data/test/command/test_plugin_generator.rb +18 -2
  54. data/test/config/test_configurable.rb +78 -40
  55. data/test/counter/test_store.rb +1 -1
  56. data/test/helper.rb +1 -0
  57. data/test/helpers/process_extenstion.rb +33 -0
  58. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  59. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  60. data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
  61. data/test/plugin/out_forward/test_load_balancer.rb +60 -0
  62. data/test/plugin/out_forward/test_socket_cache.rb +139 -0
  63. data/test/plugin/test_buf_file.rb +118 -2
  64. data/test/plugin/test_buf_file_single.rb +734 -0
  65. data/test/plugin/test_buffer.rb +4 -48
  66. data/test/plugin/test_buffer_file_chunk.rb +19 -1
  67. data/test/plugin/test_buffer_file_single_chunk.rb +620 -0
  68. data/test/plugin/test_formatter_csv.rb +16 -0
  69. data/test/plugin/test_in_syslog.rb +56 -6
  70. data/test/plugin/test_in_tail.rb +1 -1
  71. data/test/plugin/test_in_tcp.rb +25 -0
  72. data/test/plugin/test_out_forward.rb +75 -201
  73. data/test/plugin/test_out_http.rb +352 -0
  74. data/test/plugin/test_output_as_buffered.rb +27 -24
  75. data/test/plugin/test_parser.rb +40 -0
  76. data/test/plugin/test_parser_csv.rb +83 -0
  77. data/test/plugin_helper/test_record_accessor.rb +1 -1
  78. data/test/test_time_formatter.rb +140 -121
  79. metadata +33 -4
@@ -0,0 +1,111 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+ require 'fluent/plugin/out_forward/error'
19
+
20
+ module Fluent::Plugin
21
+ class ForwardOutput < Output
22
+ class LoadBalancer
23
+ def initialize(log)
24
+ @log = log
25
+ @weight_array = []
26
+ @rand_seed = Random.new.seed
27
+ @rr = 0
28
+ @mutex = Mutex.new
29
+ end
30
+
31
+ def select_healthy_node
32
+ error = nil
33
+
34
+ # Don't care about the change of @weight_array's size while looping since
35
+ # it's only used for determining the number of loops and it is not so important.
36
+ wlen = @weight_array.size
37
+ wlen.times do
38
+ node = @mutex.synchronize do
39
+ r = @rr
40
+ @rr = (@rr + 1) % @weight_array.size
41
+ @weight_array[r]
42
+ end
43
+ next unless node.available?
44
+
45
+ begin
46
+ ret = yield node
47
+ return ret, node
48
+ rescue
49
+ # for load balancing during detecting crashed servers
50
+ error = $! # use the latest error
51
+ end
52
+ end
53
+
54
+ raise error if error
55
+ raise NoNodesAvailable, "no nodes are available"
56
+ end
57
+
58
+ def rebuild_weight_array(nodes)
59
+ standby_nodes, regular_nodes = nodes.partition {|n|
60
+ n.standby?
61
+ }
62
+
63
+ lost_weight = 0
64
+ regular_nodes.each {|n|
65
+ unless n.available?
66
+ lost_weight += n.weight
67
+ end
68
+ }
69
+ @log.debug("rebuilding weight array", lost_weight: lost_weight)
70
+
71
+ if lost_weight > 0
72
+ standby_nodes.each {|n|
73
+ if n.available?
74
+ regular_nodes << n
75
+ @log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
76
+ lost_weight -= n.weight
77
+ break if lost_weight <= 0
78
+ end
79
+ }
80
+ end
81
+
82
+ weight_array = []
83
+ if regular_nodes.empty?
84
+ @log.warn('No nodes are available')
85
+ @mutex.synchronize do
86
+ @weight_array = weight_array
87
+ end
88
+ return @weight_array
89
+ end
90
+
91
+ gcd = regular_nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
92
+ regular_nodes.each {|n|
93
+ (n.weight / gcd).times {
94
+ weight_array << n
95
+ }
96
+ }
97
+
98
+ # for load balancing during detecting crashed servers
99
+ coe = (regular_nodes.size * 6) / weight_array.size
100
+ weight_array *= coe if coe > 1
101
+
102
+ r = Random.new(@rand_seed)
103
+ weight_array.sort_by! { r.rand }
104
+
105
+ @mutex.synchronize do
106
+ @weight_array = weight_array
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,138 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class ForwardOutput < Output
21
+ class SocketCache
22
+ TimedSocket = Struct.new(:timeout, :key, :sock)
23
+
24
+ def initialize(timeout, log)
25
+ @log = log
26
+ @timeout = timeout
27
+ @available_sockets = Hash.new { |obj, k| obj[k] = [] }
28
+ @inflight_sockets = {}
29
+ @inactive_sockets = []
30
+ @mutex = Mutex.new
31
+ end
32
+
33
+ def checkout_or(key)
34
+ @mutex.synchronize do
35
+ tsock = pick_socket(key)
36
+
37
+ if tsock
38
+ tsock.sock
39
+ else
40
+ sock = yield
41
+ new_tsock = TimedSocket.new(timeout, key, sock)
42
+ @log.debug("connect new socket #{new_tsock}")
43
+
44
+ @inflight_sockets[sock] = new_tsock
45
+ new_tsock.sock
46
+ end
47
+ end
48
+ end
49
+
50
+ def checkin(sock)
51
+ @mutex.synchronize do
52
+ if (s = @inflight_sockets.delete(sock))
53
+ @available_sockets[s.key] << s
54
+ else
55
+ @log.debug("there is no socket #{sock}")
56
+ end
57
+ end
58
+ end
59
+
60
+ def revoke(sock)
61
+ @mutex.synchronize do
62
+ if (s = @inflight_sockets.delete(sock))
63
+ @inactive_sockets << s
64
+ else
65
+ @log.debug("there is no socket #{sock}")
66
+ end
67
+ end
68
+ end
69
+
70
+ def purge_obsolete_socks
71
+ sockets = []
72
+
73
+ @mutex.synchronize do
74
+ # don't touch @inflight_sockets
75
+
76
+ @available_sockets.each do |_, socks|
77
+ socks.each do |sock|
78
+ if expired_socket?(sock)
79
+ sockets << sock
80
+ socks.delete(sock)
81
+ end
82
+ end
83
+ end
84
+ @available_sockets = @available_sockets.select { |_, v| !v.empty? }
85
+
86
+ sockets += @inactive_sockets
87
+ @inactive_sockets.clear
88
+ end
89
+
90
+ sockets.each do |s|
91
+ s.sock.close rescue nil
92
+ end
93
+ end
94
+
95
+ def clear
96
+ sockets = []
97
+ @mutex.synchronize do
98
+ sockets += @available_sockets.values.flat_map { |v| v }
99
+ sockets += @inflight_sockets.values
100
+ sockets += @inactive_sockets
101
+
102
+ @available_sockets.clear
103
+ @inflight_sockets.clear
104
+ @inactive_sockets.clear
105
+ end
106
+
107
+ sockets.each do |s|
108
+ s.sock.close rescue nil
109
+ end
110
+ end
111
+
112
+ private
113
+
114
+ # this method is not thread safe
115
+ def pick_socket(key)
116
+ if @available_sockets[key].empty?
117
+ return nil
118
+ end
119
+
120
+ t = Time.now
121
+ if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
122
+ @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
123
+ s
124
+ else
125
+ nil
126
+ end
127
+ end
128
+
129
+ def timeout
130
+ @timeout && Time.now + @timeout
131
+ end
132
+
133
+ def expired_socket?(sock, time: Time.now)
134
+ sock.timeout ? sock.timeout < time : false
135
+ end
136
+ end
137
+ end
138
+ end
@@ -0,0 +1,231 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'net/http'
18
+ require 'uri'
19
+ require 'openssl'
20
+ require 'fluent/plugin/output'
21
+ require 'fluent/plugin_helper/socket'
22
+
23
+ module Fluent::Plugin
24
+ class HTTPOutput < Output
25
+ Fluent::Plugin.register_output('http', self)
26
+
27
+ class RetryableResponse < StandardError; end
28
+
29
+ helpers :formatter
30
+
31
+ desc 'The endpoint for HTTP request, e.g. http://example.com/api'
32
+ config_param :endpoint, :string
33
+ desc 'The method for HTTP request'
34
+ config_param :http_method, :enum, list: [:put, :post], default: :post
35
+ desc 'The proxy for HTTP request'
36
+ config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
37
+ desc 'Content-Type for HTTP request'
38
+ config_param :content_type, :string, default: nil
39
+ desc 'Additional headers for HTTP request'
40
+ config_param :headers, :hash, default: nil
41
+
42
+ desc 'The connection open timeout in seconds'
43
+ config_param :open_timeout, :integer, default: nil
44
+ desc 'The read timeout in seconds'
45
+ config_param :read_timeout, :integer, default: nil
46
+ desc 'The TLS timeout in seconds'
47
+ config_param :ssl_timeout, :integer, default: nil
48
+
49
+ desc 'The CA certificate path for TLS'
50
+ config_param :tls_ca_cert_path, :string, default: nil
51
+ desc 'The client certificate path for TLS'
52
+ config_param :tls_client_cert_path, :string, default: nil
53
+ desc 'The client private key path for TLS'
54
+ config_param :tls_private_key_path, :string, default: nil
55
+ desc 'The client private key passphrase for TLS'
56
+ config_param :tls_private_key_passphrase, :string, default: nil, secret: true
57
+ desc 'The verify mode of TLS'
58
+ config_param :tls_verify_mode, :enum, list: [:none, :peer], default: :peer
59
+ desc 'The default version of TLS'
60
+ config_param :tls_version, :enum, list: Fluent::PluginHelper::Socket::TLS_SUPPORTED_VERSIONS, default: Fluent::PluginHelper::Socket::TLS_DEFAULT_VERSION
61
+ desc 'The cipher configuration of TLS'
62
+ config_param :tls_ciphers, :string, default: Fluent::PluginHelper::Socket::CIPHERS_DEFAULT
63
+
64
+ desc 'Raise UnrecoverableError when the response is non success, 4xx/5xx'
65
+ config_param :error_response_as_unrecoverable, :bool, default: true
66
+ desc 'The list of retryable response code'
67
+ config_param :retryable_response_codes, :array, value_type: :integer, default: [503]
68
+
69
+ config_section :format do
70
+ config_set_default :@type, 'json'
71
+ end
72
+
73
+ config_section :auth, required: false, multi: false do
74
+ desc 'The method for HTTP authentication'
75
+ config_param :method, :enum, list: [:basic], default: :basic
76
+ desc 'The username for basic authentication'
77
+ config_param :username, :string, default: nil
78
+ desc 'The password for basic authentication'
79
+ config_param :password, :string, default: nil, secret: true
80
+ end
81
+
82
+ def initialize
83
+ super
84
+
85
+ @uri = nil
86
+ @proxy_uri = nil
87
+ @formatter = nil
88
+ end
89
+
90
+ def configure(conf)
91
+ super
92
+
93
+ @http_opt = setup_http_option
94
+ @proxy_uri = URI.parse(@proxy) if @proxy
95
+ @formatter = formatter_create
96
+ @content_type = setup_content_type unless @content_type
97
+ end
98
+
99
+ def multi_workers_ready?
100
+ true
101
+ end
102
+
103
+ def formatted_to_msgpack_binary?
104
+ @formatter_configs.first[:@type] == 'msgpack'
105
+ end
106
+
107
+ def format(tag, time, record)
108
+ @formatter.format(tag, time, record)
109
+ end
110
+
111
+ def write(chunk)
112
+ uri = parse_endpoint(chunk)
113
+ req = create_request(chunk, uri)
114
+
115
+ log.debug { "#{@http_method.capitalize} data to #{uri.to_s} with chunk(#{dump_unique_id_hex(chunk.unique_id)})" }
116
+
117
+ send_request(uri, req)
118
+ end
119
+
120
+ private
121
+
122
+ def setup_content_type
123
+ case @formatter_configs.first[:@type]
124
+ when 'json'
125
+ 'application/x-ndjson'
126
+ when 'csv'
127
+ 'text/csv'
128
+ when 'tsv', 'ltsv'
129
+ 'text/tab-separated-values'
130
+ when 'msgpack'
131
+ 'application/x-msgpack'
132
+ when 'out_file', 'single_value', 'stdout', 'hash'
133
+ 'text/plain'
134
+ else
135
+ raise Fluent::ConfigError, "can't determine Content-Type from formatter type. Set content_type parameter explicitly"
136
+ end
137
+ end
138
+
139
+ def setup_http_option
140
+ use_ssl = @endpoint.start_with?('https')
141
+ opt = {
142
+ open_timeout: @open_timeout,
143
+ read_timeout: @read_timeout,
144
+ ssl_timeout: @ssl_timeout,
145
+ use_ssl: use_ssl
146
+ }
147
+
148
+ if use_ssl
149
+ if @tls_ca_cert_path
150
+ raise Fluent::ConfigError, "tls_ca_cert_path is wrong: #{@tls_ca_cert_path}" unless File.file?(@tls_ca_cert_path)
151
+ opt[:ca_file] = @tls_ca_cert_path
152
+ end
153
+ if @tls_client_cert_path
154
+ raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
155
+ opt[:cert] = OpenSSL::X509::Certificate.new(File.read(@tls_client_cert_path))
156
+ end
157
+ if @tls_private_key_path
158
+ raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
159
+ opt[:key] = OpenSSL::PKey.read(File.read(@tls_private_key_path), @tls_private_key_passphrase)
160
+ end
161
+ opt[:verify_mode] = case @tls_verify_mode
162
+ when :none
163
+ OpenSSL::SSL::VERIFY_NONE
164
+ when :peer
165
+ OpenSSL::SSL::VERIFY_PEER
166
+ end
167
+ opt[:ciphers] = @tls_ciphers
168
+ opt[:ssl_version] = @tls_version
169
+ end
170
+
171
+ opt
172
+ end
173
+
174
+ def parse_endpoint(chunk)
175
+ endpoint = extract_placeholders(@endpoint, chunk)
176
+ URI.parse(endpoint)
177
+ end
178
+
179
+ def set_headers(req)
180
+ if @headers
181
+ @headers.each do |k, v|
182
+ req[k] = v
183
+ end
184
+ end
185
+ req['Content-Type'] = @content_type
186
+ end
187
+
188
+ def create_request(chunk, uri)
189
+ req = case @http_method
190
+ when :post
191
+ Net::HTTP::Post.new(uri.request_uri)
192
+ when :put
193
+ Net::HTTP::Put.new(uri.request_uri)
194
+ end
195
+ if @auth
196
+ req.basic_auth(@auth.username, @auth.password)
197
+ end
198
+ set_headers(req)
199
+ req.body = chunk.read
200
+ req
201
+ end
202
+
203
+ def send_request(uri, req)
204
+ res = if @proxy_uri
205
+ Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
206
+ http.request(req)
207
+ }
208
+ else
209
+ Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
210
+ http.request(req)
211
+ }
212
+ end
213
+
214
+ if res.is_a?(Net::HTTPSuccess)
215
+ log.debug { "#{res.code} #{res.message}#{res.body}" }
216
+ else
217
+ msg = "#{res.code} #{res.message}#{res.body}"
218
+
219
+ if @retryable_response_codes.include?(res.code.to_i)
220
+ raise RetryableResponse, msg
221
+ end
222
+
223
+ if @error_response_as_unrecoverable
224
+ raise Fluent::UnrecoverableError, msg
225
+ else
226
+ log.error "got error response from '#{@http_method.capitalize} #{uri.to_s}' : #{msg}"
227
+ end
228
+ end
229
+ end
230
+ end
231
+ end