fluentd 1.5.2 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +15 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  5. data/CHANGELOG.md +29 -0
  6. data/CONTRIBUTING.md +1 -1
  7. data/Gemfile +5 -0
  8. data/Rakefile +6 -1
  9. data/appveyor.yml +9 -10
  10. data/lib/fluent/command/fluentd.rb +4 -0
  11. data/lib/fluent/config/literal_parser.rb +2 -2
  12. data/lib/fluent/plugin/base.rb +1 -0
  13. data/lib/fluent/plugin/buffer.rb +22 -0
  14. data/lib/fluent/plugin/in_forward.rb +2 -2
  15. data/lib/fluent/plugin/in_monitor_agent.rb +90 -131
  16. data/lib/fluent/plugin/out_forward.rb +4 -0
  17. data/lib/fluent/plugin/output.rb +31 -1
  18. data/lib/fluent/plugin/parser_none.rb +1 -2
  19. data/lib/fluent/plugin_helper.rb +1 -0
  20. data/lib/fluent/plugin_helper/cert_option.rb +1 -1
  21. data/lib/fluent/plugin_helper/http_server.rb +75 -0
  22. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  23. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  24. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  25. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  26. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  27. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  28. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  29. data/lib/fluent/plugin_helper/socket.rb +8 -2
  30. data/lib/fluent/supervisor.rb +5 -2
  31. data/lib/fluent/time.rb +13 -0
  32. data/lib/fluent/version.rb +1 -1
  33. data/test/command/test_fluentd.rb +36 -2
  34. data/test/helper.rb +1 -0
  35. data/test/helpers/fuzzy_assert.rb +89 -0
  36. data/test/plugin/test_buf_file.rb +1 -1
  37. data/test/plugin/test_in_http.rb +2 -5
  38. data/test/plugin/test_in_monitor_agent.rb +118 -17
  39. data/test/plugin/test_in_udp.rb +0 -2
  40. data/test/plugin/test_out_file.rb +15 -12
  41. data/test/plugin/test_out_forward.rb +18 -0
  42. data/test/plugin/test_out_secondary_file.rb +6 -4
  43. data/test/plugin/test_output_as_buffered.rb +4 -0
  44. data/test/plugin/test_output_as_buffered_retries.rb +0 -2
  45. data/test/plugin/test_output_as_buffered_secondary.rb +0 -3
  46. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  47. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  48. data/test/plugin_helper/data/cert/cert.pem +19 -0
  49. data/test/plugin_helper/http_server/test_app.rb +65 -0
  50. data/test/plugin_helper/http_server/test_route.rb +32 -0
  51. data/test/plugin_helper/test_cert_option.rb +16 -0
  52. data/test/plugin_helper/test_http_server_helper.rb +203 -0
  53. data/test/plugin_helper/test_server.rb +1 -7
  54. data/test/test_event_time.rb +13 -0
  55. data/test/test_log.rb +8 -6
  56. data/test/test_supervisor.rb +3 -0
  57. metadata +28 -2
@@ -39,6 +39,8 @@ module Fluent::Plugin
39
39
 
40
40
  desc 'The timeout time when sending event logs.'
41
41
  config_param :send_timeout, :time, default: 60
42
+ desc 'The timeout time for socket connect'
43
+ config_param :connect_timeout, :time, default: nil
42
44
  # TODO: add linger_timeout, recv_timeout
43
45
 
44
46
  desc 'The protocol to use for heartbeats (default is the same with "transport").'
@@ -376,6 +378,7 @@ module Fluent::Plugin
376
378
  linger_timeout: Fluent.windows? ? nil : @send_timeout,
377
379
  send_timeout: @send_timeout,
378
380
  recv_timeout: @ack_response_timeout,
381
+ connect_timeout: @connect_timeout,
379
382
  &block
380
383
  )
381
384
  when :tcp
@@ -384,6 +387,7 @@ module Fluent::Plugin
384
387
  linger_timeout: @send_timeout,
385
388
  send_timeout: @send_timeout,
386
389
  recv_timeout: @ack_response_timeout,
390
+ connect_timeout: @connect_timeout,
387
391
  &block
388
392
  )
389
393
  else
@@ -184,6 +184,8 @@ module Fluent
184
184
  @emit_records = 0
185
185
  @write_count = 0
186
186
  @rollback_count = 0
187
+ @flush_time_count = 0
188
+ @slow_flush_count = 0
187
189
 
188
190
  # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
189
191
  if implement?(:synchronous)
@@ -1173,7 +1175,10 @@ module Fluent
1173
1175
  @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
1174
1176
  end
1175
1177
  end
1176
- @buffer.takeback_chunk(chunk.unique_id)
1178
+
1179
+ if @buffer.takeback_chunk(chunk.unique_id)
1180
+ @counters_monitor.synchronize { @rollback_count += 1 }
1181
+ end
1177
1182
 
1178
1183
  update_retry_state(chunk.unique_id, using_secondary, e)
1179
1184
 
@@ -1202,7 +1207,10 @@ module Fluent
1202
1207
 
1203
1208
  def check_slow_flush(start)
1204
1209
  elapsed_time = Fluent::Clock.now - start
1210
+ elapsed_millsec = (elapsed_time * 1000).to_i
1211
+ @counters_monitor.synchronize { @flush_time_count += elapsed_millsec }
1205
1212
  if elapsed_time > @slow_flush_log_threshold
1213
+ @counters_monitor.synchronize { @slow_flush_count += 1 }
1206
1214
  log.warn "buffer flush took longer time than slow_flush_log_threshold:",
1207
1215
  elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
1208
1216
  end
@@ -1457,6 +1465,28 @@ module Fluent
1457
1465
  state.mutex.unlock
1458
1466
  end
1459
1467
  end
1468
+
1469
+ def statistics
1470
+ stats = {
1471
+ 'emit_records' => @emit_records,
1472
+ # Respect original name
1473
+ # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
1474
+ 'retry_count' => @num_errors,
1475
+ 'emit_count' => @emit_count,
1476
+ 'write_count' => @write_count,
1477
+ 'rollback_count' => @rollback_count,
1478
+ 'slow_flush_count' => @slow_flush_count,
1479
+ 'flush_time_count' => @flush_time_count,
1480
+ }
1481
+
1482
+ if @buffer && @buffer.respond_to?(:statistics)
1483
+ (@buffer.statistics['buffer'] || {}).each do |k, v|
1484
+ stats["buffer_#{k}"] = v
1485
+ end
1486
+ end
1487
+
1488
+ { 'output' => stats }
1489
+ end
1460
1490
  end
1461
1491
  end
1462
1492
  end
@@ -27,8 +27,7 @@ module Fluent
27
27
  config_param :message_key, :string, default: 'message'
28
28
 
29
29
  def parse(text)
30
- record = {}
31
- record[@message_key] = text
30
+ record = {@message_key => text}
32
31
  time = @estimate_current_event ? Fluent::EventTime.now : nil
33
32
  yield time, record
34
33
  end
@@ -22,6 +22,7 @@ require 'fluent/plugin_helper/child_process'
22
22
  require 'fluent/plugin_helper/storage'
23
23
  require 'fluent/plugin_helper/parser'
24
24
  require 'fluent/plugin_helper/formatter'
25
+ require 'fluent/plugin_helper/http_server'
25
26
  require 'fluent/plugin_helper/inject'
26
27
  require 'fluent/plugin_helper/extract'
27
28
  require 'fluent/plugin_helper/socket'
@@ -168,7 +168,7 @@ module Fluent
168
168
 
169
169
  def cert_option_certificates_from_file(path)
170
170
  data = File.read(path)
171
- pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
171
+ pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n?', Regexp::MULTILINE)
172
172
  list = []
173
173
  data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
174
174
  list
@@ -0,0 +1,75 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ begin
18
+ # raise if RUBY_VERSION < 2.3.x. see Gemfile
19
+ require 'async'
20
+ require 'fluent/plugin_helper/http_server/server'
21
+ rescue LoadError => _
22
+ require 'fluent/plugin_helper/http_server/compat/server'
23
+ Fluent::PluginHelper::HttpServer::Server = Fluent::PluginHelper::HttpServer::Compat::Server
24
+ end
25
+
26
+ require 'fluent/plugin_helper/thread'
27
+
28
+ module Fluent
29
+ module PluginHelper
30
+ module HttpServer
31
+ include Fluent::PluginHelper::Thread
32
+ # stop : stop http server and mark callback thread as stopped
33
+ # shutdown : [-]
34
+ # close : correct stopped threads
35
+ # terminate: kill thread
36
+
37
+ # @param addr [String] Listen address
38
+ # @param port [String] Listen port
39
+ # @param logger [Logger] logger used in this server
40
+ # @param default_app [Object] This method must have #call.
41
+ def create_http_server(addr:, port:, logger:, default_app: nil)
42
+ unless block_given?
43
+ raise ArgumentError, 'BUG: callback not specified'
44
+ end
45
+
46
+ @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
47
+ yield(serv)
48
+ end
49
+
50
+ _block_until_http_server_start do |notify|
51
+ thread_create(:plugin_helper_http_server) do
52
+ @_http_server.start(notify)
53
+ end
54
+ end
55
+ end
56
+
57
+ def stop
58
+ if @_http_server
59
+ @_http_server.stop
60
+ end
61
+
62
+ super
63
+ end
64
+
65
+ private
66
+
67
+ # To block until server is ready to listen
68
+ def _block_until_http_server_start
69
+ que = Queue.new
70
+ yield(que)
71
+ que.pop
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,79 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'async/http/protocol'
18
+ require 'fluent/plugin_helper/http_server/methods'
19
+ require 'fluent/plugin_helper/http_server/request'
20
+
21
+ module Fluent
22
+ module PluginHelper
23
+ module HttpServer
24
+ class App
25
+ def initialize(router, logger)
26
+ @logger = logger
27
+ @router = router
28
+ end
29
+
30
+ # Required method by async-http
31
+ def call(request)
32
+ method = request.method
33
+ resp =
34
+ case method
35
+ when HttpServer::Methods::GET
36
+ get(request)
37
+ when HttpServer::Methods::HEAD
38
+ head(request)
39
+ when HttpServer::Methods::POST
40
+ post(request)
41
+ when HttpServer::Methods::PATCH
42
+ patch(request)
43
+ when HttpServer::Methods::PUT
44
+ put(request)
45
+ when HttpServer::Methods::DELETE
46
+ delete(request)
47
+ when HttpServer::Methods::OPTIONS
48
+ options(request)
49
+ when HttpServer::Methods::CONNECT
50
+ connect(request)
51
+ when HttpServer::Methods::TRACE
52
+ trace(request)
53
+ else
54
+ raise "Unknown method #{method}"
55
+ end
56
+ Protocol::HTTP::Response[*resp]
57
+ rescue => e
58
+ @logger.error(e)
59
+ Protocol::HTTP::Response[500, { 'Content-Type' => 'text/plain' }, 'Internal Server Error']
60
+ end
61
+
62
+ HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
63
+ define_method(name) do |request|
64
+ req = Request.new(request)
65
+
66
+ path = req.path
67
+ canonical_path =
68
+ if path.size >= 2 && !path.end_with?('/')
69
+ "#{path}/"
70
+ else
71
+ path
72
+ end
73
+ @router.route!(name, canonical_path, req)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,81 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/http_server/methods'
18
+ require 'fluent/plugin_helper/http_server/compat/webrick_handler'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module HttpServer
23
+ module Compat
24
+ class Server
25
+ # @param logger [Logger]
26
+ # @param default_app [Object] ignored option. only for compat
27
+ def initialize(addr:, port:, logger:, default_app: nil)
28
+ @addr = addr
29
+ @port = port
30
+ @logger = logger
31
+ @server = WEBrick::HTTPServer.new(
32
+ BindAddress: @addr,
33
+ Port: @port,
34
+ Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
35
+ AccessLog: [],
36
+ )
37
+
38
+ # @example ["/example.json", :get, handler object]
39
+ @methods = []
40
+
41
+ if block_given?
42
+ yield(self)
43
+ end
44
+ end
45
+
46
+ def start(notify = nil)
47
+ build_handler
48
+ notify.push(:ready)
49
+ @logger.debug('Start webrick HTTP server listening')
50
+ @server.start
51
+ end
52
+
53
+ def stop
54
+ @server.shutdown
55
+ @server.stop
56
+ end
57
+
58
+ HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
59
+ define_method(name) do |path, app = nil, &block|
60
+ if (block && app) || (!block && !app)
61
+ raise 'You must specify either app or block in the same time'
62
+ end
63
+
64
+ # Do not build a handler class here to able to handle multiple methods for single path.
65
+ @methods << [path, name, app || block]
66
+ end
67
+ end
68
+
69
+ private
70
+
71
+ def build_handler
72
+ @methods.group_by(&:first).each do |(path, rest)|
73
+ klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
74
+ @server.mount(path, klass)
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,58 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'webrick'
18
+ require 'json'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module HttpServer
23
+ module Compat
24
+ class WebrickHandler
25
+ # **opt is enough. but I wrote a signature explicitly for readability
26
+ def self.build(get: nil, head: nil, post: nil, put: nil, patch: nil, delete: nil, connect: nil, options: nil, trace: nil)
27
+ opt = { get: get, head: head, post: post, put: put, patch: patch, delete: delete, connect: connect, options: options, trace: trace }
28
+
29
+ Class.new(WEBrick::HTTPServlet::AbstractServlet) do
30
+ HttpServer::Methods::ALL.each do |name|
31
+ define_method("do_#{name}") do |request, response|
32
+ code, headers, body =
33
+ if request.path_info != ''
34
+ render_json(404, 'message' => 'Not found')
35
+ else
36
+ begin
37
+ opt[name.downcase.to_sym].call(request)
38
+ rescue => _
39
+ render_json(500, 'message' => 'Something went wrong')
40
+ end
41
+ end
42
+
43
+ response.status = code
44
+ headers.each { |k, v| response[k] = v }
45
+ response.body = body
46
+ end
47
+
48
+ def render_json(code, obj)
49
+ [code, { 'Content-Type' => 'application/json' }, obj.to_json]
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,35 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module PluginHelper
19
+ module HttpServer
20
+ module Methods
21
+ GET = 'GET'.freeze
22
+ HEAD = 'HEAD'.freeze
23
+ POST = 'POST'.freeze
24
+ PUT = 'PUT'.freeze
25
+ PATCH = 'PATCH'.freeze
26
+ DELETE = 'DELETE'.freeze
27
+ OPTIONS = 'OPTIONS'.freeze
28
+ CONNECT = 'CONNECT'.freeze
29
+ TRACE = 'TRACE'.freeze
30
+
31
+ ALL = [GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS, TRACE].freeze
32
+ end
33
+ end
34
+ end
35
+ end