fluentd 1.5.2 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +15 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +39 -0
  4. data/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
  5. data/CHANGELOG.md +29 -0
  6. data/CONTRIBUTING.md +1 -1
  7. data/Gemfile +5 -0
  8. data/Rakefile +6 -1
  9. data/appveyor.yml +9 -10
  10. data/lib/fluent/command/fluentd.rb +4 -0
  11. data/lib/fluent/config/literal_parser.rb +2 -2
  12. data/lib/fluent/plugin/base.rb +1 -0
  13. data/lib/fluent/plugin/buffer.rb +22 -0
  14. data/lib/fluent/plugin/in_forward.rb +2 -2
  15. data/lib/fluent/plugin/in_monitor_agent.rb +90 -131
  16. data/lib/fluent/plugin/out_forward.rb +4 -0
  17. data/lib/fluent/plugin/output.rb +31 -1
  18. data/lib/fluent/plugin/parser_none.rb +1 -2
  19. data/lib/fluent/plugin_helper.rb +1 -0
  20. data/lib/fluent/plugin_helper/cert_option.rb +1 -1
  21. data/lib/fluent/plugin_helper/http_server.rb +75 -0
  22. data/lib/fluent/plugin_helper/http_server/app.rb +79 -0
  23. data/lib/fluent/plugin_helper/http_server/compat/server.rb +81 -0
  24. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +58 -0
  25. data/lib/fluent/plugin_helper/http_server/methods.rb +35 -0
  26. data/lib/fluent/plugin_helper/http_server/request.rb +42 -0
  27. data/lib/fluent/plugin_helper/http_server/router.rb +54 -0
  28. data/lib/fluent/plugin_helper/http_server/server.rb +87 -0
  29. data/lib/fluent/plugin_helper/socket.rb +8 -2
  30. data/lib/fluent/supervisor.rb +5 -2
  31. data/lib/fluent/time.rb +13 -0
  32. data/lib/fluent/version.rb +1 -1
  33. data/test/command/test_fluentd.rb +36 -2
  34. data/test/helper.rb +1 -0
  35. data/test/helpers/fuzzy_assert.rb +89 -0
  36. data/test/plugin/test_buf_file.rb +1 -1
  37. data/test/plugin/test_in_http.rb +2 -5
  38. data/test/plugin/test_in_monitor_agent.rb +118 -17
  39. data/test/plugin/test_in_udp.rb +0 -2
  40. data/test/plugin/test_out_file.rb +15 -12
  41. data/test/plugin/test_out_forward.rb +18 -0
  42. data/test/plugin/test_out_secondary_file.rb +6 -4
  43. data/test/plugin/test_output_as_buffered.rb +4 -0
  44. data/test/plugin/test_output_as_buffered_retries.rb +0 -2
  45. data/test/plugin/test_output_as_buffered_secondary.rb +0 -3
  46. data/test/plugin_helper/data/cert/cert-key.pem +27 -0
  47. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +19 -0
  48. data/test/plugin_helper/data/cert/cert.pem +19 -0
  49. data/test/plugin_helper/http_server/test_app.rb +65 -0
  50. data/test/plugin_helper/http_server/test_route.rb +32 -0
  51. data/test/plugin_helper/test_cert_option.rb +16 -0
  52. data/test/plugin_helper/test_http_server_helper.rb +203 -0
  53. data/test/plugin_helper/test_server.rb +1 -7
  54. data/test/test_event_time.rb +13 -0
  55. data/test/test_log.rb +8 -6
  56. data/test/test_supervisor.rb +3 -0
  57. metadata +28 -2
@@ -0,0 +1,42 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'async/http/protocol'
18
+ require 'fluent/plugin_helper/http_server/methods'
19
+
20
+ module Fluent
21
+ module PluginHelper
22
+ module HttpServer
23
+ class Request
24
+ attr_reader :path, :query_string
25
+
26
+ def initialize(request)
27
+ @request = request
28
+ path = request.path
29
+ @path, @query_string = path.split('?', 2)
30
+ end
31
+
32
+ def query
33
+ @query_string && CGI.parse(@query_string)
34
+ end
35
+
36
+ def body
37
+ @request.body && @request.body.read
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,54 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'async/http/protocol'
18
+
19
+ module Fluent
20
+ module PluginHelper
21
+ module HttpServer
22
+ class Router
23
+ class NotFoundApp
24
+ def self.call(req)
25
+ [404, { 'Content-Type' => 'text/plain' }, "404 Not Found: #{req.path}\n"]
26
+ end
27
+ end
28
+
29
+ def initialize(default_app = nil)
30
+ @router = { get: {}, head: {}, post: {}, put: {}, patch: {}, delete: {}, connect: {}, options: {}, trace: {} }
31
+ @default_app = default_app || NotFoundApp
32
+ end
33
+
34
+ # @param method [Symbol]
35
+ # @param path [String]
36
+ # @param app [Object]
37
+ def mount(method, path, app)
38
+ if @router[method].include?(path)
39
+ raise "#{path} is already mounted"
40
+ end
41
+
42
+ @router[method][path] = app
43
+ end
44
+
45
+ # @param method [Symbol]
46
+ # @param path [String]
47
+ # @param request [Fluent::PluginHelper::HttpServer::Request]
48
+ def route!(method, path, request)
49
+ @router.fetch(method).fetch(path, @default_app).call(request)
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,87 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'async'
18
+ require 'async/http'
19
+ require 'async/http/endpoint'
20
+
21
+ require 'fluent/plugin_helper/http_server/app'
22
+ require 'fluent/plugin_helper/http_server/router'
23
+ require 'fluent/plugin_helper/http_server/methods'
24
+
25
+ module Fluent
26
+ module PluginHelper
27
+ module HttpServer
28
+ class Server
29
+ # @param default_app [Object] This method must have #call.
30
+ def initialize(addr:, port:, logger:, default_app: nil)
31
+ @addr = addr
32
+ @port = port
33
+ @logger = logger
34
+
35
+ # TODO: support https and http2
36
+ @uri = URI("http://#{@addr}:#{@port}").to_s
37
+ @router = Router.new(default_app)
38
+ @reactor = Async::Reactor.new
39
+ @server = Async::HTTP::Server.new(
40
+ App.new(@router, @logger),
41
+ Async::HTTP::Endpoint.parse(@uri)
42
+ )
43
+
44
+ if block_given?
45
+ yield(self)
46
+ end
47
+ end
48
+
49
+ def start(notify = nil)
50
+ @logger.debug("Start async HTTP server listening #{@uri}")
51
+ task = @reactor.run do
52
+ @server.run
53
+
54
+ if notify
55
+ notify.push(:ready)
56
+ end
57
+ end
58
+
59
+ task.stop
60
+ @logger.debug('Finished HTTP server')
61
+ end
62
+
63
+ def stop
64
+ @logger.debug('closing HTTP server')
65
+
66
+ if @reactor
67
+ @reactor.stop
68
+ end
69
+ end
70
+
71
+ HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
72
+ define_method(name) do |path, app = nil, &block|
73
+ unless path.end_with?('/')
74
+ path << '/'
75
+ end
76
+
77
+ if (block && app) || (!block && !app)
78
+ raise 'You must specify either app or block in the same time'
79
+ end
80
+
81
+ @router.mount(name, path, app || block)
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
@@ -55,8 +55,14 @@ module Fluent
55
55
  end
56
56
  end
57
57
 
58
- def socket_create_tcp(host, port, resolve_name: false, **kwargs, &block)
59
- sock = WrappedSocket::TCP.new(host, port)
58
+ def socket_create_tcp(host, port, resolve_name: false, connect_timeout: nil, **kwargs, &block)
59
+ sock = if connect_timeout
60
+ s = ::Socket.tcp(host, port, connect_timeout: connect_timeout)
61
+ s.autoclose = false # avoid GC triggered close
62
+ WrappedSocket::TCP.for_fd(s.fileno)
63
+ else
64
+ WrappedSocket::TCP.new(host, port)
65
+ end
60
66
  socket_option_set(sock, resolve_name: resolve_name, **kwargs)
61
67
  if block
62
68
  begin
@@ -249,7 +249,7 @@ module Fluent
249
249
  config_fname = File.basename(path)
250
250
  config_basedir = File.dirname(path)
251
251
  # Assume fluent.conf encoding is UTF-8
252
- config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
252
+ config_data = File.open(path, "r:#{params['conf_encoding']}:utf-8") {|f| f.read }
253
253
  inline_config = params['inline_config']
254
254
  if inline_config == '-'
255
255
  config_data << "\n" << STDIN.read
@@ -422,6 +422,7 @@ module Fluent
422
422
  supervise: true,
423
423
  standalone_worker: false,
424
424
  signame: nil,
425
+ conf_encoding: 'utf-8'
425
426
  }
426
427
  end
427
428
 
@@ -440,6 +441,7 @@ module Fluent
440
441
  @config_path = opt[:config_path]
441
442
  @inline_config = opt[:inline_config]
442
443
  @use_v1_config = opt[:use_v1_config]
444
+ @conf_encoding = opt[:conf_encoding]
443
445
  @log_path = opt[:log_path]
444
446
  @dry_run = opt[:dry_run]
445
447
  @show_plugin_config = opt[:show_plugin_config]
@@ -618,6 +620,7 @@ module Fluent
618
620
  params['chuser'] = @chuser
619
621
  params['chgroup'] = @chgroup
620
622
  params['use_v1_config'] = @use_v1_config
623
+ params['conf_encoding'] = @conf_encoding
621
624
 
622
625
  # system config parameters
623
626
  params['workers'] = @workers
@@ -763,7 +766,7 @@ module Fluent
763
766
  def read_config
764
767
  @config_fname = File.basename(@config_path)
765
768
  @config_basedir = File.dirname(@config_path)
766
- @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
769
+ @config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
767
770
  if @inline_config == '-'
768
771
  @config_data << "\n" << STDIN.read
769
772
  elsif @inline_config
@@ -69,6 +69,19 @@ module Fluent
69
69
  @sec.to_s
70
70
  end
71
71
 
72
+ begin
73
+ # ruby 2.5 or later
74
+ Time.at(0, 0, :nanosecond)
75
+
76
+ def to_time
77
+ Time.at(@sec, @nsec, :nanosecond)
78
+ end
79
+ rescue
80
+ def to_time
81
+ Time.at(@sec, @nsec / 1000.0)
82
+ end
83
+ end
84
+
72
85
  def to_json(*args)
73
86
  @sec.to_s
74
87
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.5.2'
19
+ VERSION = '1.6.0'
20
20
 
21
21
  end
@@ -28,9 +28,9 @@ class TestFluentdCommand < ::Test::Unit::TestCase
28
28
  end
29
29
  end
30
30
 
31
- def create_conf_file(name, content)
31
+ def create_conf_file(name, content, ext_enc = 'utf-8')
32
32
  conf_path = File.join(TMP_DIR, name)
33
- File.open(conf_path, 'w') do |file|
33
+ File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
34
34
  file.write content
35
35
  end
36
36
  conf_path
@@ -203,6 +203,40 @@ CONF
203
203
  end
204
204
  end
205
205
 
206
+ sub_test_case 'with --conf-encoding' do
207
+ test 'runs successfully' do
208
+ conf = <<CONF
209
+ # テスト
210
+ <source>
211
+ @type dummy
212
+ tag dummy
213
+ dummy {"message": "yay!"}
214
+ </source>
215
+ <match dummy>
216
+ @type null
217
+ </match>
218
+ CONF
219
+ conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
220
+ assert_log_matches(create_cmdline(conf_path, '--conf-encoding', 'shift_jis'), "fluentd worker is now running", 'worker=0')
221
+ end
222
+
223
+ test 'failed to run by invalid encoding' do
224
+ conf = <<CONF
225
+ # テスト
226
+ <source>
227
+ @type dummy
228
+ tag dummy
229
+ dummy {"message": "yay!"}
230
+ </source>
231
+ <match dummy>
232
+ @type null
233
+ </match>
234
+ CONF
235
+ conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
236
+ assert_fluentd_fails_to_start(create_cmdline(conf_path), "invalid byte sequence in UTF-8")
237
+ end
238
+ end
239
+
206
240
  sub_test_case 'with system configuration about root directory' do
207
241
  setup do
208
242
  @root_path = File.join(TMP_DIR, "rootpath")
@@ -49,6 +49,7 @@ require 'fluent/plugin_helper'
49
49
  require 'fluent/msgpack_factory'
50
50
  require 'fluent/time'
51
51
  require 'serverengine'
52
+ require 'helpers/fuzzy_assert'
52
53
 
53
54
  module Fluent
54
55
  module Plugin
@@ -0,0 +1,89 @@
1
+ require 'test/unit'
2
+
3
+ class FuzzyIncludeAssertion
4
+ include Test::Unit::Assertions
5
+
6
+ def self.assert(expected, actual, message = nil)
7
+ new(expected, actual, message).assert
8
+ end
9
+
10
+ def initialize(expected, actual, message)
11
+ @expected = expected
12
+ @actual = actual
13
+ @message = message
14
+ end
15
+
16
+ def assert
17
+ if collection?
18
+ assert_same_collection
19
+ else
20
+ assert_same_value
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ def assert_same_value
27
+ m = "expected(#{@expected}) !== actual(#{@actual.inspect})"
28
+ if @message
29
+ m = "#{@message}: #{m}"
30
+ end
31
+ assert_true(@expected === @actual, m)
32
+ end
33
+
34
+ def assert_same_class
35
+ if @expected.class != @actual.class
36
+ if (@expected.class.ancestors | @actual.class.ancestors).empty?
37
+ assert_equal(@expected.class, @actual.class, @message)
38
+ end
39
+ end
40
+ end
41
+
42
+ def assert_same_collection
43
+ assert_same_class
44
+ assert_same_values
45
+ end
46
+
47
+ def assert_same_values
48
+ if @expected.is_a?(Array)
49
+ @expected.each_with_index do |val, i|
50
+ self.class.assert(val, @actual[i], @message)
51
+ end
52
+ else
53
+ @expected.each do |key, val|
54
+ self.class.assert(val, @actual[key], "#{key}: ")
55
+ end
56
+ end
57
+ end
58
+
59
+ def collection?
60
+ @actual.is_a?(Array) || @actual.is_a?(Hash)
61
+ end
62
+ end
63
+
64
+ class FuzzyAssertion < FuzzyIncludeAssertion
65
+ private
66
+
67
+ def assert_same_collection
68
+ super
69
+ assert_same_keys
70
+ end
71
+
72
+ def assert_same_keys
73
+ if @expected.is_a?(Array)
74
+ assert_equal(@expected.size, @actual.size, "expected.size(#{@expected}) != actual.size(#{@expected})")
75
+ else
76
+ assert_equal(@expected.keys.sort, @actual.keys.sort)
77
+ end
78
+ end
79
+ end
80
+
81
+ module FuzzyAssert
82
+ def assert_fuzzy_include(left, right, message = nil)
83
+ FuzzyIncludeAssertion.new(left, right, message).assert
84
+ end
85
+
86
+ def assert_fuzzy_equal(left, right, message = nil)
87
+ FuzzyAssertion.new(left, right, message).assert
88
+ end
89
+ end
@@ -912,7 +912,7 @@ class FileBufferTest < Test::Unit::TestCase
912
912
  assert_equal mode, staged[m].state
913
913
  end
914
914
 
915
- def compare_queued_chunk(queued, id, num, mode)
915
+ def compare_queued_chunk(queued, id, num, mode)
916
916
  assert_equal 1, queued.size
917
917
  assert_equal id, queued[0].unique_id
918
918
  assert_equal num, queued[0].size
@@ -707,7 +707,6 @@ class HttpInputTest < Test::Unit::TestCase
707
707
  ["tag2", time, {"a"=>2}],
708
708
  ]
709
709
  res_codes = []
710
- res_headers = []
711
710
 
712
711
  d.run do
713
712
  events.each do |tag, time, record|
@@ -731,7 +730,6 @@ class HttpInputTest < Test::Unit::TestCase
731
730
  ["tag2", time, {"a"=>2}],
732
731
  ]
733
732
  res_codes = []
734
- res_headers = []
735
733
 
736
734
  d.run do
737
735
  events.each do |tag, time, record|
@@ -803,10 +801,9 @@ class HttpInputTest < Test::Unit::TestCase
803
801
  # Send two requests the second one has no Content-Type in Keep-Alive
804
802
  Net::HTTP.start("127.0.0.1", PORT) do |http|
805
803
  req = Net::HTTP::Post.new("/foodb/bartbl", {"connection" => "keepalive", "Content-Type" => "application/json"})
806
- res = http.request(req)
807
-
804
+ http.request(req)
808
805
  req = Net::HTTP::Get.new("/foodb/bartbl", {"connection" => "keepalive"})
809
- res = http.request(req)
806
+ http.request(req)
810
807
  end
811
808
 
812
809
  end