fluentd 1.8.1-x86-mingw32 → 1.9.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.gitlab-ci.yml +14 -58
  3. data/.travis.yml +2 -17
  4. data/CHANGELOG.md +39 -0
  5. data/Gemfile +1 -4
  6. data/README.md +2 -6
  7. data/fluentd.gemspec +6 -6
  8. data/lib/fluent/command/cat.rb +1 -3
  9. data/lib/fluent/command/plugin_generator.rb +2 -1
  10. data/lib/fluent/config.rb +19 -0
  11. data/lib/fluent/config/literal_parser.rb +13 -8
  12. data/lib/fluent/config/v1_parser.rb +5 -3
  13. data/lib/fluent/engine.rb +60 -9
  14. data/lib/fluent/ext_monitor_require.rb +28 -0
  15. data/lib/fluent/load.rb +1 -1
  16. data/lib/fluent/msgpack_factory.rb +16 -4
  17. data/lib/fluent/plugin/base.rb +5 -0
  18. data/lib/fluent/plugin/buf_file.rb +10 -6
  19. data/lib/fluent/plugin/buf_file_single.rb +10 -6
  20. data/lib/fluent/plugin/buffer.rb +40 -22
  21. data/lib/fluent/plugin/buffer/chunk.rb +1 -1
  22. data/lib/fluent/plugin/in_http.rb +9 -9
  23. data/lib/fluent/plugin/in_tail.rb +8 -6
  24. data/lib/fluent/plugin/out_http.rb +2 -2
  25. data/lib/fluent/plugin/output.rb +2 -2
  26. data/lib/fluent/plugin/parser.rb +6 -0
  27. data/lib/fluent/plugin_helper/http_server.rb +0 -1
  28. data/lib/fluent/plugin_helper/record_accessor.rb +0 -8
  29. data/lib/fluent/plugin_helper/server.rb +6 -21
  30. data/lib/fluent/plugin_id.rb +9 -4
  31. data/lib/fluent/static_config_analysis.rb +194 -0
  32. data/lib/fluent/supervisor.rb +103 -28
  33. data/lib/fluent/system_config.rb +2 -1
  34. data/lib/fluent/test/driver/base.rb +4 -3
  35. data/lib/fluent/variable_store.rb +40 -0
  36. data/lib/fluent/version.rb +1 -1
  37. data/test/config/test_config_parser.rb +19 -16
  38. data/test/config/test_system_config.rb +6 -4
  39. data/test/plugin/test_in_exec.rb +9 -9
  40. data/test/plugin/test_in_forward.rb +10 -11
  41. data/test/plugin/test_in_http.rb +35 -3
  42. data/test/plugin/test_in_object_space.rb +3 -7
  43. data/test/plugin/test_out_exec_filter.rb +28 -45
  44. data/test/plugin/test_out_forward.rb +2 -2
  45. data/test/plugin/test_out_http.rb +8 -2
  46. data/test/plugin/test_output.rb +3 -3
  47. data/test/plugin/test_output_as_buffered.rb +1 -1
  48. data/test/plugin/test_output_as_buffered_overflow.rb +1 -1
  49. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  50. data/test/plugin_helper/test_child_process.rb +45 -56
  51. data/test/plugin_helper/test_server.rb +13 -0
  52. data/test/plugin_helper/test_timer.rb +11 -13
  53. data/test/test_config.rb +27 -5
  54. data/test/test_engine.rb +203 -0
  55. data/test/test_output.rb +2 -2
  56. data/test/test_static_config_analysis.rb +177 -0
  57. data/test/test_supervisor.rb +18 -80
  58. data/test/test_test_drivers.rb +4 -3
  59. data/test/test_variable_store.rb +65 -0
  60. metadata +40 -27
  61. data/.gitlab/cicd-template.yaml +0 -10
  62. data/Vagrantfile +0 -17
@@ -15,7 +15,6 @@
15
15
  #
16
16
 
17
17
  begin
18
- # raise if RUBY_VERSION < 2.3.x. see Gemfile
19
18
  require 'async'
20
19
  require 'fluent/plugin_helper/http_server/server'
21
20
  rescue LoadError => _
@@ -15,14 +15,6 @@
15
15
  #
16
16
 
17
17
  require 'fluent/config/error'
18
- unless {}.respond_to?(:dig)
19
- begin
20
- # backport_dig is faster than dig_rb so prefer backport_dig.
21
- require 'backport_dig'
22
- rescue LoadError
23
- require 'dig_rb'
24
- end
25
- end
26
18
 
27
19
  module Fluent
28
20
  module PluginHelper
@@ -370,10 +370,10 @@ module Fluent
370
370
  sock = if shared
371
371
  server_socket_manager_client.listen_udp(bind, port)
372
372
  else
373
- family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
374
- usock = UDPSocket.new(family)
375
- usock.bind(bind, port)
376
- usock
373
+ # UDPSocket.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.
374
+ usock = Addrinfo.udp(bind, port).bind
375
+ usock.autoclose = false
376
+ UDPSocket.for_fd(usock.fileno)
377
377
  end
378
378
  # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
379
379
  sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
@@ -693,26 +693,11 @@ module Fluent
693
693
  end
694
694
  end
695
695
 
696
- if RUBY_VERSION.to_f >= 2.3
697
- NONBLOCK_ARG = { exception: false }
698
- def try_handshake
699
- @_handler_socket.accept_nonblock(**NONBLOCK_ARG)
700
- end
701
- else
702
- def try_handshake
703
- @_handler_socket.accept_nonblock
704
- rescue IO::WaitReadable
705
- :wait_readable
706
- rescue IO::WaitWritable
707
- :wait_writable
708
- end
709
- end
710
-
711
696
  def try_tls_accept
712
697
  return true if @_handler_accepted
713
698
 
714
699
  begin
715
- result = try_handshake # this method call actually try to do handshake via TLS
700
+ result = @_handler_socket.accept_nonblock(exception: false) # this method call actually try to do handshake via TLS
716
701
  if result == :wait_readable || result == :wait_writable
717
702
  # retry accept_nonblock: there aren't enough data in underlying socket buffer
718
703
  else
@@ -726,7 +711,7 @@ module Fluent
726
711
 
727
712
  return true
728
713
  end
729
- rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e
714
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, OpenSSL::SSL::SSLError => e
730
715
  @log.trace "unexpected error before accepting TLS connection", error: e
731
716
  close rescue nil
732
717
  end
@@ -15,26 +15,29 @@
15
15
  #
16
16
 
17
17
  require 'set'
18
+ require 'fluent/variable_store'
18
19
 
19
20
  module Fluent
20
21
  module PluginId
21
- @@configured_ids = Set.new
22
22
 
23
23
  def initialize
24
24
  super
25
+
26
+ @_plugin_id_variable_store = nil
25
27
  @_plugin_root_dir = nil
26
28
  @id = nil
27
29
  end
28
30
 
29
31
  def configure(conf)
32
+ @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:pluing_id, default_value: Set.new)
30
33
  @id = conf['@id']
31
34
  @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
32
35
  if @id
33
36
  @id = @id.to_s
34
- if @@configured_ids.include?(@id) && !plugin_id_for_test?
37
+ if @_plugin_id_variable_store.include?(@id) && !plugin_id_for_test?
35
38
  raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
36
39
  end
37
- @@configured_ids.add(@id)
40
+ @_plugin_id_variable_store.add(@id)
38
41
  end
39
42
 
40
43
  super
@@ -79,7 +82,9 @@ module Fluent
79
82
  end
80
83
 
81
84
  def stop
82
- @@configured_ids.delete(@id)
85
+ if @_plugin_id_variable_store
86
+ @_plugin_id_variable_store.delete(@id)
87
+ end
83
88
 
84
89
  super
85
90
  end
@@ -0,0 +1,194 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config'
18
+ require 'fluent/plugin'
19
+
20
+ module Fluent
21
+ # Static Analysis means analysing all plugins and Fluent::Element without invokeing Plugin#configure
22
+ class StaticConfigAnalysis
23
+ module Elem
24
+ Input = Struct.new(:plugin, :config)
25
+ Output = Struct.new(:plugin, :config)
26
+ Filter = Struct.new(:plugin, :config)
27
+ Label = Struct.new(:name, :config, :nodes)
28
+ Worker = Struct.new(:ids, :config, :nodes)
29
+ end
30
+
31
+ Result = Struct.new(:tree, :outputs, :inputs, :filters, :labels) do
32
+ def all_plugins
33
+ (outputs + inputs + filters).map(&:plugin)
34
+ end
35
+ end
36
+
37
+ # @param workers [Integer] Number of workers
38
+ # @return [Fluent::StaticConfigAnalysis::Result]
39
+ def self.call(conf, workers: 1)
40
+ new(workers).call(conf)
41
+ end
42
+
43
+ def initialize(workers)
44
+ @workers = workers
45
+
46
+ reset
47
+ end
48
+
49
+ def call(config)
50
+ reset
51
+
52
+ tree = [
53
+ static_worker_analyse(config),
54
+ static_label_analyse(config),
55
+ static_filter_and_output_analyse(config),
56
+ static_input_analyse(config),
57
+ ].flatten
58
+
59
+ Result.new(tree, @outputs, @inputs, @filters, @labels.values)
60
+ end
61
+
62
+ private
63
+
64
+ def reset
65
+ @outputs = []
66
+ @inputs = []
67
+ @filters = []
68
+ @labels = {}
69
+ end
70
+
71
+ def static_worker_analyse(conf)
72
+ available_worker_ids = [*0...@workers]
73
+
74
+ ret = []
75
+ conf.elements(name: 'worker').each do |config|
76
+ ids = parse_worker_id(config)
77
+ ids.each do |id|
78
+ if available_worker_ids.include?(id)
79
+ available_worker_ids.delete(id)
80
+ else
81
+ raise Fluent::ConfigError, "specified worker_id<#{id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
82
+ end
83
+ end
84
+
85
+ config.elements.each do |elem|
86
+ unless %w[source match filter label].include?(elem.name)
87
+ raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
88
+ end
89
+ end
90
+
91
+ nodes = [
92
+ static_label_analyse(config),
93
+ static_filter_and_output_analyse(config),
94
+ static_input_analyse(config),
95
+ ].flatten
96
+ ret << Elem::Worker.new(ids, config, nodes)
97
+ end
98
+
99
+ ret
100
+ end
101
+
102
+ def parse_worker_id(conf)
103
+ worker_id_str = conf.arg
104
+
105
+ if worker_id_str.empty?
106
+ raise Fluent::ConfigError, 'Missing worker id on <worker> directive'
107
+ end
108
+
109
+ l, r =
110
+ begin
111
+ worker_id_str.split('-', 2).map { |v| Integer(v) }
112
+ rescue TypeError, ArgumentError
113
+ raise Fluent::ConfigError, "worker id should be integer: #{worker_id_str}"
114
+ end
115
+
116
+ if l < 0 || l >= @workers
117
+ raise Fluent::ConfigError, "worker id #{l} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
118
+ end
119
+
120
+ # e.g. specified one worker id like `<worker 0>`
121
+ if r.nil?
122
+ return [l]
123
+ end
124
+
125
+ if r < 0 || r >= @workers
126
+ raise Fluent::ConfigError, "worker id #{r} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
127
+ end
128
+
129
+ if l > r
130
+ raise Fluent::ConfigError, "greater first_worker_id<#{l}> than last_worker_id<#{r}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
131
+ end
132
+
133
+ [l, r]
134
+ end
135
+
136
+ def static_label_analyse(conf)
137
+ ret = []
138
+ conf.elements(name: 'label').each do |e|
139
+ name = e.arg
140
+ if name.empty?
141
+ raise ConfigError, 'Missing symbol argument on <label> directive'
142
+ end
143
+
144
+ if @labels[name]
145
+ raise ConfigError, "Section <label #{name}> appears twice"
146
+ end
147
+
148
+ l = Elem::Label.new(name, e, static_filter_and_output_analyse(e))
149
+ ret << l
150
+ @labels[name] = l
151
+ end
152
+
153
+ ret
154
+ end
155
+
156
+ def static_filter_and_output_analyse(conf)
157
+ ret = []
158
+ conf.elements('filter', 'match').each do |e|
159
+ type = e['@type']
160
+ if type.nil? || type.empty?
161
+ raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
162
+ end
163
+
164
+ if e.name == 'filter'
165
+ f = Elem::Filter.new(Fluent::Plugin.new_filter(type), e)
166
+ ret << f
167
+ @filters << f
168
+ else
169
+ o = Elem::Output.new(Fluent::Plugin.new_output(type), e)
170
+ ret << o
171
+ @outputs << o
172
+ end
173
+ end
174
+
175
+ ret
176
+ end
177
+
178
+ def static_input_analyse(conf)
179
+ ret = []
180
+ conf.elements(name: 'source').each do |e|
181
+ type = e['@type']
182
+ if type.nil? || type.empty?
183
+ raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
184
+ end
185
+
186
+ i = Elem::Input.new(Fluent::Plugin.new_input(type), e)
187
+ @inputs << i
188
+ ret << i
189
+ end
190
+
191
+ ret
192
+ end
193
+ end
194
+ end
@@ -26,6 +26,7 @@ require 'fluent/plugin'
26
26
  require 'fluent/rpc'
27
27
  require 'fluent/system_config'
28
28
  require 'fluent/msgpack_factory'
29
+ require 'fluent/variable_store'
29
30
  require 'serverengine'
30
31
 
31
32
  if Fluent.windows?
@@ -42,6 +43,7 @@ end
42
43
  module Fluent
43
44
  module ServerModule
44
45
  def before_run
46
+ @fluentd_conf = config[:fluentd_conf]
45
47
  @rpc_server = nil
46
48
  @counter = nil
47
49
 
@@ -120,6 +122,15 @@ module Fluent
120
122
  nil
121
123
  }
122
124
 
125
+ @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
126
+ $log.debug "fluentd RPC got /api/config.gracefulReload request"
127
+ unless Fluent.windows?
128
+ Process.kill :USR2, $$
129
+ end
130
+
131
+ nil
132
+ }
133
+
123
134
  @rpc_server.mount_proc('/api/config.getDump') { |req, res|
124
135
  $log.debug "fluentd RPC got /api/config.getDump request"
125
136
  $log.info "get dump in-memory config via HTTP"
@@ -156,6 +167,11 @@ module Fluent
156
167
  $log.debug "fluentd supervisor process get SIGUSR1"
157
168
  supervisor_sigusr1_handler
158
169
  end unless Fluent.windows?
170
+
171
+ trap :USR2 do
172
+ $log.debug 'fluentd supervisor process got SIGUSR2'
173
+ supervisor_sigusr2_handler
174
+ end unless Fluent.windows?
159
175
  end
160
176
 
161
177
  def install_windows_event_handler
@@ -179,20 +195,36 @@ module Fluent
179
195
  end
180
196
 
181
197
  def supervisor_sigusr1_handler
182
- if log = config[:logger_initializer]
183
- # Creating new thread due to mutex can't lock
184
- # in main thread during trap context
185
- Thread.new do
186
- log.reopen!
187
- end
188
- end
198
+ reopen_log
199
+ send_signal_to_workers(:USR1)
200
+ end
189
201
 
190
- if config[:worker_pid]
191
- config[:worker_pid].each_value do |pid|
192
- Process.kill(:USR1, pid)
193
- # don't rescue Errno::ESRCH here (invalid status)
202
+ def supervisor_sigusr2_handler
203
+ conf = nil
204
+ t = Thread.new do
205
+ $log.info 'Reloading new config'
206
+
207
+ # Validate that loading config is valid at first
208
+ conf = Fluent::Config.build(
209
+ config_path: config[:config_path],
210
+ encoding: config[:conf_encoding],
211
+ additional_config: config[:inline_config],
212
+ use_v1_config: config[:use_v1_config],
213
+ )
214
+
215
+ Fluent::VariableStore.try_to_reset do
216
+ Fluent::Engine.reload_config(conf, supervisor: true)
194
217
  end
195
218
  end
219
+
220
+ t.report_on_exception = false # Error is handled by myself
221
+ t.join
222
+
223
+ reopen_log
224
+ send_signal_to_workers(:USR2)
225
+ @fluentd_conf = conf.to_s
226
+ rescue => e
227
+ $log.error "Failed to reload config file: #{e}"
196
228
  end
197
229
 
198
230
  def kill_worker
@@ -210,11 +242,32 @@ module Fluent
210
242
  end
211
243
 
212
244
  def supervisor_dump_config_handler
213
- $log.info config[:fluentd_conf]
245
+ $log.info @fluentd_conf
214
246
  end
215
247
 
216
248
  def supervisor_get_dump_config_handler
217
- {conf: config[:fluentd_conf]}
249
+ { conf: @fluentd_conf }
250
+ end
251
+
252
+ private
253
+
254
+ def reopen_log
255
+ if (log = config[:logger_initializer])
256
+ # Creating new thread due to mutex can't lock
257
+ # in main thread during trap context
258
+ Thread.new do
259
+ log.reopen!
260
+ end
261
+ end
262
+ end
263
+
264
+ def send_signal_to_workers(signal)
265
+ return unless config[:worker_pid]
266
+
267
+ config[:worker_pid].each_value do |pid|
268
+ # don't rescue Errno::ESRCH here (invalid status)
269
+ Process.kill(signal, pid)
270
+ end
218
271
  end
219
272
  end
220
273
 
@@ -302,6 +355,9 @@ module Fluent
302
355
  JSON.dump(params)],
303
356
  command_sender: command_sender,
304
357
  fluentd_conf: params['fluentd_conf'],
358
+ conf_encoding: params['conf_encoding'],
359
+ inline_config: params['inline_config'],
360
+ config_path: path,
305
361
  main_cmd: params['main_cmd'],
306
362
  signame: params['signame'],
307
363
  }
@@ -480,7 +536,7 @@ module Fluent
480
536
 
481
537
  begin
482
538
  ServerEngine::Privilege.change(@chuser, @chgroup)
483
- MessagePackFactory.init
539
+ MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
484
540
  Fluent::Engine.init(@system_config, supervisor_mode: true)
485
541
  Fluent::Engine.run_configure(@conf, dry_run: dry_run)
486
542
  rescue Fluent::ConfigError => e
@@ -528,7 +584,7 @@ module Fluent
528
584
  main_process do
529
585
  create_socket_manager if @standalone_worker
530
586
  ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
531
- MessagePackFactory.init
587
+ MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
532
588
  Fluent::Engine.init(@system_config)
533
589
  Fluent::Engine.run_configure(@conf)
534
590
  Fluent::Engine.run
@@ -558,8 +614,7 @@ module Fluent
558
614
  $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
559
615
  @inline_config = STDIN.read
560
616
  end
561
-
562
- @conf = read_config
617
+ @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
563
618
  @system_config = build_system_config(@conf)
564
619
 
565
620
  @log.level = @system_config.log_level
@@ -674,6 +729,10 @@ module Fluent
674
729
  flush_buffer
675
730
  end unless Fluent.windows?
676
731
 
732
+ trap :USR2 do
733
+ reload_config
734
+ end unless Fluent.windows?
735
+
677
736
  if Fluent.windows?
678
737
  command_pipe = STDIN.dup
679
738
  STDIN.reopen(File::NULL, "rb")
@@ -714,6 +773,32 @@ module Fluent
714
773
  end
715
774
  end
716
775
 
776
+ def reload_config
777
+ Thread.new do
778
+ $log.debug('worker got SIGUSR2')
779
+
780
+ begin
781
+ conf = Fluent::Config.build(
782
+ config_path: @config_path,
783
+ encoding: @conf_encoding,
784
+ additional_config: @inline_config,
785
+ use_v1_config: @use_v1_config,
786
+ )
787
+
788
+ Fluent::VariableStore.try_to_reset do
789
+ Fluent::Engine.reload_config(conf)
790
+ end
791
+ rescue => e
792
+ # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
793
+ # So need this rescue code
794
+ $log.error("failed to reload config: #{e}")
795
+ next
796
+ end
797
+
798
+ @conf = conf
799
+ end
800
+ end
801
+
717
802
  def logging_with_console_output
718
803
  yield $log
719
804
  unless @log.stdout?
@@ -770,16 +855,6 @@ module Fluent
770
855
  exit!(unrecoverable_error ? 2 : 1)
771
856
  end
772
857
 
773
- def read_config
774
- config_fname = File.basename(@config_path)
775
- config_basedir = File.dirname(@config_path)
776
- config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
777
- if @inline_config
778
- config_data << "\n" << @inline_config.gsub("\\n", "\n")
779
- end
780
- Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
781
- end
782
-
783
858
  def build_system_config(conf)
784
859
  system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
785
860
  opt = {}
@@ -794,7 +869,7 @@ module Fluent
794
869
  opt[param] = @cl_opt[param]
795
870
  end
796
871
  end
797
- system_config.overwrite_variables(opt)
872
+ system_config.overwrite_variables(**opt)
798
873
  system_config
799
874
  end
800
875
  end