fluentd 1.12.4-x86-mingw32 → 1.13.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

@@ -227,31 +227,14 @@ module Fluent::Plugin
227
227
  socket_cache: socket_cache,
228
228
  )
229
229
 
230
- configs = []
231
-
232
- # rewrite for using server as sd_static
233
- conf.elements(name: 'server').each do |s|
234
- s.name = 'service'
235
- end
236
-
237
- unless conf.elements(name: 'service').empty?
238
- # To copy `services` element only
239
- new_elem = Fluent::Config::Element.new('static_service_discovery', {}, {}, conf.elements(name: 'service'))
240
- configs << { type: :static, conf: new_elem }
241
- end
242
-
243
- conf.elements(name: 'service_discovery').each_with_index do |c, i|
244
- configs << { type: @service_discovery[i][:@type], conf: c }
245
- end
246
-
247
- service_discovery_create_manager(
230
+ service_discovery_configure(
248
231
  :out_forward_service_discovery_watcher,
249
- configurations: configs,
232
+ static_default_service_directive: 'server',
250
233
  load_balancer: LoadBalancer.new(log),
251
234
  custom_build_method: method(:build_node),
252
235
  )
253
236
 
254
- discovery_manager.services.each do |server|
237
+ service_discovery_services.each do |server|
255
238
  # it's only for test
256
239
  @nodes << server
257
240
  unless @heartbeat_type == :none
@@ -273,7 +256,7 @@ module Fluent::Plugin
273
256
  end
274
257
  end
275
258
 
276
- if discovery_manager.services.empty?
259
+ if service_discovery_services.empty?
277
260
  raise Fluent::ConfigError, "forward output plugin requires at least one node is required. Add <server> or <service_discovery>"
278
261
  end
279
262
 
@@ -306,7 +289,7 @@ module Fluent::Plugin
306
289
 
307
290
  unless @heartbeat_type == :none
308
291
  if @heartbeat_type == :udp
309
- @usock = socket_create_udp(discovery_manager.services.first.host, discovery_manager.services.first.port, nonblock: true)
292
+ @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)
310
293
  server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
311
294
  end
312
295
  timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
@@ -318,7 +301,7 @@ module Fluent::Plugin
318
301
  end
319
302
 
320
303
  if @verify_connection_at_startup
321
- discovery_manager.services.each do |node|
304
+ service_discovery_services.each do |node|
322
305
  begin
323
306
  node.verify_connection
324
307
  rescue StandardError => e
@@ -374,7 +357,7 @@ module Fluent::Plugin
374
357
  return if chunk.empty?
375
358
  tag = chunk.metadata.tag
376
359
 
377
- discovery_manager.select_service { |node| node.send_data(tag, chunk) }
360
+ service_discovery_select_service { |node| node.send_data(tag, chunk) }
378
361
  end
379
362
 
380
363
  def try_write(chunk)
@@ -384,7 +367,7 @@ module Fluent::Plugin
384
367
  return
385
368
  end
386
369
  tag = chunk.metadata.tag
387
- discovery_manager.select_service { |node| node.send_data(tag, chunk) }
370
+ service_discovery_select_service { |node| node.send_data(tag, chunk) }
388
371
  last_ack if @require_ack_response && @suspend_flush
389
372
  end
390
373
 
@@ -434,7 +417,7 @@ module Fluent::Plugin
434
417
 
435
418
  def statistics
436
419
  stats = super
437
- services = discovery_manager.services
420
+ services = service_discovery_services
438
421
  healthy_nodes_count = 0
439
422
  registed_nodes_count = services.size
440
423
  services.each do |s|
@@ -471,7 +454,7 @@ module Fluent::Plugin
471
454
 
472
455
  def on_heartbeat_timer
473
456
  need_rebuild = false
474
- discovery_manager.services.each do |n|
457
+ service_discovery_services.each do |n|
475
458
  begin
476
459
  log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
477
460
  n.usock = @usock if @usock
@@ -486,16 +469,16 @@ module Fluent::Plugin
486
469
  end
487
470
 
488
471
  if need_rebuild
489
- discovery_manager.rebalance
472
+ service_discovery_rebalance
490
473
  end
491
474
  end
492
475
 
493
476
  def on_udp_heatbeat_response_recv(data, sock)
494
477
  sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
495
- if node = discovery_manager.services.find { |n| n.sockaddr == sockaddr }
478
+ if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
496
479
  # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
497
480
  if node.heartbeat
498
- discovery_manager.rebalance
481
+ service_discovery_rebalance
499
482
  end
500
483
  else
501
484
  log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out")
@@ -22,10 +22,19 @@ module Fluent
22
22
  module ServiceDiscovery
23
23
  include Fluent::PluginHelper::Timer
24
24
 
25
+ # For the compatibility with older versions without `param_name: :service_discovery_configs`
26
+ attr_reader :service_discovery
27
+
25
28
  def self.included(mod)
26
29
  mod.include ServiceDiscoveryParams
27
30
  end
28
31
 
32
+ def configure(conf)
33
+ super
34
+ # For the compatibility with older versions without `param_name: :service_discovery_configs`
35
+ @service_discovery = @service_discovery_configs
36
+ end
37
+
29
38
  def start
30
39
  unless @discovery_manager
31
40
  log.warn('There is no discovery_manager. skip start them')
@@ -52,6 +61,35 @@ module Fluent
52
61
 
53
62
  private
54
63
 
64
+ # @param title [Symbol] the thread name. this value should be unique.
65
+ # @param static_default_service_directive [String] the directive name of each service when "static" service discovery is enabled in default
66
+ # @param load_balancer [Object] object which has two methods #rebalance and #select_service
67
+ # @param custom_build_method [Proc]
68
+ def service_discovery_configure(title, static_default_service_directive: nil, load_balancer: nil, custom_build_method: nil, interval: 3)
69
+ configs = @service_discovery_configs.map(&:corresponding_config_element)
70
+ if static_default_service_directive
71
+ configs.prepend Fluent::Config::Element.new(
72
+ 'service_discovery',
73
+ '',
74
+ {'@type' => 'static'},
75
+ @config.elements(name: static_default_service_directive.to_s).map{|e| Fluent::Config::Element.new('service', e.arg, e.dup, e.elements, e.unused) }
76
+ )
77
+ end
78
+ service_discovery_create_manager(title, configurations: configs, load_balancer: load_balancer, custom_build_method: custom_build_method, interval: interval)
79
+ end
80
+
81
+ def service_discovery_select_service(&block)
82
+ @discovery_manager.select_service(&block)
83
+ end
84
+
85
+ def service_discovery_services
86
+ @discovery_manager.services
87
+ end
88
+
89
+ def service_discovery_rebalance
90
+ @discovery_manager.rebalance
91
+ end
92
+
55
93
  # @param title [Symbol] the thread name. this value should be unique.
56
94
  # @param configurations [Hash] hash which must has discivery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
57
95
  # @param load_balancer [Object] object which has two methods #rebalance and #select_service
@@ -78,7 +116,7 @@ module Fluent
78
116
  module ServiceDiscoveryParams
79
117
  include Fluent::Configurable
80
118
 
81
- config_section :service_discovery do
119
+ config_section :service_discovery, multi: true, param_name: :service_discovery_configs do
82
120
  config_param :@type, :string
83
121
  end
84
122
  end
@@ -32,17 +32,23 @@ module Fluent
32
32
  @static_config = true
33
33
  end
34
34
 
35
- def configure(opts, parent: nil)
36
- opts.each do |opt|
37
- sd = Fluent::Plugin.new_sd(opt[:type], parent: parent)
38
- sd.configure(opt[:conf])
35
+ def configure(configs, parent: nil)
36
+ configs.each do |config|
37
+ type, conf = if config.has_key?(:conf) # for compatibility with initial API
38
+ [config[:type], config[:conf]]
39
+ else
40
+ [config['@type'], config]
41
+ end
42
+
43
+ sd = Fluent::Plugin.new_sd(type, parent: parent)
44
+ sd.configure(conf)
39
45
 
40
46
  sd.services.each do |s|
41
47
  @services[s.discovery_id] = build_service(s)
42
48
  end
43
49
  @discoveries << sd
44
50
 
45
- if @static_config && opt[:type] != :static
51
+ if @static_config && type.to_sym != :static
46
52
  @static_config = false
47
53
  end
48
54
  end
@@ -518,6 +518,8 @@ module Fluent
518
518
  dl_opts = {}
519
519
  # subtract 1 to match serverengine daemon logger side logging severity.
520
520
  dl_opts[:log_level] = @level - 1
521
+ dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
522
+ dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
521
523
  logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
522
524
  $log = Fluent::Log.new(logger, @opts)
523
525
  $log.enable_color(false) if @path
@@ -606,6 +608,19 @@ module Fluent
606
608
 
607
609
  @cl_opt = opt
608
610
  @conf = nil
611
+ # parse configuration immediately to initialize logger in early stage
612
+ if @config_path and File.exist?(@config_path)
613
+ @conf = Fluent::Config.build(config_path: @config_path,
614
+ encoding: @conf_encoding ? @conf_encoding : 'utf-8',
615
+ additional_config: @inline_config ? @inline_config : nil,
616
+ use_v1_config: !!@use_v1_config)
617
+ @system_config = build_system_config(@conf)
618
+ if @system_config.log
619
+ @log_rotate_age ||= @system_config.log.rotate_age
620
+ @log_rotate_size ||= @system_config.log.rotate_size
621
+ end
622
+ @conf = nil
623
+ end
609
624
 
610
625
  log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
611
626
  ignore_same_log_interval: opt[:ignore_same_log_interval]}
@@ -55,6 +55,20 @@ module Fluent
55
55
  config_section :log, required: false, init: true, multi: false do
56
56
  config_param :format, :enum, list: [:text, :json], default: :text
57
57
  config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
58
+ config_param :rotate_age, default: nil do |v|
59
+ if Fluent::Log::LOG_ROTATE_AGE.include?(v)
60
+ v.to_sym
61
+ else
62
+ begin
63
+ Integer(v)
64
+ rescue ArgumentError => e
65
+ raise Fluent::ConfigError, e.message
66
+ else
67
+ v.to_i
68
+ end
69
+ end
70
+ end
71
+ config_param :rotate_size, :size, default: nil
58
72
  end
59
73
 
60
74
  config_section :counter_server, multi: false do
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.12.4'
19
+ VERSION = '1.13.0'
20
20
 
21
21
  end
@@ -0,0 +1,96 @@
1
+ require_relative '../helper'
2
+
3
+ require 'test-unit'
4
+ require 'open3'
5
+ require 'fluent/plugin/output'
6
+ require 'fluent/plugin/in_forward'
7
+ require 'fluent/plugin/out_secondary_file'
8
+ require 'fluent/test/driver/output'
9
+ require 'fluent/test/driver/input'
10
+
11
+ class TestFluentCat < ::Test::Unit::TestCase
12
+ def setup
13
+ Fluent::Test.setup
14
+ FileUtils.mkdir_p(TMP_DIR)
15
+ @record = { 'key' => 'value' }
16
+ @time = event_time
17
+ @es = Fluent::OneEventStream.new(@time, @record)
18
+ @primary = create_primary
19
+ metadata = @primary.buffer.new_metadata
20
+ @chunk = create_chunk(@primary, metadata, @es)
21
+ end
22
+
23
+ def teardown
24
+ FileUtils.rm_rf(TMP_DIR)
25
+ end
26
+
27
+ TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}")
28
+ FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + "/../../bin/fluent-cat")
29
+
30
+ PORT = unused_port
31
+ CONFIG = %[
32
+ port #{PORT}
33
+ bind 127.0.0.1
34
+ ]
35
+
36
+ SECONDARY_CONFIG = %[
37
+ directory #{TMP_DIR}
38
+ ]
39
+
40
+ class DummyOutput < Fluent::Plugin::Output
41
+ def write(chunk); end
42
+ end
43
+
44
+ def create_driver(conf=CONFIG)
45
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
46
+ end
47
+
48
+ def create_primary(buffer_cofig = config_element('buffer'))
49
+ DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_cofig]))
50
+ end
51
+
52
+ def create_secondary_driver(conf=SECONDARY_CONFIG)
53
+ c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
54
+ c.instance.acts_as_secondary(@primary)
55
+ c.configure(conf)
56
+ end
57
+
58
+ def create_chunk(primary, metadata, es)
59
+ primary.buffer.generate_chunk(metadata).tap do |c|
60
+ c.concat(es.to_msgpack_stream, es.size)
61
+ c.commit
62
+ end
63
+ end
64
+
65
+ sub_test_case "json" do
66
+ def test_cat_json
67
+ d = create_driver
68
+ d.run(expect_records: 1) do
69
+ Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} json") do |stdin|
70
+ stdin.puts('{"key":"value"}')
71
+ stdin.close
72
+ end
73
+ end
74
+ event = d.events.first
75
+ assert_equal([1, "json", @record],
76
+ [d.events.size, event.first, event.last])
77
+ end
78
+ end
79
+
80
+ sub_test_case "msgpack" do
81
+ def test_cat_secondary_file
82
+ d = create_secondary_driver
83
+ path = d.instance.write(@chunk)
84
+ d = create_driver
85
+ d.run(expect_records: 1) do
86
+ Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{PORT} --format msgpack secondary") do |stdin|
87
+ stdin.write(File.read(path))
88
+ stdin.close
89
+ end
90
+ end
91
+ event = d.events.first
92
+ assert_equal([1, "secondary", @record],
93
+ [d.events.size, event.first, event.last])
94
+ end
95
+ end
96
+ end
@@ -143,5 +143,51 @@ module Fluent::Config
143
143
  sc.overwrite_variables(**s.for_system_config)
144
144
  assert_equal(level, sc.log_level)
145
145
  end
146
+
147
+ sub_test_case "log rotation" do
148
+ data('daily' => "daily",
149
+ 'weekly' => 'weekly',
150
+ 'monthly' => 'monthly')
151
+ test "symbols for rotate_age" do |age|
152
+ conf = parse_text(<<-EOS)
153
+ <system>
154
+ <log>
155
+ rotate_age #{age}
156
+ </log>
157
+ </system>
158
+ EOS
159
+ sc = Fluent::SystemConfig.new(conf)
160
+ assert_equal(age.to_sym, sc.log.rotate_age)
161
+ end
162
+
163
+ test "numeric number for rotate age" do
164
+ conf = parse_text(<<-EOS)
165
+ <system>
166
+ <log>
167
+ rotate_age 3
168
+ </log>
169
+ </system>
170
+ EOS
171
+ s = FakeSupervisor.new
172
+ sc = Fluent::SystemConfig.new(conf)
173
+ assert_equal(3, sc.log.rotate_age)
174
+ end
175
+
176
+ data(h: ['100', 100],
177
+ k: ['1k', 1024],
178
+ m: ['1m', 1024 * 1024],
179
+ g: ['1g', 1024 * 1024 * 1024])
180
+ test "numeric and SI prefix for rotate_size" do |(label, size)|
181
+ conf = parse_text(<<-EOS)
182
+ <system>
183
+ <log>
184
+ rotate_size #{label}
185
+ </log>
186
+ </system>
187
+ EOS
188
+ sc = Fluent::SystemConfig.new(conf)
189
+ assert_equal(size, sc.log.rotate_size)
190
+ end
191
+ end
146
192
  end
147
193
  end
@@ -30,7 +30,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
30
30
  end
31
31
 
32
32
  returned_lines = ''
33
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
33
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
34
34
  returned_lines << lines.join
35
35
  true
36
36
  end
@@ -62,7 +62,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
62
62
  end
63
63
 
64
64
  returned_lines = ''
65
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
65
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
66
66
  returned_lines << lines.join
67
67
  true
68
68
  end
@@ -93,7 +93,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
93
93
  end
94
94
 
95
95
  returned_lines = []
96
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
96
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
97
97
  returned_lines << lines.dup
98
98
  true
99
99
  end
@@ -119,7 +119,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
119
119
  end
120
120
 
121
121
  returned_lines = []
122
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
122
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
123
123
  returned_lines << lines.dup
124
124
  true
125
125
  end