fluentd 1.10.1-x86-mingw32 → 1.10.2-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f0cc7030846081961256c91d06ccf96666d771e21b9b97718b35ec176a051462
4
- data.tar.gz: 3718a462fc5dbf3b2c151e912d90faa298437ff5d657aba20cfc5210a522c990
3
+ metadata.gz: 2987fa7f12e342b3a20a9f374eac70214a7a5d4d1ba3cd471b1d0bec7eb8465d
4
+ data.tar.gz: b65963401cb734ce1a3bae1178d6add5277b563b0037a5afb4200acd3562cc50
5
5
  SHA512:
6
- metadata.gz: 81a3c1a791d4de6ff0d1b965c5fd1ac685e46cdb5b6780683a85633574c6626d4e2b93e32c51252ecc9d5fc5db662937c70521aabc0272fd75e5bea1bfb2a9fe
7
- data.tar.gz: 5c75619ed35a92696b0a3dcb425aef1d2f684c381f8870d00ac4e9e8112911b4579cffd46db5fd003078d8bc6e07db91ac405e4385aaaf2f9db43cb7b9461efd
6
+ metadata.gz: 1d5e3cef4e44db09112c559b099c095420af2b0e2de36c4d934da53ef6027f163a397e2a3ff19b70d8b3c10796377cd917d6a96f2ce701335725672c57b63da1
7
+ data.tar.gz: '0194915dc5d86d4a2d631e13fa127337fb8125a54f3689ca18851891fb0ae5f4e5e3598784c71d8ba7b5804a9e690ba977ce0dabcd9874b30633a73bd877771d'
@@ -1,5 +1,37 @@
1
1
  # v1.10
2
2
 
3
+ ## Release v1.10.2 - 2020/04/15
4
+
5
+ ### Enhancement
6
+
7
+ * out_copy: Add plugin_id to log message
8
+ https://github.com/fluent/fluentd/pull/2934
9
+ * socket: Allow cert chains in mutual auth
10
+ https://github.com/fluent/fluentd/pull/2930
11
+ * system: Add ignore_repeated_log_interval parameter
12
+ https://github.com/fluent/fluentd/pull/2937
13
+ * windows: Allow to launch fluentd from whitespace included path
14
+ https://github.com/fluent/fluentd/pull/2920
15
+ * Refactor code
16
+ https://github.com/fluent/fluentd/pull/2935
17
+ https://github.com/fluent/fluentd/pull/2936
18
+ https://github.com/fluent/fluentd/pull/2938
19
+ https://github.com/fluent/fluentd/pull/2939
20
+ https://github.com/fluent/fluentd/pull/2946
21
+
22
+ ### Bug fix
23
+
24
+ * in_syslog: Fix octet-counting mode bug
25
+ https://github.com/fluent/fluentd/pull/2942
26
+ * out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set
27
+ https://github.com/fluent/fluentd/pull/2943
28
+ * out_forward: Need authentication when sending tcp heartbeat with keepalive
29
+ https://github.com/fluent/fluentd/pull/2945
30
+ * command: Fix fluent-debug start failure
31
+ https://github.com/fluent/fluentd/pull/2948
32
+ * command: Fix regression of supervisor's worker and `--daemon` combo
33
+ https://github.com/fluent/fluentd/pull/2950
34
+
3
35
  ## Release v1.10.1 - 2020/04/02
4
36
 
5
37
  ### Enhancement
@@ -61,6 +61,7 @@ else
61
61
  end
62
62
 
63
63
  require 'fluent/log'
64
+ require 'fluent/env'
64
65
  require 'fluent/engine'
65
66
  require 'fluent/system_config'
66
67
  require 'serverengine'
@@ -336,7 +336,7 @@ else
336
336
  worker = Fluent::Supervisor.new(opts)
337
337
  worker.configure
338
338
 
339
- if opts[:daemonize]
339
+ if opts[:daemonize] && opts[:standalone_worker]
340
340
  require 'fluent/daemonizer'
341
341
  args = ARGV.dup
342
342
  i = args.index('--daemon')
@@ -112,6 +112,7 @@ module Fluent
112
112
  @optional_attrs = nil
113
113
 
114
114
  @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
115
+ @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
115
116
 
116
117
  @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
117
118
  @process_type ||= :standalone # to keep behavior of existing code
@@ -139,7 +140,8 @@ module Fluent
139
140
  dl_opts = {}
140
141
  dl_opts[:log_level] = @level - 1
141
142
  logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
142
- clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
143
+ clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
144
+ worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
143
145
  clone.format = @format
144
146
  clone.time_format = @time_format
145
147
  clone.log_event_enabled = @log_event_enabled
@@ -149,7 +151,7 @@ module Fluent
149
151
 
150
152
  attr_reader :format
151
153
  attr_reader :time_format
152
- attr_accessor :log_event_enabled
154
+ attr_accessor :log_event_enabled, :ignore_repeated_log_interval
153
155
  attr_accessor :out
154
156
  attr_accessor :level
155
157
  attr_accessor :optional_header, :optional_attrs
@@ -278,6 +280,7 @@ module Fluent
278
280
  return if skipped_type?(type)
279
281
  args << block.call if block
280
282
  time, msg = event(:trace, args)
283
+ return if time.nil?
281
284
  puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
282
285
  rescue
283
286
  # logger should not raise an exception. This rescue prevents unexpected behaviour.
@@ -299,6 +302,7 @@ module Fluent
299
302
  return if skipped_type?(type)
300
303
  args << block.call if block
301
304
  time, msg = event(:debug, args)
305
+ return if time.nil?
302
306
  puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
303
307
  rescue
304
308
  end
@@ -319,6 +323,7 @@ module Fluent
319
323
  return if skipped_type?(type)
320
324
  args << block.call if block
321
325
  time, msg = event(:info, args)
326
+ return if time.nil?
322
327
  puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
323
328
  rescue
324
329
  end
@@ -339,6 +344,7 @@ module Fluent
339
344
  return if skipped_type?(type)
340
345
  args << block.call if block
341
346
  time, msg = event(:warn, args)
347
+ return if time.nil?
342
348
  puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
343
349
  rescue
344
350
  end
@@ -359,6 +365,7 @@ module Fluent
359
365
  return if skipped_type?(type)
360
366
  args << block.call if block
361
367
  time, msg = event(:error, args)
368
+ return if time.nil?
362
369
  puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
363
370
  rescue
364
371
  end
@@ -379,6 +386,7 @@ module Fluent
379
386
  return if skipped_type?(type)
380
387
  args << block.call if block
381
388
  time, msg = event(:fatal, args)
389
+ return if time.nil?
382
390
  puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
383
391
  rescue
384
392
  end
@@ -412,6 +420,20 @@ module Fluent
412
420
  @out.reset if @out.respond_to?(:reset)
413
421
  end
414
422
 
423
+ CachedLog = Struct.new(:msg, :time)
424
+
425
+ def ignore_repeated_log?(key, time, message)
426
+ cached_log = Thread.current[key]
427
+ return false if cached_log.nil?
428
+ (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
429
+ end
430
+
431
+ def suppress_stacktrace?(backtrace)
432
+ cached_log = Thread.current[:last_repeated_stacktrace]
433
+ return false if cached_log.nil?
434
+ cached_log.msg == backtrace
435
+ end
436
+
415
437
  def dump_stacktrace(type, backtrace, level)
416
438
  return if @level > level
417
439
 
@@ -419,13 +441,16 @@ module Fluent
419
441
 
420
442
  if @format == :text
421
443
  line = caller_line(type, time, 5, level)
422
- if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
444
+ if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
445
+ return
446
+ elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
423
447
  puts [" ", line, 'suppressed same stacktrace'].join
448
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
424
449
  else
425
450
  backtrace.each { |msg|
426
451
  puts [" ", line, msg].join
427
452
  }
428
- Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
453
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
429
454
  end
430
455
  else
431
456
  r = {
@@ -436,11 +461,14 @@ module Fluent
436
461
  r['worker_id'] = wid
437
462
  end
438
463
 
439
- if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
464
+ if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
465
+ return
466
+ elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
440
467
  r['message'] = 'suppressed same stacktrace'
468
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
441
469
  else
442
470
  r['message'] = backtrace.join("\n")
443
- Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
471
+ Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
444
472
  end
445
473
 
446
474
  puts Yajl.dump(r)
@@ -479,6 +507,14 @@ module Fluent
479
507
  end
480
508
  }
481
509
 
510
+ if @ignore_repeated_log_interval
511
+ if ignore_repeated_log?(:last_repeated_log, time, message)
512
+ return nil, nil
513
+ else
514
+ Thread.current[:last_repeated_log] = CachedLog.new(message, time)
515
+ end
516
+ end
517
+
482
518
  if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
483
519
  record = map.dup
484
520
  record.keys.each {|key|
@@ -530,6 +566,9 @@ module Fluent
530
566
  if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
531
567
  @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
532
568
  end
569
+ if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
570
+ @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
571
+ end
533
572
 
534
573
  self.format = @logger.format
535
574
  self.time_format = @logger.time_format
@@ -181,12 +181,12 @@ module Fluent::Plugin
181
181
  if octet_count_frame
182
182
  while idx = buffer.index(delimiter, pos)
183
183
  num = Integer(buffer[pos..idx])
184
- pos = idx + num
185
- msg = buffer[idx + 1...pos]
186
- if msg.size < num - 1
187
- pos = pos - num - num.to_s.size
184
+ msg = buffer[idx + delimiter_size, num]
185
+ if msg.size != num
188
186
  break
189
187
  end
188
+
189
+ pos = idx + delimiter_size + num
190
190
  message_handler(msg, conn)
191
191
  end
192
192
  else
@@ -61,7 +61,7 @@ module Fluent::Plugin
61
61
  output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
62
62
  rescue => e
63
63
  if @ignore_errors[i]
64
- log.error "ignore emit error", error: e
64
+ log.error "ignore emit error in #{output.plugin_id}", error: e
65
65
  else
66
66
  raise e
67
67
  end
@@ -324,7 +324,7 @@ module Fluent::Plugin
324
324
  end
325
325
  end
326
326
 
327
- if @keepalive && @keepalive_timeout
327
+ if @keepalive
328
328
  timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))
329
329
  end
330
330
  end
@@ -578,12 +578,7 @@ module Fluent::Plugin
578
578
 
579
579
  def verify_connection
580
580
  connect do |sock, ri|
581
- if ri.state != :established
582
- establish_connection(sock, ri)
583
- if ri.state != :established
584
- raise "Failed to establish connection to #{@host}:#{@port}"
585
- end
586
- end
581
+ ensure_established_connection(sock, ri)
587
582
  end
588
583
  end
589
584
 
@@ -652,14 +647,7 @@ module Fluent::Plugin
652
647
  def send_data(tag, chunk)
653
648
  ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self)
654
649
  connect(nil, ack: ack) do |sock, ri|
655
- if ri.state != :established
656
- establish_connection(sock, ri)
657
-
658
- if ri.state != :established
659
- raise ConnectionClosedError, "failed to establish connection with node #{@name}"
660
- end
661
- end
662
-
650
+ ensure_established_connection(sock, ri)
663
651
  send_data_actual(sock, tag, chunk)
664
652
  end
665
653
 
@@ -684,7 +672,9 @@ module Fluent::Plugin
684
672
 
685
673
  case @sender.heartbeat_type
686
674
  when :transport
687
- connect(dest_addr) do |_ri, _sock|
675
+ connect(dest_addr) do |sock, ri|
676
+ ensure_established_connection(sock, ri)
677
+
688
678
  ## don't send any data to not cause a compatibility problem
689
679
  # sock.write FORWARD_TCP_HEARTBEAT_DATA
690
680
 
@@ -776,6 +766,16 @@ module Fluent::Plugin
776
766
 
777
767
  private
778
768
 
769
+ def ensure_established_connection(sock, request_info)
770
+ if request_info.state != :established
771
+ establish_connection(sock, request_info)
772
+
773
+ if request_info.state != :established
774
+ raise ConnectionClosedError, "failed to establish connection with node #{@name}"
775
+ end
776
+ end
777
+ end
778
+
779
779
  def connect(host = nil, ack: false, &block)
780
780
  @connection_manager.connect(host: host || resolved_host, port: port, hostname: @hostname, ack: ack, &block)
781
781
  end
@@ -145,9 +145,16 @@ module Fluent
145
145
  context.ciphers = ciphers
146
146
  context.verify_mode = OpenSSL::SSL::VERIFY_PEER
147
147
  context.cert_store = cert_store
148
- context.verify_hostname = true if verify_fqdn && fqdn && context.respond_to?(:verify_hostname=)
149
- context.cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) if cert_path
148
+ context.verify_hostname = verify_fqdn && fqdn
150
149
  context.key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase) if private_key_path
150
+
151
+ if cert_path
152
+ certs = socket_certificates_from_file(cert_path)
153
+ context.cert = certs.shift
154
+ unless certs.empty?
155
+ context.extra_chain_cert = certs
156
+ end
157
+ end
151
158
  end
152
159
  Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
153
160
 
@@ -186,6 +193,17 @@ module Fluent
186
193
  end
187
194
  end
188
195
 
196
+ def socket_certificates_from_file(path)
197
+ data = File.read(path)
198
+ pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
199
+ list = []
200
+ data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
201
+ if list.length == 0
202
+ log.warn "cert_path does not contain a valid certificate"
203
+ end
204
+ list
205
+ end
206
+
189
207
  def self.tls_verify_result_name(code)
190
208
  case code
191
209
  when OpenSSL::X509::V_OK then 'V_OK'
@@ -301,6 +301,7 @@ module Fluent
301
301
 
302
302
  log_level = params['log_level']
303
303
  suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
304
+ ignore_repeated_log_interval = params['ignore_repeated_log_interval']
304
305
 
305
306
  log_path = params['log_path']
306
307
  chuser = params['chuser']
@@ -308,7 +309,7 @@ module Fluent
308
309
  log_rotate_age = params['log_rotate_age']
309
310
  log_rotate_size = params['log_rotate_size']
310
311
 
311
- log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
312
+ log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
312
313
  logger_initializer = Supervisor::LoggerInitializer.new(
313
314
  log_path, log_level, chuser, chgroup, log_opts,
314
315
  log_rotate_age: log_rotate_age,
@@ -345,6 +346,7 @@ module Fluent
345
346
  chgroup: chgroup,
346
347
  chumask: 0,
347
348
  suppress_repeated_stacktrace: suppress_repeated_stacktrace,
349
+ ignore_repeated_log_interval: ignore_repeated_log_interval,
348
350
  daemonize: daemonize,
349
351
  rpc_endpoint: params['rpc_endpoint'],
350
352
  counter_server: params['counter_server'],
@@ -439,9 +441,10 @@ module Fluent
439
441
  self
440
442
  end
441
443
 
442
- def apply_options(format: nil, time_format: nil, log_dir_perm: nil)
444
+ def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
443
445
  $log.format = format if format
444
446
  $log.time_format = time_format if time_format
447
+ $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
445
448
 
446
449
  if @path && log_dir_perm
447
450
  File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -468,6 +471,7 @@ module Fluent
468
471
  root_dir: nil,
469
472
  suppress_interval: 0,
470
473
  suppress_repeated_stacktrace: true,
474
+ ignore_repeated_log_interval: nil,
471
475
  without_source: nil,
472
476
  use_v1_config: true,
473
477
  strict_config_value: nil,
@@ -507,7 +511,7 @@ module Fluent
507
511
  @cl_opt = opt
508
512
  @conf = nil
509
513
 
510
- log_opts = { suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace] }
514
+ log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
511
515
  @log = LoggerInitializer.new(
512
516
  @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
513
517
  log_rotate_age: @log_rotate_age,
@@ -628,6 +632,7 @@ module Fluent
628
632
  format: @system_config.log.format,
629
633
  time_format: @system_config.log.time_format,
630
634
  log_dir_perm: @system_config.dir_permission,
635
+ ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
631
636
  )
632
637
 
633
638
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -690,6 +695,7 @@ module Fluent
690
695
  'root_dir' => @system_config.root_dir,
691
696
  'log_level' => @system_config.log_level,
692
697
  'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
698
+ 'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
693
699
  'rpc_endpoint' => @system_config.rpc_endpoint,
694
700
  'enable_get_dump' => @system_config.enable_get_dump,
695
701
  'counter_server' => @system_config.counter_server,
@@ -882,7 +888,11 @@ module Fluent
882
888
  RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
883
889
 
884
890
  def build_spawn_command
885
- fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
891
+ if ENV['TEST_RUBY_PATH']
892
+ fluentd_spawn_cmd = [ENV['TEST_RUBY_PATH']]
893
+ else
894
+ fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
895
+ end
886
896
 
887
897
  rubyopt = ENV['RUBYOPT']
888
898
  if rubyopt
@@ -897,10 +907,9 @@ module Fluent
897
907
 
898
908
  # Adding `-h` so that it can avoid ruby's command blocking
899
909
  # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
900
- cmd = fluentd_spawn_cmd.join(' ')
901
- _, e, s = Open3.capture3("#{cmd} -h")
910
+ _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
902
911
  if s.exitstatus != 0
903
- $log.error('Invalid option is passed to RUBYOPT', command: cmd, error: e)
912
+ $log.error('Invalid option is passed to RUBYOPT', command: fluentd_spawn_cmd, error: e)
904
913
  exit s.exitstatus
905
914
  end
906
915
 
@@ -24,7 +24,7 @@ module Fluent
24
24
  SYSTEM_CONFIG_PARAMETERS = [
25
25
  :workers, :root_dir, :log_level,
26
26
  :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
27
- :log_event_verbose,
27
+ :log_event_verbose, :ignore_repeated_log_interval,
28
28
  :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
29
29
  :file_permission, :dir_permission, :counter_server, :counter_client,
30
30
  :strict_config_value, :enable_msgpack_time_support
@@ -34,6 +34,7 @@ module Fluent
34
34
  config_param :root_dir, :string, default: nil
35
35
  config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
36
36
  config_param :suppress_repeated_stacktrace, :bool, default: nil
37
+ config_param :ignore_repeated_log_interval, :time, default: nil
37
38
  config_param :emit_error_log_interval, :time, default: nil
38
39
  config_param :suppress_config_dump, :bool, default: nil
39
40
  config_param :log_event_verbose, :bool, default: nil
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.10.1'
19
+ VERSION = '1.10.2'
20
20
 
21
21
  end
@@ -16,6 +16,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
16
16
  FileUtils.mkdir_p(TMP_DIR)
17
17
  @supervisor_pid = nil
18
18
  @worker_pids = []
19
+ ENV["TEST_RUBY_PATH"] = nil
19
20
  end
20
21
 
21
22
  def process_exist?(pid)
@@ -98,7 +99,8 @@ class TestFluentdCommand < ::Test::Unit::TestCase
98
99
 
99
100
  def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
100
101
  matched = false
101
- assert_error_msg = "matched correctly"
102
+ matched_wrongly = false
103
+ assert_error_msg = ""
102
104
  stdio_buf = ""
103
105
  begin
104
106
  execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
@@ -128,13 +130,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
128
130
  end
129
131
  end
130
132
  rescue Timeout::Error
131
- assert_error_msg = "execution timeout with command out:\n" + stdio_buf
133
+ assert_error_msg = "execution timeout"
132
134
  rescue => e
133
- assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
135
+ assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
136
+ else
137
+ assert_error_msg = "log doesn't match" unless matched
134
138
  end
135
- assert matched, assert_error_msg
136
139
 
137
- unless patterns_not_match.empty?
140
+ if patterns_not_match.empty?
141
+ assert_error_msg = build_message(assert_error_msg,
142
+ "<?>\nwas expected to include:\n<?>",
143
+ stdio_buf, pattern_list)
144
+ else
138
145
  lines = stdio_buf.split("\n")
139
146
  patterns_not_match.each do |ptn|
140
147
  matched_wrongly = if ptn.is_a? Regexp
@@ -142,9 +149,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
142
149
  else
143
150
  lines.any?{|line| line.include?(ptn) }
144
151
  end
145
- assert_false matched_wrongly, "pattern exists in logs wrongly:\n" + stdio_buf
152
+ if matched_wrongly
153
+ assert_error_msg << "\n" unless assert_error_msg.empty?
154
+ assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
155
+ end
146
156
  end
157
+ assert_error_msg = build_message(assert_error_msg,
158
+ "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
159
+ stdio_buf, pattern_list, patterns_not_match)
147
160
  end
161
+
162
+ assert matched && !matched_wrongly, assert_error_msg
148
163
  end
149
164
 
150
165
  def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
@@ -842,8 +857,7 @@ CONF
842
857
  '-external-encoding' => '--external-encoding=utf-8',
843
858
  '-internal-encoding' => '--internal-encoding=utf-8',
844
859
  )
845
- test "-E option is set to RUBYOPT3" do |opt|
846
- omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
860
+ test "-E option is set to RUBYOPT" do |opt|
847
861
  conf = <<CONF
848
862
  <source>
849
863
  @type dummy
@@ -854,6 +868,7 @@ CONF
854
868
  </match>
855
869
  CONF
856
870
  conf_path = create_conf_file('rubyopt_test.conf', conf)
871
+ opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
857
872
  assert_log_matches(
858
873
  create_cmdline(conf_path),
859
874
  *opt.split(' '),
@@ -862,7 +877,7 @@ CONF
862
877
  )
863
878
  end
864
879
 
865
- test "without RUBYOPT" do
880
+ test "without RUBYOPT" do
866
881
  conf = <<CONF
867
882
  <source>
868
883
  @type dummy
@@ -877,7 +892,7 @@ CONF
877
892
  end
878
893
 
879
894
  test 'invalid values are set to RUBYOPT' do
880
- omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
895
+ omit "hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT" if Fluent.windows?
881
896
  conf = <<CONF
882
897
  <source>
883
898
  @type dummy
@@ -895,6 +910,38 @@ CONF
895
910
  )
896
911
  end
897
912
 
913
+ # https://github.com/fluent/fluentd/issues/2915
914
+ test "ruby path contains spaces" do
915
+ conf = <<CONF
916
+ <source>
917
+ @type dummy
918
+ tag dummy
919
+ </source>
920
+ <match>
921
+ @type null
922
+ </match>
923
+ CONF
924
+ ruby_path = ServerEngine.ruby_bin_path
925
+ tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
926
+ if Fluent.windows?
927
+ tmp_ruby_path << ".bat"
928
+ File.open(tmp_ruby_path, "w") do |file|
929
+ file.write "#{ruby_path} %*"
930
+ end
931
+ else
932
+ FileUtils.ln_sf(ruby_path, tmp_ruby_path)
933
+ end
934
+ ENV["TEST_RUBY_PATH"] = tmp_ruby_path
935
+ cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
936
+ conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf)
937
+ args = ["bundle", "exec", tmp_ruby_path, cmd_path, "-c", conf_path]
938
+ assert_log_matches(
939
+ args,
940
+ 'spawn command to main:',
941
+ '-Eascii-8bit:ascii-8bit'
942
+ )
943
+ end
944
+
898
945
  test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
899
946
  conf = <<CONF
900
947
  <system>
@@ -73,6 +73,7 @@ module Fluent::Config
73
73
  assert_nil(sc.root_dir)
74
74
  assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
75
75
  assert_nil(sc.suppress_repeated_stacktrace)
76
+ assert_nil(sc.ignore_repeated_log_interval)
76
77
  assert_nil(sc.emit_error_log_interval)
77
78
  assert_nil(sc.suppress_config_dump)
78
79
  assert_nil(sc.without_source)
@@ -86,6 +87,7 @@ module Fluent::Config
86
87
  'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
87
88
  'log_level' => ['log_level', 'error'],
88
89
  'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
90
+ 'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10],
89
91
  'log_event_verbose' => ['log_event_verbose', true],
90
92
  'suppress_config_dump' => ['suppress_config_dump', true],
91
93
  'without_source' => ['without_source', true],
@@ -375,7 +375,7 @@ EOS
375
375
  ]
376
376
  msgs.each { |msg|
377
377
  m = msg['msg']
378
- msg['msg'] = "#{m.size + 1} #{m}"
378
+ msg['msg'] = "#{m.size} #{m}"
379
379
  }
380
380
  msgs
381
381
  end
@@ -23,9 +23,6 @@ class TailInputTest < Test::Unit::TestCase
23
23
  def cleanup_directory(path)
24
24
  FileUtils.rm_rf(path, secure: true)
25
25
  if File.exist?(path)
26
- # ensure files are closed for Windows, on which deleted files
27
- # are still visible from filesystem
28
- GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
29
26
  FileUtils.remove_entry_secure(path, true)
30
27
  end
31
28
  FileUtils.mkdir_p(path)
@@ -1065,9 +1062,6 @@ class TailInputTest < Test::Unit::TestCase
1065
1062
  assert_equal expected_files, plugin.expand_paths.sort
1066
1063
  end
1067
1064
 
1068
- # For https://github.com/fluent/fluentd/issues/1455
1069
- # This test is fragile because test content depends on internal implementation.
1070
- # So if you modify in_tail internal, this test may break.
1071
1065
  def test_unwatched_files_should_be_removed
1072
1066
  config = config_element("", "", {
1073
1067
  "tag" => "tail",
@@ -1079,22 +1073,15 @@ class TailInputTest < Test::Unit::TestCase
1079
1073
  })
1080
1074
  d = create_driver(config, false)
1081
1075
  d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1082
- d.run(expect_emits: 1, shutdown: false, timeout: 1) do
1076
+ d.run(expect_emits: 1, shutdown: false) do
1083
1077
  File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
1084
1078
  end
1085
1079
 
1086
1080
  cleanup_directory(TMP_DIR)
1087
1081
  waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
1088
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
1082
+ waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1089
1083
 
1090
- # Previous implementation has an infinite watcher creation bug.
1091
- # Following code checks such unexpected bug by counting actual object allocation.
1092
- base_num = count_timer_object
1093
- 2.times {
1094
- sleep 1
1095
- num = count_timer_object
1096
- assert_equal base_num, num
1097
- }
1084
+ assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
1098
1085
 
1099
1086
  d.instance_shutdown
1100
1087
  end
@@ -1056,7 +1056,7 @@ EOL
1056
1056
  e = assert_raise Fluent::UnrecoverableError do
1057
1057
  d.instance_start
1058
1058
  end
1059
- assert_match(/Failed to establish connection/, e.message)
1059
+ assert_match(/failed to establish connection/, e.message)
1060
1060
  end
1061
1061
  end
1062
1062
 
@@ -1092,7 +1092,7 @@ EOL
1092
1092
  d.instance_start
1093
1093
  end
1094
1094
 
1095
- assert_match(/Failed to establish connection/, e.message)
1095
+ assert_match(/failed to establish connection/, e.message)
1096
1096
  end
1097
1097
  end
1098
1098
 
@@ -1200,6 +1200,15 @@ EOL
1200
1200
  end
1201
1201
  end
1202
1202
 
1203
+ test 'create timer of purging obsolete sockets' do
1204
+ output_conf = CONFIG + %[keepalive true]
1205
+ d = create_driver(output_conf)
1206
+
1207
+ mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
1208
+ mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
1209
+ d.instance_start
1210
+ end
1211
+
1203
1212
  sub_test_case 'with require_ack_response' do
1204
1213
  test 'Create connection per send_data' do
1205
1214
  target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
@@ -40,7 +40,7 @@ class NullOutputTest < Test::Unit::TestCase
40
40
  assert_equal [], d.instance.chunk_keys
41
41
  end
42
42
 
43
- test 'writes standard formattted chunks' do
43
+ test 'writes standard formatted chunks' do
44
44
  d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
45
45
  t = event_time("2016-05-23 00:22:13 -0800")
46
46
  d.run(default_tag: 'test', flush: true) do
@@ -895,17 +895,25 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
895
895
 
896
896
  @i.flush_thread_wakeup
897
897
  waiting(4){ Thread.pass until @i.write_count > 0 }
898
+ waiting(4) do
899
+ state = @i.instance_variable_get(:@output_flush_threads).first
900
+ state.thread.status == 'sleep'
901
+ end
898
902
 
899
- assert{ @i.write_count > 0 }
900
- assert{ @i.num_errors > 0 }
903
+ assert(@i.write_count > 0)
904
+ assert(@i.num_errors > 0)
901
905
 
902
906
  now = @i.next_flush_time
903
907
  Timecop.freeze( now )
904
908
  @i.flush_thread_wakeup
905
909
  waiting(4){ Thread.pass until @i.write_count > 1 }
910
+ waiting(4) do
911
+ state = @i.instance_variable_get(:@output_flush_threads).first
912
+ state.thread.status == 'sleep'
913
+ end
906
914
 
907
- assert{ @i.write_count > 1 }
908
- assert{ @i.num_errors > 1 }
915
+ assert(@i.write_count > 1)
916
+ assert(@i.num_errors > 1)
909
917
  end
910
918
  end
911
919
  end
@@ -0,0 +1,27 @@
1
+ -----BEGIN RSA PRIVATE KEY-----
2
+ MIIEpAIBAAKCAQEA21in8qRqBrfYlRHr1N6eGq3d4CnKe+lTt0sGbab1670s0+h1
3
+ lGtjWny6gLSnt5PBY36CUoFB8MtRciVG2zUhB+dAZe4Mst5dzJO6cONT/l/USbCV
4
+ rQ/x8m3YzeJNj89fCZ7/7po8CpEfn16BMAd6TMXZZ8O+DCgv3SliEyxpi9CLg2PK
5
+ wAV4lwktbGenYadWLFWjv5a+QElFGH/tu5jUDhggoisoQGyZCls2Vmbr6RH+uiy8
6
+ js3gxy+5YgbuwkrQtjbfwxe5yEsd6RgzO8oLqlowCwCrn4CNaXFwaZa/vqcWzY4L
7
+ qiEWLfmFpFcSZdoWjr1WHsJS/PM3AyJea8nAnQIDAQABAoIBAGkMpaqsmWbMR7rl
8
+ EVgqoffPCzMfcK01ivV+xf5f9ulG+aAndaB2aefdUojvfF+MMRNQdGPFKeqDxWbw
9
+ eWXkpQQe+ZWXk5darfubSLBl/0UVahs8qgJvX4WmnC3GUzUrsK1v68y/K0A4TrfJ
10
+ z/9LpYP9QWjTs0IpQPsfpavfGlFt1TJvPxmomn7D+n0N6zzD3kFlGdeCSYwtPURR
11
+ dJ7ifX2vU4E8UtSm7gLrXlmAl1O7bJjFvL3+QlzKDM4M1WJpA9MkmM2icKxdYpew
12
+ biRmIiUHdV4soad2Aq58v4q5UgdMqGI8f+fWPO9qgXN7AKbT9NkXr1w/pziOt5rq
13
+ /77lbKECgYEA9bmOSA1LfRRclJCaoWWu4PiaXdvMHvsNC9r8NUGuwmiosEMKHFEJ
14
+ z8wvzVXprziXAwDTwqkPJmKseNPC8WCq2GfNu7lqJ1V1wHxpzHKhiAWhs7rE7skW
15
+ mF2xn2Tocba/L3Nb+K0tXhOZ7daduz94YXFhmRtsutZ9JvFv3G+QmHkCgYEA5IS4
16
+ +cOFyha486U9jXyLcPmZXZY9ql4ILuKuQ5pHA7CBO/zU2behuwcainW6P0WzbhKs
17
+ CP8xR5W/otqcaHZxy6hg9/nUFVVZIR/0jgKyxCqJA1W49jt23gykmU6q0vj7haZJ
18
+ QMhcAbleHBC7YvYoMTpTMe7tZP4YsFNysBn76EUCgYBqtFAn07YjM7NcREsRqSE+
19
+ ylXmSisijOxGaKq6ybIE9APEvufmEf7LwKRFa3hVwaI6CKLsVhOhHJo+wd5WiR7H
20
+ aJQ7X7HMMN04YA5lXKXudluYu5MHCkWIlq8qQ1x4/N2a0mJu42ze/G4MjPTjuhUh
21
+ Y2X5YaJepAOm5JMpyzykKQKBgQDj9eaU+dBgLdSo8UD7ALAVrlio/HRdnNoq82SF
22
+ +cQ30P7KucgXvFDxQv/d+d0mu0BoYOYPP4uIbsEyE0SODQIt+LVrCmTgNzjni3op
23
+ pFVyzT/K/Nu7fsxwbEpSySAtv8UhqSVQI89sxN81vhdAfHDR0u4lVMSqx7QXSdeS
24
+ Bwm9xQKBgQDSeoHoXbmiqRBss9Em69JPwWRZOc9eLns+fFJ1x/WkhY4bnnVXTYkZ
25
+ LzRN4ICq18xHy8egbl93uxDcjkMxi3Wj64QZcygI9gWmDBW2hdMpEArR/grsM/FH
26
+ DYu9KUMhm5T1KtXtV3izSVCprthdGjbKSbmR6hGw3n9Z1uP6V1rnXQ==
27
+ -----END RSA PRIVATE KEY-----
@@ -0,0 +1,20 @@
1
+ -----BEGIN CERTIFICATE-----
2
+ MIIDPTCCAiWgAwIBAgILAJJ76Kpa6DYtsncwDQYJKoZIhvcNAQELBQAwUzELMAkG
3
+ A1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MR8w
4
+ HQYDVQQDDBZjYS50ZXN0aW5nLmZsdWVudGQub3JnMCAXDTcwMDEwMTAwMDAwMFoY
5
+ DzIxMTgxMDI3MDgwNjU3WjBTMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExFjAU
6
+ BgNVBAcMDU1vdW50YWluIFZpZXcxHzAdBgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50
7
+ ZC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDbWKfypGoGt9iV
8
+ EevU3p4ard3gKcp76VO3SwZtpvXrvSzT6HWUa2NafLqAtKe3k8FjfoJSgUHwy1Fy
9
+ JUbbNSEH50Bl7gyy3l3Mk7pw41P+X9RJsJWtD/HybdjN4k2Pz18Jnv/umjwKkR+f
10
+ XoEwB3pMxdlnw74MKC/dKWITLGmL0IuDY8rABXiXCS1sZ6dhp1YsVaO/lr5ASUUY
11
+ f+27mNQOGCCiKyhAbJkKWzZWZuvpEf66LLyOzeDHL7liBu7CStC2Nt/DF7nISx3p
12
+ GDM7yguqWjALAKufgI1pcXBplr++pxbNjguqIRYt+YWkVxJl2haOvVYewlL88zcD
13
+ Il5rycCdAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB
14
+ AGPhEAc+vb9PHyet9t5gKJTZF7S4x207hKZ0zLqCQCLM7VjrTVBh/TUFT+saiPsv
15
+ 4k2TYSZKInYXh6tkSzdGlRyzk166hF1eOTOae/92Da5PEQMT0AC+SDRDHXrNTyD8
16
+ ZxP2xOl0j+FNeKQDdNL+PCU56PYehY40dYZ/EjRyu4VjKaSCMHcVx7eMTyxU3wtB
17
+ HbOa8UJdUXEjn4h7icz4aca22aqLedXNaDr6YX+mo25t1jsTO7cEMfr4Ce7GvxdA
18
+ msey/b5jnsKcpL7/2uYz5+owD39JZQBRQt+1YAzIU+BNiz8yzdXJO4aQpGWTflIB
19
+ jO31o7x4loEvaBTQbX3iP2g=
20
+ -----END CERTIFICATE-----
@@ -0,0 +1,27 @@
1
+ -----BEGIN RSA PRIVATE KEY-----
2
+ MIIEogIBAAKCAQEAzQtGZgClcBIDKC0U1sQpznVV9HzAZANDVazrDGA8Euf8Kx4X
3
+ dnVoE+HzwAviL4VLDPzNBLyoZ2wf3wg6D+1UAhA0DqPRCihhfJqBX0WgewakSl5a
4
+ wrc1XIXtiMxcWE/5+XtodjO71gECBgmd/l0UtdFY7mMt/gArH8PQf29ABGiB1zzn
5
+ d0MNgA0r2OlXICe4kDzTHqDBv7nFgQFro2vkI/QJD5fABbb62HRWgUfQ13DUShSG
6
+ H5FPkMxTkCs5GShyjYCC5IhI+AKx5RWvb4kuMRsxWC7JHc/XKPdbeLkNWRCFzTnO
7
+ OR09Qdp45tBbXtX0IjInxsnNOBzW0zTSg0xUtwIDAQABAoIBAG4N5zNIlYOZp2gh
8
+ ClZb47SU9hXL/9euiK2rql1yKcxcB9V8yUsjqUFCvfoOZtDq0mWeKsyoFhusxU6I
9
+ s+FomPaii85vzvuMwQaIR3hDfueJoRTpn/1zKIkIuX37cnVUN+/YdTE8g01SLSvg
10
+ bZThkQQl4X3SbhUvMfZSu84qgEncdtJ08zx81CB1Fpn0HIjpmpRM1F9jjDwtkVUw
11
+ 1gVQm9g7fwbgRfqSUyKkEqKvCVYnLQx3X5inuAEps4zIzAXF8RtxFymyHfExzZqr
12
+ Pj5uYjTbf/wkaU9nWQzXeCbRTz3ZrjvFnisFt1HI2uijd1DjJ7CB7Ls752RWekXS
13
+ Yc9FFwECgYEA/gZ3Hx4dIb6NW6UmPKjiQex/tVqwCRaxGtpA7qensf187N2nn4FD
14
+ jdQJ69XmoDRKr2X+e7tsgKpETGufdDQ8MNBHz2krfqXVSlsEIOjeLfRRqhh+8Bcr
15
+ flri0sumVbk0X9bnXzyCc4AGcRhtT9U69HCMTlly2z0pjIlVvmFXARECgYEAzqNV
16
+ FSHHbEU+5zToJK0jyTaE/F+u9n5YKDcRN9RrKrEYcoUzKbIYJrUulbRGfsqAxY5L
17
+ KtIDRI9NmPDBYqBwqSFnm9LcLn8CgQH/rRSFJv1Iw+sPHd/Nrk1KWjpO0OYRgfiB
18
+ oxcgGSzHp0OZP+e7Trtq1TKW6VyadEsPZ7oKeUcCgYB9TiMkrm4gXybLtkOOWKCD
19
+ dG3qv7lmQlNKs66kCv+lxS0CirRM8i6on5flRbZmAGV28BEAaAu1zEe0isI1SC8I
20
+ xTUnEvHpn1P/QbZfpX8zm/lMtpinRkamJZ8N7Hc4ggtb2152lBqlbtm+oBYL81sJ
21
+ iRss6uLFUv5T3Mr3Bn0sgQKBgGr4xQf+h6V2J307t12dQCRfE/MueX3jpDGVaFV1
22
+ otDkAxrt97GDH9uR+f7H56KlpIohAqq1M7nfUbV2FTbAhfIYd/GD9DYhzCMK7Ngm
23
+ AlRP1MaPvjCh9nFgU7hn7PtZzwBwrHPIefZuZyEg7onVpfK5NTIPUW6XYOIJJX12
24
+ IwvrAoGAGwg2Cq6Vs/KkiCDxoy42HVZLS/DS/iHfUTAXtekF88qwT1zHFqF1ImvO
25
+ +FEUx8IbxY1oifSQc3Jw9HYnmYqmFW1PThNG3CwyaOGgtlEFcvhHsNjLRGJSQbN2
26
+ cUYBe2hHXii7OL8T8kIvLlCAKGA8IIfPGhsu3uUtrvMBo5c0WmQ=
27
+ -----END RSA PRIVATE KEY-----
@@ -0,0 +1,40 @@
1
+ -----BEGIN CERTIFICATE-----
2
+ MIIDQjCCAiqgAwIBAgILANkmymLgb5e04tcwDQYJKoZIhvcNAQELBQAwVDELMAkG
3
+ A1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MSAw
4
+ HgYDVQQDDBdjYTIudGVzdGluZy5mbHVlbnRkLm9yZzAgFw03MDAxMDEwMDAwMDBa
5
+ GA8yMTE4MTAyNzA4MDY1N1owVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYw
6
+ FAYDVQQHDA1Nb3VudGFpbiBWaWV3MSMwIQYDVQQDDBpzZXJ2ZXIudGVzdGluZy5m
7
+ bHVlbnRkLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM0LRmYA
8
+ pXASAygtFNbEKc51VfR8wGQDQ1Ws6wxgPBLn/CseF3Z1aBPh88AL4i+FSwz8zQS8
9
+ qGdsH98IOg/tVAIQNA6j0QooYXyagV9FoHsGpEpeWsK3NVyF7YjMXFhP+fl7aHYz
10
+ u9YBAgYJnf5dFLXRWO5jLf4AKx/D0H9vQARogdc853dDDYANK9jpVyAnuJA80x6g
11
+ wb+5xYEBa6Nr5CP0CQ+XwAW2+th0VoFH0Ndw1EoUhh+RT5DMU5ArORkoco2AguSI
12
+ SPgCseUVr2+JLjEbMVguyR3P1yj3W3i5DVkQhc05zjkdPUHaeObQW17V9CIyJ8bJ
13
+ zTgc1tM00oNMVLcCAwEAAaMQMA4wDAYDVR0TBAUwAwEBADANBgkqhkiG9w0BAQsF
14
+ AAOCAQEAMzoxZJqhu8zyKI20Hz5SAqva+jcHbZbR9AXlt4LWQEttsXy4S52XLZBB
15
+ 4Es/Fah4lyou9R1xdUfF6iGBV6HgDMIuh+QYfqA/jJ4HXNMKLtDXakS8xYp2A0Wn
16
+ g0EnjGx44DNweBkKGB9asldgwOM7MYMoQvS3Nkyu3oFqpfLpuWpWB/3Vw3PMGEdx
17
+ r/c8Ev5XoVtl2+dXszsEp2jwUOFo8+DZzFdBZdA2F7RmcP+zxbAHIw/jHf/ptDGg
18
+ NtzHmLhxiLblGlJHw/4a0HhXa453jzLsD1+hQpqBRUNbXBp1qpjqEr5SF1iVNGpZ
19
+ peWFawp53MkIBXRls66ViJsl//fPBg==
20
+ -----END CERTIFICATE-----
21
+ -----BEGIN CERTIFICATE-----
22
+ MIIDPTCCAiWgAwIBAgIKRbRr9ZfmrY72TjANBgkqhkiG9w0BAQsFADBTMQswCQYD
23
+ VQQGEwJVUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDU1vdW50YWluIFZpZXcxHzAd
24
+ BgNVBAMMFmNhLnRlc3RpbmcuZmx1ZW50ZC5vcmcwIBcNNzAwMTAxMDAwMDAwWhgP
25
+ MjExODEwMjcwODA2NTdaMFQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQG
26
+ A1UEBwwNTW91bnRhaW4gVmlldzEgMB4GA1UEAwwXY2EyLnRlc3RpbmcuZmx1ZW50
27
+ ZC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6gnGxD4RQuAA1
28
+ fQNMl9mctiIvcJ0Dz5f3JtCJlKKqBDKJ17py7tjFZ2MZ5vytIl1DUU2oxHew8F/u
29
+ U8s+8ljmL3seiXx2kMCOxCvSW3JYUQrzbTHKszna7Ffy94SzpRDh/4/2RGatnSqi
30
+ n4fRpalrchhH6Ds6v486o7YcwEKPR2dlD+Ltd/V3lMAOAmoRoe2aQlLudOUQjHJU
31
+ YWGim8XS2dNAEy4WR5DvjYEA+CwKEX7f9p0Dihs8FKBjB2TvBwUkdfILLEnwA1rs
32
+ r0coUt3kwYt9TYNYv6/SZYvFtaYjM3BmLVHI3mtEchaA+Tij4V4gRZ573Ai1+Fb8
33
+ rxn+oTsVAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB
34
+ AHID7VD2PS7svmCXRg4qe7MkhA0veIadZRE6WZgJo3Z64p9jUWu/Y3ZrlZuDg0qI
35
+ yV65UgWrgUOiGN6RZmI5A850DH3ryjvQElPHWjRavm0GtrOxFhHYpX8N7DRFW7t0
36
+ 7mld7YvbKRj0BrovU3avPC1P9w8FpegMKh48HsoRCO3NhHvYwtAf2B767UuAaqnI
37
+ DW9rMPRTZv+Pv+G+kJN+updOs6HiHFxeS8Bwb8GlADcwWNGMqyJJyKmaefYVzDKt
38
+ xfBiKiwev6Hm8RVTk1H3rKVZox/ZkDiZfZhmBJqO4mdDR4LLFIRIPqjUPxENsKFJ
39
+ MyrDcwH+0Aq9ZSiRaVyfCkQ=
40
+ -----END CERTIFICATE-----
@@ -7,6 +7,7 @@ end
7
7
 
8
8
  WITHOUT_CA_DIR = './without_ca'.freeze
9
9
  WITH_CA_DIR = './with_ca'.freeze
10
+ WITH_CERT_CHAIN_DIR = './cert_chains'.freeze
10
11
 
11
12
  CA_OPTION = {
12
13
  private_key_length: 2048,
@@ -83,5 +84,42 @@ def create_with_ca
83
84
  create_server_pair_signed_by_ca(ca_cert_pass_path, ca_key_pass_path, 'orange', cert_pass_path, cert_key_pass_path, 'apple')
84
85
  end
85
86
 
87
+ def create_cert_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
88
+ root_cert, root_key, _ = CertUtil.cert_option_generate_ca_pair_self_signed(CA_OPTION)
89
+ write_cert_and_key(ca_cert_path, root_cert, ca_key_path, root_key, ca_key_passphrase)
90
+
91
+ intermediate_ca_options = CA_OPTION.dup
92
+ intermediate_ca_options[:common_name] = 'ca2.testing.fluentd.org'
93
+ chain_cert, chain_key = CertUtil.cert_option_generate_pair(intermediate_ca_options, root_cert.subject)
94
+ chain_cert.add_extension(OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)])))
95
+ chain_cert.sign(root_key, 'sha256')
96
+
97
+ cert, server_key, _ = CertUtil.cert_option_generate_pair(SERVER_OPTION, chain_cert.subject)
98
+ cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
99
+ cert.sign(chain_key, 'sha256')
100
+
101
+ # write chained cert
102
+ File.open(cert_path, 'w') do |f|
103
+ f.write(cert.to_pem)
104
+ f.write(chain_cert.to_pem)
105
+ end
106
+
107
+ key_str = passphrase ? server_key.export(OpenSSL::Cipher.new("AES-256-CBC"), passphrase) : server_key.export
108
+ File.open(private_key_path, "w") { |f| f.write(key_str) }
109
+ File.chmod(0600, cert_path, private_key_path)
110
+ end
111
+
112
+ def create_cert_chain
113
+ FileUtils.mkdir_p(WITH_CERT_CHAIN_DIR)
114
+ ca_cert_path = File.join(WITH_CERT_CHAIN_DIR, 'ca-cert.pem')
115
+ ca_key_path = File.join(WITH_CERT_CHAIN_DIR, 'ca-cert-key.pem')
116
+
117
+ cert_path = File.join(WITH_CERT_CHAIN_DIR, 'cert.pem')
118
+ private_key_path = File.join(WITH_CERT_CHAIN_DIR, 'cert-key.pem')
119
+
120
+ create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, nil, cert_path, private_key_path, nil)
121
+ end
122
+
86
123
  create_without_ca
87
124
  create_with_ca
125
+ create_cert_chain
@@ -9,7 +9,7 @@ rescue LoadError => _
9
9
  end
10
10
 
11
11
  unless skip
12
- class HtttpHelperAppTest < Test::Unit::TestCase
12
+ class HttpHelperAppTest < Test::Unit::TestCase
13
13
  NULL_LOGGER = Logger.new(nil)
14
14
 
15
15
  class DummyRounter
@@ -9,7 +9,7 @@ rescue LoadError => _
9
9
  end
10
10
 
11
11
  unless skip
12
- class HtttpHelperRouterTest < Test::Unit::TestCase
12
+ class HttpHelperRouterTest < Test::Unit::TestCase
13
13
  sub_test_case '#mount' do
14
14
  test 'mount with method and path' do
15
15
  router = Fluent::PluginHelper::HttpServer::Router.new
@@ -8,7 +8,7 @@ require 'uri'
8
8
  require 'openssl'
9
9
  require 'async'
10
10
 
11
- class HtttpHelperTest < Test::Unit::TestCase
11
+ class HttpHelperTest < Test::Unit::TestCase
12
12
  PORT = unused_port
13
13
  NULL_LOGGER = Logger.new(nil)
14
14
  CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
@@ -110,7 +110,7 @@ class HtttpHelperTest < Test::Unit::TestCase
110
110
  end
111
111
 
112
112
  context.cert_store = cert_store
113
- if !hostname && context.respond_to?(:verify_hostname=)
113
+ if !hostname
114
114
  context.verify_hostname = false # In test code, using hostname to be connected is very difficult
115
115
  end
116
116
 
@@ -860,7 +860,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
860
860
  end
861
861
  context.verify_mode = OpenSSL::SSL::VERIFY_PEER
862
862
  context.cert_store = cert_store
863
- if !hostname && context.respond_to?(:verify_hostname=)
863
+ if !hostname
864
864
  context.verify_hostname = false # In test code, using hostname to be connected is very difficult
865
865
  end
866
866
  else
@@ -0,0 +1,131 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/socket'
3
+ require 'fluent/plugin/base'
4
+
5
+ require 'socket'
6
+ require 'openssl'
7
+
8
+ class SocketHelperTest < Test::Unit::TestCase
9
+ PORT = unused_port
10
+ CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
11
+ CA_CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
12
+ CERT_CHAINS_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/cert_chains')
13
+
14
+ class SocketHelperTestPlugin < Fluent::Plugin::TestBase
15
+ helpers :socket
16
+ end
17
+
18
+ class EchoTLSServer
19
+ def initialize(host = '127.0.0.1', port = PORT, cert_path: nil, private_key_path: nil, ca_path: nil)
20
+ server = TCPServer.open(host, port)
21
+ ctx = OpenSSL::SSL::SSLContext.new
22
+ ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert_path)) if cert_path
23
+
24
+ cert_store = OpenSSL::X509::Store.new
25
+ cert_store.set_default_paths
26
+ cert_store.add_file(ca_path) if ca_path
27
+ ctx.cert_store = cert_store
28
+
29
+ ctx.key = OpenSSL::PKey::RSA.new(File.open(private_key_path)) if private_key_path
30
+ ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
31
+ ctx.verify_hostname = false
32
+
33
+ @server = OpenSSL::SSL::SSLServer.new(server, ctx)
34
+ @thread = nil
35
+ @r, @w = IO.pipe
36
+ end
37
+
38
+ def start
39
+ do_start
40
+
41
+ if block_given?
42
+ begin
43
+ yield
44
+ @thread.join(5)
45
+ ensure
46
+ stop
47
+ end
48
+ end
49
+ end
50
+
51
+ def stop
52
+ unless @w.closed?
53
+ @w.write('stop')
54
+ end
55
+
56
+ [@server, @w, @r].each do |s|
57
+ next if s.closed?
58
+ s.close
59
+ end
60
+
61
+ @thread.join(5)
62
+ end
63
+
64
+ private
65
+
66
+ def do_start
67
+ @thread = Thread.new(@server) do |s|
68
+ socks, _, _ = IO.select([s.accept, @r], nil, nil)
69
+
70
+ if socks.include?(@r)
71
+ break
72
+ end
73
+
74
+ sock = socks.first
75
+ buf = +''
76
+ loop do
77
+ b = sock.read_nonblock(1024, nil, exception: false)
78
+ if b == :wait_readable || b.nil?
79
+ break
80
+ end
81
+ buf << b
82
+ end
83
+
84
+ sock.write(buf)
85
+ sock.close
86
+ end
87
+ end
88
+ end
89
+
90
+ test 'with self-signed cert/key pair' do
91
+ cert_path = File.join(CERT_DIR, 'cert.pem')
92
+ private_key_path = File.join(CERT_DIR, 'cert-key.pem')
93
+
94
+ EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
95
+ client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [cert_path])
96
+ client.write('hello')
97
+ assert_equal 'hello', client.readpartial(100)
98
+ client.close
99
+ end
100
+ end
101
+
102
+ test 'with cert/key signed by self-signed CA' do
103
+ cert_path = File.join(CA_CERT_DIR, 'cert.pem')
104
+ private_key_path = File.join(CA_CERT_DIR, 'cert-key.pem')
105
+
106
+ ca_cert_path = File.join(CA_CERT_DIR, 'ca-cert.pem')
107
+
108
+ EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
109
+ client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [ca_cert_path])
110
+ client.write('hello')
111
+ assert_equal 'hello', client.readpartial(100)
112
+ client.close
113
+ end
114
+ end
115
+
116
+ test 'with cert/key signed by self-signed CA in server and client cert chain' do
117
+ cert_path = File.join(CERT_DIR, 'cert.pem')
118
+ private_key_path = File.join(CERT_DIR, 'cert-key.pem')
119
+
120
+ client_ca_cert_path = File.join(CERT_CHAINS_DIR, 'ca-cert.pem')
121
+ client_cert_path = File.join(CERT_CHAINS_DIR, 'cert.pem')
122
+ client_private_key_path = File.join(CERT_CHAINS_DIR, 'cert-key.pem')
123
+
124
+ EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
125
+ client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
126
+ client.write('hello')
127
+ assert_equal 'hello', client.readpartial(100)
128
+ client.close
129
+ end
130
+ end
131
+ end
@@ -367,6 +367,50 @@ class LogTest < Test::Unit::TestCase
367
367
  end
368
368
  end
369
369
 
370
+ sub_test_case "ignore_repeated_log_interval" do
371
+ def test_same_message
372
+ message = "This is test"
373
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
374
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 5})
375
+
376
+ log.error message
377
+ 10.times { |i|
378
+ Timecop.freeze(@timestamp + i)
379
+ log.error message
380
+ }
381
+
382
+ expected = [
383
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
384
+ "2016-04-21 02:58:47 +0000 [error]: This is test\n"
385
+ ]
386
+ assert_equal(expected, log.out.logs)
387
+ end
388
+
389
+ def test_different_message
390
+ message = "This is test"
391
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
392
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 10})
393
+
394
+ log.error message
395
+ 3.times { |i|
396
+ Timecop.freeze(@timestamp + i)
397
+ log.error message
398
+ log.error message
399
+ log.info "Hello! " + message
400
+ }
401
+
402
+ expected = [
403
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
404
+ "2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
405
+ "2016-04-21 02:58:42 +0000 [error]: This is test\n",
406
+ "2016-04-21 02:58:42 +0000 [info]: Hello! This is test\n",
407
+ "2016-04-21 02:58:43 +0000 [error]: This is test\n",
408
+ "2016-04-21 02:58:43 +0000 [info]: Hello! This is test\n",
409
+ ]
410
+ assert_equal(expected, log.out.logs)
411
+ end
412
+ end
413
+
370
414
  def test_dup
371
415
  dl_opts = {}
372
416
  dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.1
4
+ version: 1.10.2
5
5
  platform: x86-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-04-02 00:00:00.000000000 Z
11
+ date: 2020-04-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -873,6 +873,10 @@ files:
873
873
  - test/plugin_helper/data/cert/cert-with-CRLF.pem
874
874
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
875
875
  - test/plugin_helper/data/cert/cert.pem
876
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
877
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
878
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
879
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
876
880
  - test/plugin_helper/data/cert/generate_cert.rb
877
881
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
878
882
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -904,6 +908,7 @@ files:
904
908
  - test/plugin_helper/test_retry_state.rb
905
909
  - test/plugin_helper/test_server.rb
906
910
  - test/plugin_helper/test_service_discovery.rb
911
+ - test/plugin_helper/test_socket.rb
907
912
  - test/plugin_helper/test_storage.rb
908
913
  - test/plugin_helper/test_thread.rb
909
914
  - test/plugin_helper/test_timer.rb
@@ -1101,6 +1106,10 @@ test_files:
1101
1106
  - test/plugin_helper/data/cert/cert-with-CRLF.pem
1102
1107
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
1103
1108
  - test/plugin_helper/data/cert/cert.pem
1109
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
1110
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
1111
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
1112
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
1104
1113
  - test/plugin_helper/data/cert/generate_cert.rb
1105
1114
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
1106
1115
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -1132,6 +1141,7 @@ test_files:
1132
1141
  - test/plugin_helper/test_retry_state.rb
1133
1142
  - test/plugin_helper/test_server.rb
1134
1143
  - test/plugin_helper/test_service_discovery.rb
1144
+ - test/plugin_helper/test_socket.rb
1135
1145
  - test/plugin_helper/test_storage.rb
1136
1146
  - test/plugin_helper/test_thread.rb
1137
1147
  - test/plugin_helper/test_timer.rb