fluentd 1.14.4-x64-mingw-ucrt → 1.15.0-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +115 -19
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +2 -23
  10. data/fluentd.gemspec +3 -1
  11. data/lib/fluent/command/ctl.rb +4 -1
  12. data/lib/fluent/command/fluentd.rb +14 -0
  13. data/lib/fluent/config/error.rb +12 -0
  14. data/lib/fluent/config/literal_parser.rb +2 -2
  15. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  16. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  17. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  18. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  19. data/lib/fluent/config/yaml_parser.rb +56 -0
  20. data/lib/fluent/config.rb +14 -1
  21. data/lib/fluent/event_router.rb +19 -1
  22. data/lib/fluent/plugin/bare_output.rb +1 -1
  23. data/lib/fluent/plugin/base.rb +1 -1
  24. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  25. data/lib/fluent/plugin/in_forward.rb +1 -1
  26. data/lib/fluent/plugin/in_http.rb +11 -1
  27. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  28. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  29. data/lib/fluent/plugin/in_tail.rb +66 -47
  30. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  31. data/lib/fluent/plugin/output.rb +43 -33
  32. data/lib/fluent/plugin/parser.rb +3 -4
  33. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  34. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  35. data/lib/fluent/plugin_helper/server.rb +23 -4
  36. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  37. data/lib/fluent/plugin_helper/socket.rb +13 -2
  38. data/lib/fluent/registry.rb +2 -1
  39. data/lib/fluent/rpc.rb +4 -3
  40. data/lib/fluent/supervisor.rb +114 -27
  41. data/lib/fluent/system_config.rb +2 -1
  42. data/lib/fluent/version.rb +1 -1
  43. data/lib/fluent/winsvc.rb +2 -0
  44. data/test/command/test_ctl.rb +0 -1
  45. data/test/command/test_fluentd.rb +33 -0
  46. data/test/compat/test_parser.rb +1 -1
  47. data/test/config/test_system_config.rb +3 -1
  48. data/test/config/test_types.rb +1 -1
  49. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  50. data/test/plugin/in_tail/test_position_file.rb +0 -63
  51. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  52. data/test/plugin/test_file_wrapper.rb +0 -68
  53. data/test/plugin/test_filter_parser.rb +1 -1
  54. data/test/plugin/test_filter_stdout.rb +2 -2
  55. data/test/plugin/test_in_forward.rb +0 -2
  56. data/test/plugin/test_in_http.rb +23 -0
  57. data/test/plugin/test_in_object_space.rb +9 -3
  58. data/test/plugin/test_in_syslog.rb +1 -1
  59. data/test/plugin/test_in_tail.rb +629 -353
  60. data/test/plugin/test_out_forward.rb +30 -20
  61. data/test/plugin/test_out_stdout.rb +2 -2
  62. data/test/plugin/test_output_as_buffered_retries.rb +53 -6
  63. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  64. data/test/plugin/test_parser_syslog.rb +1 -1
  65. data/test/plugin_helper/test_cert_option.rb +1 -1
  66. data/test/plugin_helper/test_child_process.rb +16 -4
  67. data/test/plugin_helper/test_retry_state.rb +602 -38
  68. data/test/plugin_helper/test_server.rb +18 -0
  69. data/test/test_config.rb +135 -4
  70. data/test/test_event_router.rb +17 -0
  71. data/test/test_formatter.rb +1 -1
  72. data/test/test_supervisor.rb +196 -6
  73. metadata +39 -5
@@ -0,0 +1,204 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/input'
18
+
19
+ module Fluent::Plugin
20
+ class TailInput < Fluent::Plugin::Input
21
+ module GroupWatchParams
22
+ include Fluent::Configurable
23
+
24
+ DEFAULT_KEY = /.*/
25
+ DEFAULT_LIMIT = -1
26
+ REGEXP_JOIN = "_"
27
+
28
+ config_section :group, param_name: :group, required: false, multi: false do
29
+ desc 'Regex for extracting group\'s metadata'
30
+ config_param :pattern,
31
+ :regexp,
32
+ default: /^\/var\/log\/containers\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{64})\.log$/
33
+
34
+ desc 'Period of time in which the group_line_limit is applied'
35
+ config_param :rate_period, :time, default: 5
36
+
37
+ config_section :rule, param_name: :rule, required: true, multi: true do
38
+ desc 'Key-value pairs for grouping'
39
+ config_param :match, :hash, value_type: :regexp, default: { namespace: [DEFAULT_KEY], podname: [DEFAULT_KEY] }
40
+ desc 'Maximum number of log lines allowed per group over a period of rate_period'
41
+ config_param :limit, :integer, default: DEFAULT_LIMIT
42
+ end
43
+ end
44
+ end
45
+
46
+ module GroupWatch
47
+ def self.included(mod)
48
+ mod.include GroupWatchParams
49
+ end
50
+
51
+ attr_reader :group_watchers, :default_group_key
52
+
53
+ def initialize
54
+ super
55
+ @group_watchers = {}
56
+ @group_keys = nil
57
+ @default_group_key = nil
58
+ end
59
+
60
+ def configure(conf)
61
+ super
62
+
63
+ unless @group.nil?
64
+ ## Ensuring correct time period syntax
65
+ @group.rule.each { |rule|
66
+ raise "Metadata Group Limit >= DEFAULT_LIMIT" unless rule.limit >= GroupWatchParams::DEFAULT_LIMIT
67
+ }
68
+
69
+ @group_keys = Regexp.compile(@group.pattern).named_captures.keys
70
+ @default_group_key = ([GroupWatchParams::DEFAULT_KEY] * @group_keys.length).join(GroupWatchParams::REGEXP_JOIN)
71
+
72
+ ## Ensures that "specific" rules (with larger number of `rule.match` keys)
73
+ ## have a higher priority against "generic" rules (with less number of `rule.match` keys).
74
+ ## This will be helpful when a file satisfies more than one rule.
75
+ @group.rule.sort_by! { |rule| -rule.match.length() }
76
+ construct_groupwatchers
77
+ @group_watchers[@default_group_key] ||= GroupWatcher.new(@group.rate_period, GroupWatchParams::DEFAULT_LIMIT)
78
+ end
79
+ end
80
+
81
+ def add_path_to_group_watcher(path)
82
+ return nil if @group.nil?
83
+ group_watcher = find_group_from_metadata(path)
84
+ group_watcher.add(path) unless group_watcher.include?(path)
85
+ group_watcher
86
+ end
87
+
88
+ def remove_path_from_group_watcher(path)
89
+ return if @group.nil?
90
+ group_watcher = find_group_from_metadata(path)
91
+ group_watcher.delete(path)
92
+ end
93
+
94
+ def construct_group_key(named_captures)
95
+ match_rule = []
96
+ @group_keys.each { |key|
97
+ match_rule.append(named_captures.fetch(key, GroupWatchParams::DEFAULT_KEY))
98
+ }
99
+ match_rule = match_rule.join(GroupWatchParams::REGEXP_JOIN)
100
+
101
+ match_rule
102
+ end
103
+
104
+ def construct_groupwatchers
105
+ @group.rule.each { |rule|
106
+ match_rule = construct_group_key(rule.match)
107
+ @group_watchers[match_rule] ||= GroupWatcher.new(@group.rate_period, rule.limit)
108
+ }
109
+ end
110
+
111
+ def find_group(metadata)
112
+ metadata_key = construct_group_key(metadata)
113
+ gw_key = @group_watchers.keys.find { |regexp| metadata_key.match?(regexp) && regexp != @default_group_key }
114
+ gw_key ||= @default_group_key
115
+
116
+ @group_watchers[gw_key]
117
+ end
118
+
119
+ def find_group_from_metadata(path)
120
+ begin
121
+ metadata = @group.pattern.match(path).named_captures
122
+ group_watcher = find_group(metadata)
123
+ rescue
124
+ log.warn "Cannot find group from metadata, Adding file in the default group"
125
+ group_watcher = @group_watchers[@default_group_key]
126
+ end
127
+
128
+ group_watcher
129
+ end
130
+ end
131
+
132
+ class GroupWatcher
133
+ attr_accessor :current_paths, :limit, :number_lines_read, :start_reading_time, :rate_period
134
+
135
+ FileCounter = Struct.new(
136
+ :number_lines_read,
137
+ :start_reading_time,
138
+ )
139
+
140
+ def initialize(rate_period = 60, limit = -1)
141
+ @current_paths = {}
142
+ @rate_period = rate_period
143
+ @limit = limit
144
+ end
145
+
146
+ def add(path)
147
+ @current_paths[path] = FileCounter.new(0, nil)
148
+ end
149
+
150
+ def include?(path)
151
+ @current_paths.key?(path)
152
+ end
153
+
154
+ def size
155
+ @current_paths.size
156
+ end
157
+
158
+ def delete(path)
159
+ @current_paths.delete(path)
160
+ end
161
+
162
+ def update_reading_time(path)
163
+ @current_paths[path].start_reading_time ||= Fluent::Clock.now
164
+ end
165
+
166
+ def update_lines_read(path, value)
167
+ @current_paths[path].number_lines_read += value
168
+ end
169
+
170
+ def reset_counter(path)
171
+ @current_paths[path].start_reading_time = nil
172
+ @current_paths[path].number_lines_read = 0
173
+ end
174
+
175
+ def time_spent_reading(path)
176
+ Fluent::Clock.now - @current_paths[path].start_reading_time
177
+ end
178
+
179
+ def limit_time_period_reached?(path)
180
+ time_spent_reading(path) < @rate_period
181
+ end
182
+
183
+ def limit_lines_reached?(path)
184
+ return true unless include?(path)
185
+ return true if @limit == 0
186
+
187
+ return false if @limit < 0
188
+ return false if @current_paths[path].number_lines_read < @limit / size
189
+
190
+ # update_reading_time(path)
191
+ if limit_time_period_reached?(path) # Exceeds limit
192
+ true
193
+ else # Does not exceed limit
194
+ reset_counter(path)
195
+ false
196
+ end
197
+ end
198
+
199
+ def to_s
200
+ super + " current_paths: #{@current_paths} rate_period: #{@rate_period} limit: #{@limit}"
201
+ end
202
+ end
203
+ end
204
+ end
@@ -250,20 +250,6 @@ module Fluent::Plugin
250
250
  end
251
251
  end
252
252
 
253
- TargetInfo = Struct.new(:path, :ino) do
254
- def ==(other)
255
- return false unless other.is_a?(TargetInfo)
256
- self.path == other.path
257
- end
258
-
259
- def hash
260
- self.path.hash
261
- end
262
-
263
- def eql?(other)
264
- return false unless other.is_a?(TargetInfo)
265
- self.path == other.path
266
- end
267
- end
253
+ TargetInfo = Struct.new(:path, :ino)
268
254
  end
269
255
  end
@@ -24,6 +24,7 @@ require 'fluent/plugin/parser_multiline'
24
24
  require 'fluent/variable_store'
25
25
  require 'fluent/capability'
26
26
  require 'fluent/plugin/in_tail/position_file'
27
+ require 'fluent/plugin/in_tail/group_watch'
27
28
 
28
29
  if Fluent.windows?
29
30
  require_relative 'file_wrapper'
@@ -33,6 +34,8 @@ end
33
34
 
34
35
  module Fluent::Plugin
35
36
  class TailInput < Fluent::Plugin::Input
37
+ include GroupWatch
38
+
36
39
  Fluent::Plugin.register_input('tail', self)
37
40
 
38
41
  helpers :timer, :event_loop, :parser, :compat_parameters
@@ -354,11 +357,11 @@ module Fluent::Plugin
354
357
 
355
358
  def existence_path
356
359
  hash = {}
357
- @tails.each_key {|target_info|
360
+ @tails.each {|path, tw|
358
361
  if @follow_inodes
359
- hash[target_info.ino] = target_info
362
+ hash[tw.ino] = TargetInfo.new(tw.path, tw.ino)
360
363
  else
361
- hash[target_info.path] = target_info
364
+ hash[tw.path] = TargetInfo.new(tw.path, tw.ino)
362
365
  end
363
366
  }
364
367
  hash
@@ -406,6 +409,8 @@ module Fluent::Plugin
406
409
  event_loop_attach(watcher)
407
410
  end
408
411
 
412
+ tw.group_watcher = add_path_to_group_watcher(target_info.path)
413
+
409
414
  tw
410
415
  rescue => e
411
416
  if tw
@@ -420,36 +425,31 @@ module Fluent::Plugin
420
425
  end
421
426
 
422
427
  def construct_watcher(target_info)
428
+ path = target_info.path
429
+
430
+ # The file might be rotated or removed after collecting paths, so check inode again here.
431
+ begin
432
+ target_info.ino = Fluent::FileWrapper.stat(path).ino
433
+ rescue Errno::ENOENT, Errno::EACCES
434
+ $log.warn "stat() for #{path} failed. Continuing without tailing it."
435
+ return
436
+ end
437
+
423
438
  pe = nil
424
439
  if @pf
425
440
  pe = @pf[target_info]
426
- if @read_from_head && pe.read_inode.zero?
427
- begin
428
- pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
429
- rescue Errno::ENOENT, Errno::EACCES
430
- $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
431
- end
432
- end
441
+ pe.update(target_info.ino, 0) if @read_from_head && pe.read_inode.zero?
433
442
  end
434
443
 
435
444
  begin
436
445
  tw = setup_watcher(target_info, pe)
437
446
  rescue WatcherSetupError => e
438
- log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
447
+ log.warn "Skip #{path} because unexpected setup error happens: #{e}"
439
448
  return
440
449
  end
441
450
 
442
- begin
443
- target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
444
- @tails.delete(target_info)
445
- @tails[target_info] = tw
446
- tw.on_notify
447
- rescue Errno::ENOENT, Errno::EACCES => e
448
- $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
449
- # explicitly detach and unwatch watcher `tw`.
450
- tw.unwatched = true
451
- detach_watcher(tw, target_info.ino, false)
452
- end
451
+ @tails[path] = tw
452
+ tw.on_notify
453
453
  end
454
454
 
455
455
  def start_watchers(targets_info)
@@ -461,10 +461,12 @@ module Fluent::Plugin
461
461
 
462
462
  def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
463
463
  targets_info.each_value { |target_info|
464
+ remove_path_from_group_watcher(target_info.path)
465
+
464
466
  if remove_watcher
465
- tw = @tails.delete(target_info)
467
+ tw = @tails.delete(target_info.path)
466
468
  else
467
- tw = @tails[target_info]
469
+ tw = @tails[target_info.path]
468
470
  end
469
471
  if tw
470
472
  tw.unwatched = unwatched
@@ -478,8 +480,8 @@ module Fluent::Plugin
478
480
  end
479
481
 
480
482
  def close_watcher_handles
481
- @tails.keys.each do |target_info|
482
- tw = @tails.delete(target_info)
483
+ @tails.keys.each do |path|
484
+ tw = @tails.delete(path)
483
485
  if tw
484
486
  tw.close
485
487
  end
@@ -488,20 +490,20 @@ module Fluent::Plugin
488
490
 
489
491
  # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
490
492
  def update_watcher(target_info, pe)
491
- log.info("detected rotation of #{target_info.path}; waiting #{@rotate_wait} seconds")
493
+ path = target_info.path
494
+
495
+ log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
492
496
 
493
497
  if @pf
494
498
  pe_inode = pe.read_inode
495
- target_info_from_position_entry = TargetInfo.new(target_info.path, pe_inode)
499
+ target_info_from_position_entry = TargetInfo.new(path, pe_inode)
496
500
  unless pe_inode == @pf[target_info_from_position_entry].read_inode
497
501
  log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
498
502
  return
499
503
  end
500
504
  end
501
505
 
502
- rotated_target_info = TargetInfo.new(target_info.path, pe.read_inode)
503
- rotated_tw = @tails[rotated_target_info]
504
- new_target_info = target_info.dup
506
+ rotated_tw = @tails[path]
505
507
 
506
508
  if @follow_inodes
507
509
  new_position_entry = @pf[target_info]
@@ -509,17 +511,13 @@ module Fluent::Plugin
509
511
  if new_position_entry.read_inode == 0
510
512
  # When follow_inodes is true, it's not cleaned up by refresh_watcher.
511
513
  # So it should be unwatched here explicitly.
512
- rotated_tw.unwatched = true
513
- # Make sure to delete old key, it has a different ino while the hash key is same.
514
- @tails.delete(rotated_target_info)
515
- @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
516
- @tails[new_target_info].on_notify
514
+ rotated_tw.unwatched = true if rotated_tw
515
+ @tails[path] = setup_watcher(target_info, new_position_entry)
516
+ @tails[path].on_notify
517
517
  end
518
518
  else
519
- # Make sure to delete old key, it has a different ino while the hash key is same.
520
- @tails.delete(rotated_target_info)
521
- @tails[new_target_info] = setup_watcher(new_target_info, pe)
522
- @tails[new_target_info].on_notify
519
+ @tails[path] = setup_watcher(target_info, pe)
520
+ @tails[path].on_notify
523
521
  end
524
522
  detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
525
523
  end
@@ -542,18 +540,19 @@ module Fluent::Plugin
542
540
  end
543
541
  end
544
542
 
543
+ def throttling_is_enabled?(tw)
544
+ return true if @read_bytes_limit_per_second > 0
545
+ return true if tw.group_watcher && tw.group_watcher.limit >= 0
546
+ false
547
+ end
548
+
545
549
  def detach_watcher_after_rotate_wait(tw, ino)
546
550
  # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
547
551
  # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
548
552
  if @open_on_every_update
549
553
  # Detach now because it's already closed, waiting it doesn't make sense.
550
554
  detach_watcher(tw, ino)
551
- elsif @read_bytes_limit_per_second < 0
552
- # throttling isn't enabled, just wait @rotate_wait
553
- timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
554
- detach_watcher(tw, ino)
555
- end
556
- else
555
+ elsif throttling_is_enabled?(tw)
557
556
  # When the throttling feature is enabled, it might not reach EOF yet.
558
557
  # Should ensure to read all contents before closing it, with keeping throttling.
559
558
  start_time_to_wait = Fluent::Clock.now
@@ -564,6 +563,11 @@ module Fluent::Plugin
564
563
  detach_watcher(tw, ino)
565
564
  end
566
565
  end
566
+ else
567
+ # when the throttling feature isn't enabled, just wait @rotate_wait
568
+ timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
569
+ detach_watcher(tw, ino)
570
+ end
567
571
  end
568
572
  end
569
573
 
@@ -775,6 +779,7 @@ module Fluent::Plugin
775
779
  attr_reader :line_buffer_timer_flusher
776
780
  attr_accessor :unwatched # This is used for removing position entry from PositionFile
777
781
  attr_reader :watchers
782
+ attr_accessor :group_watcher
778
783
 
779
784
  def tag
780
785
  @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
@@ -997,6 +1002,10 @@ module Fluent::Plugin
997
1002
  @log.info "following tail of #{@path}"
998
1003
  end
999
1004
 
1005
+ def group_watcher
1006
+ @watcher.group_watcher
1007
+ end
1008
+
1000
1009
  def on_notify
1001
1010
  @notify_mutex.synchronize { handle_notify }
1002
1011
  end
@@ -1054,6 +1063,7 @@ module Fluent::Plugin
1054
1063
 
1055
1064
  def handle_notify
1056
1065
  return if limit_bytes_per_second_reached?
1066
+ return if group_watcher&.limit_lines_reached?(@path)
1057
1067
 
1058
1068
  with_io do |io|
1059
1069
  begin
@@ -1063,17 +1073,26 @@ module Fluent::Plugin
1063
1073
  begin
1064
1074
  while true
1065
1075
  @start_reading_time ||= Fluent::Clock.now
1076
+ group_watcher&.update_reading_time(@path)
1077
+
1066
1078
  data = io.readpartial(BYTES_TO_READ, @iobuf)
1067
1079
  @eof = false
1068
1080
  @number_bytes_read += data.bytesize
1069
1081
  @fifo << data
1082
+
1083
+ n_lines_before_read = @lines.size
1070
1084
  @fifo.read_lines(@lines)
1085
+ group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
1086
+
1087
+ group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
1088
+ @log.debug "Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}" if group_watcher_limit
1071
1089
 
1072
- if limit_bytes_per_second_reached? || should_shutdown_now?
1090
+ if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
1073
1091
  # Just get out from tailing loop.
1074
1092
  read_more = false
1075
1093
  break
1076
1094
  end
1095
+
1077
1096
  if @lines.size >= @read_lines_limit
1078
1097
  # not to use too much memory in case the file is very large
1079
1098
  read_more = true
@@ -50,6 +50,7 @@ module Fluent::Plugin
50
50
  def checkin(sock)
51
51
  @mutex.synchronize do
52
52
  if (s = @inflight_sockets.delete(sock))
53
+ s.timeout = timeout
53
54
  @available_sockets[s.key] << s
54
55
  else
55
56
  @log.debug("there is no socket #{sock}")
@@ -122,6 +123,7 @@ module Fluent::Plugin
122
123
  t = Time.now
123
124
  if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
124
125
  @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
126
+ s.timeout = timeout
125
127
  s
126
128
  else
127
129
  nil
@@ -235,6 +235,7 @@ module Fluent
235
235
  @dequeued_chunks_mutex = nil
236
236
  @output_enqueue_thread = nil
237
237
  @output_flush_threads = nil
238
+ @output_flush_thread_current_position = 0
238
239
 
239
240
  @simple_chunking = nil
240
241
  @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
@@ -273,7 +274,7 @@ module Fluent
273
274
  super
274
275
 
275
276
  @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
276
- @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
277
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_count", help_text: "Number of count emits")
277
278
  @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
278
279
  @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
279
280
  @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
@@ -492,6 +493,7 @@ module Fluent
492
493
  @dequeued_chunks = []
493
494
  @dequeued_chunks_mutex = Mutex.new
494
495
 
496
+ @output_flush_thread_current_position = 0
495
497
  @buffer_config.flush_thread_count.times do |i|
496
498
  thread_title = "flush_thread_#{i}".to_sym
497
499
  thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
@@ -503,7 +505,6 @@ module Fluent
503
505
  @output_flush_threads << thread_state
504
506
  end
505
507
  end
506
- @output_flush_thread_current_position = 0
507
508
 
508
509
  if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
509
510
  @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
@@ -1275,48 +1276,57 @@ module Fluent
1275
1276
 
1276
1277
  unless @retry
1277
1278
  @retry = retry_state(@buffer_config.retry_randomize)
1278
- if error
1279
- log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1280
- log.warn_backtrace error.backtrace
1279
+
1280
+ if @retry.limit?
1281
+ handle_limit_reached(error)
1282
+ elsif error
1283
+ log_retry_error(error, chunk_id_hex, using_secondary)
1281
1284
  end
1285
+
1282
1286
  return
1283
1287
  end
1284
1288
 
1285
1289
  # @retry exists
1286
1290
 
1287
- if @retry.limit?
1288
- if error
1289
- records = @buffer.queued_records
1290
- msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
1291
- log.error msg, retry_times: @retry.steps, records: records, error: error
1292
- log.error_backtrace error.backtrace
1293
- end
1294
- @buffer.clear_queue!
1295
- log.debug "buffer queue cleared"
1296
- @retry = nil
1291
+ # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
1292
+ # @retry.step is called almost as many times as the number of flush threads in a short time.
1293
+ if Time.now >= @retry.next_time
1294
+ @retry.step
1297
1295
  else
1298
- # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
1299
- # @retry.step is called almost as many times as the number of flush threads in a short time.
1300
- if Time.now >= @retry.next_time
1301
- @retry.step
1302
- else
1303
- @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
1304
- end
1305
- if error
1306
- if using_secondary
1307
- msg = "failed to flush the buffer with secondary output."
1308
- log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1309
- log.warn_backtrace error.backtrace
1310
- else
1311
- msg = "failed to flush the buffer."
1312
- log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1313
- log.warn_backtrace error.backtrace
1314
- end
1315
- end
1296
+ @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
1297
+ end
1298
+
1299
+ if @retry.limit?
1300
+ handle_limit_reached(error)
1301
+ elsif error
1302
+ log_retry_error(error, chunk_id_hex, using_secondary)
1316
1303
  end
1317
1304
  end
1318
1305
  end
1319
1306
 
1307
+ def log_retry_error(error, chunk_id_hex, using_secondary)
1308
+ return unless error
1309
+ if using_secondary
1310
+ msg = "failed to flush the buffer with secondary output."
1311
+ else
1312
+ msg = "failed to flush the buffer."
1313
+ end
1314
+ log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error)
1315
+ log.warn_backtrace(error.backtrace)
1316
+ end
1317
+
1318
+ def handle_limit_reached(error)
1319
+ if error
1320
+ records = @buffer.queued_records
1321
+ msg = "Hit limit for retries. dropping all chunks in the buffer queue."
1322
+ log.error msg, retry_times: @retry.steps, records: records, error: error
1323
+ log.error_backtrace error.backtrace
1324
+ end
1325
+ @buffer.clear_queue!
1326
+ log.debug "buffer queue cleared"
1327
+ @retry = nil
1328
+ end
1329
+
1320
1330
  def retry_state(randomize)
1321
1331
  if @secondary
1322
1332
  retry_state_create(
@@ -89,7 +89,7 @@ module Fluent
89
89
  # : format[, timezone]
90
90
 
91
91
  config_param :time_key, :string, default: nil
92
- config_param :null_value_pattern, :string, default: nil
92
+ config_param :null_value_pattern, :regexp, default: nil
93
93
  config_param :null_empty_string, :bool, default: false
94
94
  config_param :estimate_current_event, :bool, default: true
95
95
  config_param :keep_time_key, :bool, default: false
@@ -115,9 +115,8 @@ module Fluent
115
115
  super
116
116
 
117
117
  @time_parser = time_parser_create
118
- @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
119
118
  @type_converters = build_type_converters(@types)
120
- @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
119
+ @execute_convert_values = @type_converters || @null_value_pattern || @null_empty_string
121
120
  @timeout_checker = if @timeout
122
121
  class << self
123
122
  alias_method :parse_orig, :parse
@@ -220,7 +219,7 @@ module Fluent
220
219
  return time, record
221
220
  end
222
221
 
223
- def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
222
+ def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)
224
223
  null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
225
224
  end
226
225
 
@@ -484,7 +484,7 @@ module Fluent
484
484
 
485
485
  time = begin
486
486
  @time_parser_rfc5424.parse(time_str)
487
- rescue Fluent::TimeParser::TimeParseError => e
487
+ rescue Fluent::TimeParser::TimeParseError
488
488
  @time_parser_rfc5424_without_subseconds.parse(time_str)
489
489
  end
490
490
  record['time'] = time_str if @keep_time_key