fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +151 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/command/ctl.rb +2 -2
  15. data/lib/fluent/command/fluentd.rb +55 -53
  16. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  17. data/lib/fluent/config/dsl.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +2 -2
  19. data/lib/fluent/counter/server.rb +1 -1
  20. data/lib/fluent/counter/validator.rb +3 -3
  21. data/lib/fluent/daemon.rb +2 -4
  22. data/lib/fluent/engine.rb +1 -1
  23. data/lib/fluent/event.rb +8 -4
  24. data/lib/fluent/log/console_adapter.rb +66 -0
  25. data/lib/fluent/log.rb +44 -5
  26. data/lib/fluent/match.rb +1 -1
  27. data/lib/fluent/msgpack_factory.rb +6 -1
  28. data/lib/fluent/plugin/base.rb +6 -8
  29. data/lib/fluent/plugin/buf_file.rb +32 -3
  30. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  32. data/lib/fluent/plugin/buffer.rb +21 -0
  33. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  34. data/lib/fluent/plugin/in_forward.rb +1 -1
  35. data/lib/fluent/plugin/in_http.rb +8 -8
  36. data/lib/fluent/plugin/in_sample.rb +1 -1
  37. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  38. data/lib/fluent/plugin/in_tail.rb +58 -24
  39. data/lib/fluent/plugin/in_tcp.rb +47 -2
  40. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  41. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  42. data/lib/fluent/plugin/out_forward.rb +2 -2
  43. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  44. data/lib/fluent/plugin/output.rb +50 -13
  45. data/lib/fluent/plugin/parser_json.rb +1 -1
  46. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  47. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  48. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  49. data/lib/fluent/plugin_helper/server.rb +8 -0
  50. data/lib/fluent/plugin_helper/thread.rb +3 -3
  51. data/lib/fluent/plugin_id.rb +1 -1
  52. data/lib/fluent/supervisor.rb +157 -251
  53. data/lib/fluent/test/driver/base.rb +11 -5
  54. data/lib/fluent/test/driver/filter.rb +4 -0
  55. data/lib/fluent/test/startup_shutdown.rb +6 -8
  56. data/lib/fluent/version.rb +1 -1
  57. data/templates/new_gem/test/helper.rb.erb +0 -1
  58. data/test/command/test_ctl.rb +1 -1
  59. data/test/command/test_fluentd.rb +137 -6
  60. data/test/command/test_plugin_config_formatter.rb +0 -1
  61. data/test/compat/test_parser.rb +5 -5
  62. data/test/config/test_system_config.rb +0 -8
  63. data/test/log/test_console_adapter.rb +110 -0
  64. data/test/plugin/in_tail/test_position_file.rb +31 -1
  65. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  66. data/test/plugin/test_base.rb +99 -1
  67. data/test/plugin/test_buf_file.rb +62 -23
  68. data/test/plugin/test_buf_file_single.rb +65 -0
  69. data/test/plugin/test_buffer_chunk.rb +11 -0
  70. data/test/plugin/test_in_forward.rb +9 -9
  71. data/test/plugin/test_in_http.rb +2 -3
  72. data/test/plugin/test_in_monitor_agent.rb +2 -3
  73. data/test/plugin/test_in_tail.rb +379 -0
  74. data/test/plugin/test_in_tcp.rb +87 -2
  75. data/test/plugin/test_in_udp.rb +28 -0
  76. data/test/plugin/test_in_unix.rb +2 -2
  77. data/test/plugin/test_multi_output.rb +1 -1
  78. data/test/plugin/test_out_exec_filter.rb +2 -2
  79. data/test/plugin/test_out_file.rb +2 -2
  80. data/test/plugin/test_out_forward.rb +14 -18
  81. data/test/plugin/test_out_http.rb +1 -0
  82. data/test/plugin/test_output.rb +281 -12
  83. data/test/plugin/test_output_as_buffered.rb +44 -44
  84. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  85. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  86. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  87. data/test/plugin/test_parser_regexp.rb +1 -6
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  90. data/test/plugin_helper/test_server.rb +60 -6
  91. data/test/test_config.rb +0 -21
  92. data/test/test_formatter.rb +23 -20
  93. data/test/test_log.rb +108 -36
  94. data/test/test_msgpack_factory.rb +32 -0
  95. data/test/test_supervisor.rb +287 -279
  96. metadata +15 -21
  97. data/.drone.yml +0 -35
  98. data/.gitlab-ci.yml +0 -103
  99. data/test/test_logger_initializer.rb +0 -46
@@ -110,7 +110,7 @@ module Fluent
110
110
 
111
111
  def include(*args)
112
112
  ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
113
- if args.first =~ /\.rb$/
113
+ if /\.rb$/.match?(args.first)
114
114
  path = File.expand_path(args.first)
115
115
  data = File.read(path)
116
116
  self.instance_eval(data, path)
@@ -150,8 +150,8 @@ module Fluent
150
150
  def eval_include(attrs, elems, uri)
151
151
  # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
152
152
  # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
153
- u = URI.parse(uri.gsub(/ /, '+'))
154
- if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
153
+ u = URI.parse(uri.tr(' ', '+'))
154
+ if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
155
155
  # When the Windows absolute path then u.scheme.length == 1
156
156
  # e.g. C:
157
157
  path = URI.decode_www_form_component(u.path)
@@ -27,7 +27,7 @@ module Fluent
27
27
  DEFAULT_PORT = 24321
28
28
 
29
29
  def initialize(name, opt = {})
30
- raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
30
+ raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
31
31
  @name = name
32
32
  @opt = opt
33
33
  @addr = @opt[:addr] || DEFAULT_ADDR
@@ -82,7 +82,7 @@ module Fluent
82
82
  raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
83
83
  end
84
84
 
85
- unless VALID_NAME =~ name
85
+ unless VALID_NAME.match?(name)
86
86
  raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
87
87
  end
88
88
  end
@@ -92,7 +92,7 @@ module Fluent
92
92
  raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
93
93
  end
94
94
 
95
- unless VALID_SCOPE_NAME =~ name
95
+ unless VALID_SCOPE_NAME.match?(name)
96
96
  raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
97
97
  end
98
98
  end
@@ -109,7 +109,7 @@ module Fluent
109
109
  raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
110
110
  end
111
111
 
112
- unless VALID_NAME =~ name
112
+ unless VALID_NAME.match?(name)
113
113
  raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
114
114
  end
115
115
  end
data/lib/fluent/daemon.rb CHANGED
@@ -9,7 +9,5 @@ require 'fluent/supervisor'
9
9
 
10
10
  server_module = Fluent.const_get(ARGV[0])
11
11
  worker_module = Fluent.const_get(ARGV[1])
12
- # it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
13
- config_path = ARGV[2]
14
- params = JSON.parse(ARGV[3])
15
- ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
12
+ params = JSON.parse(ARGV[2])
13
+ ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }
data/lib/fluent/engine.rb CHANGED
@@ -68,7 +68,7 @@ module Fluent
68
68
  end
69
69
 
70
70
  def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
71
- if fname =~ /\.rb$/
71
+ if /\.rb$/.match?(fname)
72
72
  require 'fluent/config/dsl'
73
73
  Config::DSL::Parser.parse(io, File.join(basepath, fname))
74
74
  else
data/lib/fluent/event.rb CHANGED
@@ -49,7 +49,7 @@ module Fluent
49
49
  raise NotImplementedError, "DO NOT USE THIS CLASS directly."
50
50
  end
51
51
 
52
- def each(unapcker: nil, &block)
52
+ def each(unpacker: nil, &block)
53
53
  raise NotImplementedError, "DO NOT USE THIS CLASS directly."
54
54
  end
55
55
 
@@ -294,7 +294,7 @@ module Fluent
294
294
  super
295
295
  end
296
296
 
297
- def to_compressed_msgpack_stream(time_int: false)
297
+ def to_compressed_msgpack_stream(time_int: false, packer: nil)
298
298
  # time_int is always ignored because @data is always packed binary in this class
299
299
  @compressed_data
300
300
  end
@@ -308,11 +308,15 @@ module Fluent
308
308
  end
309
309
 
310
310
  module ChunkMessagePackEventStreamer
311
- # chunk.extend(ChunkEventStreamer)
311
+ # chunk.extend(ChunkMessagePackEventStreamer)
312
312
  # => chunk.each{|time, record| ... }
313
313
  def each(unpacker: nil, &block)
314
+ # Note: If need to use `unpacker`, then implement it,
315
+ # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
316
+ raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
317
+
314
318
  open do |io|
315
- (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
319
+ Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
316
320
  end
317
321
  nil
318
322
  end
@@ -0,0 +1,66 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'console/terminal/logger'
18
+
19
+ module Fluent
20
+ class Log
21
+ # Async gem which is used by http_server helper switched logger mechanism to
22
+ # Console gem which isn't complatible with Ruby's standard Logger (since
23
+ # v1.17). This class adapts it to Fluentd's logger mechanism.
24
+ class ConsoleAdapter < Console::Terminal::Logger
25
+ def self.wrap(logger)
26
+ _, level = Console::Logger::LEVELS.find { |key, value|
27
+ if logger.level <= 0
28
+ key == :debug
29
+ else
30
+ value == logger.level - 1
31
+ end
32
+ }
33
+ Console::Logger.new(ConsoleAdapter.new(logger), level: level)
34
+ end
35
+
36
+ def initialize(logger)
37
+ @logger = logger
38
+ # When `verbose` is `true`, following items will be added as a prefix or
39
+ # suffix of the subject:
40
+ # * Severity
41
+ # * Object ID
42
+ # * PID
43
+ # * Time
44
+ # Severity and Time are added by Fluentd::Log too so they are redundant.
45
+ # PID is the worker's PID so it's also redundant.
46
+ # Object ID will be too verbose in usual cases.
47
+ # So set it as `false` here to suppress redundant items.
48
+ super(StringIO.new, verbose: false)
49
+ end
50
+
51
+ def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
52
+ if LEVEL_TEXT.include?(severity.to_s)
53
+ level = severity
54
+ else
55
+ @logger.warn("Unknown severity: #{severity}")
56
+ level = 'warn'
57
+ end
58
+
59
+ @io.seek(0)
60
+ @io.truncate(0)
61
+ super
62
+ @logger.send(level, @io.string.chomp)
63
+ end
64
+ end
65
+ end
66
+ end
data/lib/fluent/log.rb CHANGED
@@ -51,6 +51,8 @@ module Fluent
51
51
  LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
52
52
  LOG_ROTATE_AGE = %w(daily weekly monthly)
53
53
 
54
+ IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
55
+
54
56
  def self.str_to_level(log_level_str)
55
57
  case log_level_str.downcase
56
58
  when "trace" then LEVEL_TRACE
@@ -67,8 +69,29 @@ module Fluent
67
69
  LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
68
70
  end
69
71
 
72
+ # Create a unique path for each process.
73
+ #
74
+ # >>> per_process_path("C:/tmp/test.log", :worker, 1)
75
+ # C:/tmp/test-1.log
76
+ # >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
77
+ # C:/tmp/test-supervisor-0.log
78
+ def self.per_process_path(path, process_type, worker_id)
79
+ path = Pathname(path)
80
+ ext = path.extname
81
+
82
+ if process_type == :supervisor
83
+ suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
84
+ else
85
+ suffix = "-#{worker_id}#{ext}"
86
+ end
87
+ return path.sub_ext(suffix).to_s
88
+ end
89
+
70
90
  def initialize(logger, opts={})
71
- # overwrites logger.level= so that config reloading resets level of Fluentd::Log
91
+ # When ServerEngine changes the logger.level, the Fluentd logger level should also change.
92
+ # So overwrites logger.level= below.
93
+ # However, currently Fluentd doesn't use the ServerEngine's reloading feature,
94
+ # so maybe we don't need this overwriting anymore.
72
95
  orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
73
96
  me = self
74
97
  # The original ruby logger sets the number as each log level like below.
@@ -92,6 +115,7 @@ module Fluent
92
115
  # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
93
116
  logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
94
117
 
118
+ @path = opts[:path]
95
119
  @logger = logger
96
120
  @out = logger.instance_variable_get(:@logdev)
97
121
  @level = logger.level + 1
@@ -102,7 +126,8 @@ module Fluent
102
126
  @time_format = nil
103
127
  @formatter = nil
104
128
 
105
- self.format = :text
129
+ self.format = opts.fetch(:format, :text)
130
+ self.time_format = opts[:time_format] if opts.key?(:time_format)
106
131
  enable_color out.tty?
107
132
  # TODO: This variable name is unclear so we should change to better name.
108
133
  @threads_exclude_events = []
@@ -154,8 +179,12 @@ module Fluent
154
179
 
155
180
  attr_reader :format
156
181
  attr_reader :time_format
157
- attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
182
+ attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
158
183
  attr_accessor :out
184
+ # Strictly speaking, we should also change @logger.level when the setter of @level is called.
185
+ # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
186
+ # Since We overwrites logger.level= so that @logger.level is applied to @level,
187
+ # we need to find a good way to do this, otherwise we will end up in an endless loop.
159
188
  attr_accessor :level
160
189
  attr_accessor :optional_header, :optional_attrs
161
190
 
@@ -202,9 +231,12 @@ module Fluent
202
231
  @time_formatter = Strftime.new(@time_format) rescue nil
203
232
  end
204
233
 
234
+ def stdout?
235
+ @out == $stdout
236
+ end
237
+
205
238
  def reopen!
206
- # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
207
- @logger.reopen! if @logger
239
+ @out.reopen(@path, "a") if @path && @path != "-"
208
240
  nil
209
241
  end
210
242
 
@@ -447,6 +479,13 @@ module Fluent
447
479
  false
448
480
  end
449
481
  else
482
+ if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
483
+ cached_log.reject! do |_, cached_time|
484
+ (time - cached_time) > @ignore_same_log_interval
485
+ end
486
+ end
487
+ # If the size is still over, we have no choice but to clear it.
488
+ cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
450
489
  cached_log[message] = time
451
490
  false
452
491
  end
data/lib/fluent/match.rb CHANGED
@@ -115,7 +115,7 @@ module Fluent
115
115
  stack.last << regex.pop
116
116
  regex.push ''
117
117
 
118
- elsif c =~ /[a-zA-Z0-9_]/
118
+ elsif /[a-zA-Z0-9_]/.match?(c)
119
119
  regex.last << c
120
120
 
121
121
  else
@@ -100,7 +100,12 @@ module Fluent
100
100
  end
101
101
 
102
102
  def self.thread_local_msgpack_unpacker
103
- Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
103
+ unpacker = Thread.current[:local_msgpack_unpacker]
104
+ if unpacker.nil?
105
+ return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
106
+ end
107
+ unpacker.reset
108
+ unpacker
104
109
  end
105
110
  end
106
111
  end
@@ -53,14 +53,12 @@ module Fluent
53
53
  end
54
54
 
55
55
  def configure(conf)
56
- if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
57
- workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
58
- conf.target_worker_ids.size
59
- else
60
- 1
61
- end
62
- system_config_override(workers: workers)
56
+ raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
57
+
58
+ if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
59
+ system_config_override(workers: conf.target_worker_ids.size)
63
60
  end
61
+
64
62
  super(conf, system_config.strict_config_value)
65
63
  @_state ||= State.new(false, false, false, false, false, false, false, false, false)
66
64
  @_state.configure = true
@@ -192,7 +190,7 @@ module Fluent
192
190
  # Thread::Backtrace::Location#path returns base filename or absolute path.
193
191
  # #absolute_path returns absolute_path always.
194
192
  # https://bugs.ruby-lang.org/issues/12159
195
- if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
193
+ if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
196
194
  return true
197
195
  end
198
196
  end
@@ -139,13 +139,20 @@ module Fluent
139
139
  def resume
140
140
  stage = {}
141
141
  queue = []
142
+ exist_broken_file = false
142
143
 
143
144
  patterns = [@path]
144
145
  patterns.unshift @additional_resume_path if @additional_resume_path
145
146
  Dir.glob(escaped_patterns(patterns)) do |path|
146
147
  next unless File.file?(path)
147
148
 
148
- log.debug { "restoring buffer file: path = #{path}" }
149
+ if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
150
+ # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
151
+ # since there may be a power failure or similar failure.
152
+ log.warn { "restoring buffer file: path = #{path}" }
153
+ else
154
+ log.debug { "restoring buffer file: path = #{path}" }
155
+ end
149
156
 
150
157
  m = new_metadata() # this metadata will be overwritten by resuming .meta file content
151
158
  # so it should not added into @metadata_list for now
@@ -158,6 +165,7 @@ module Fluent
158
165
  begin
159
166
  chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
160
167
  rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
168
+ exist_broken_file = true
161
169
  handle_broken_files(path, mode, e)
162
170
  next
163
171
  end
@@ -182,6 +190,15 @@ module Fluent
182
190
 
183
191
  queue.sort_by!{ |chunk| chunk.modified_at }
184
192
 
193
+ # If one of the files is corrupted, other files may also be corrupted and be undetected.
194
+ # The time priods of each chunk are helpful to check the data.
195
+ if exist_broken_file
196
+ log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
197
+ (stage.values + queue).each { |chunk|
198
+ log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
199
+ }
200
+ end
201
+
185
202
  return stage, queue
186
203
  end
187
204
 
@@ -195,8 +212,20 @@ module Fluent
195
212
  end
196
213
 
197
214
  def handle_broken_files(path, mode, e)
198
- log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
199
- # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
215
+ log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
216
+ unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
217
+ backup(unique_id) { |f|
218
+ File.open(path, 'rb') { |chunk|
219
+ chunk.set_encoding(Encoding::ASCII_8BIT)
220
+ chunk.sync = true
221
+ chunk.binmode
222
+ IO.copy_stream(chunk, f)
223
+ }
224
+ }
225
+ rescue => error
226
+ log.error "backup failed. Delete corresponding files.", :err_msg => error.message
227
+ ensure
228
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
200
229
  File.unlink(path, path + '.meta') rescue nil
201
230
  end
202
231
 
@@ -160,13 +160,20 @@ module Fluent
160
160
  def resume
161
161
  stage = {}
162
162
  queue = []
163
+ exist_broken_file = false
163
164
 
164
165
  patterns = [@path]
165
166
  patterns.unshift @additional_resume_path if @additional_resume_path
166
167
  Dir.glob(escaped_patterns(patterns)) do |path|
167
168
  next unless File.file?(path)
168
169
 
169
- log.debug { "restoring buffer file: path = #{path}" }
170
+ if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
171
+ # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
172
+ # since there may be a power failure or similar failure.
173
+ log.warn { "restoring buffer file: path = #{path}" }
174
+ else
175
+ log.debug { "restoring buffer file: path = #{path}" }
176
+ end
170
177
 
171
178
  m = new_metadata() # this metadata will be updated in FileSingleChunk.new
172
179
  mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
@@ -179,6 +186,7 @@ module Fluent
179
186
  chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
180
187
  chunk.restore_size(@chunk_format) if @calc_num_records
181
188
  rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
189
+ exist_broken_file = true
182
190
  handle_broken_files(path, mode, e)
183
191
  next
184
192
  end
@@ -193,6 +201,15 @@ module Fluent
193
201
 
194
202
  queue.sort_by!(&:modified_at)
195
203
 
204
+ # If one of the files is corrupted, other files may also be corrupted and be undetected.
205
+ # The time priods of each chunk are helpful to check the data.
206
+ if exist_broken_file
207
+ log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
208
+ (stage.values + queue).each { |chunk|
209
+ log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
210
+ }
211
+ end
212
+
196
213
  return stage, queue
197
214
  end
198
215
 
@@ -207,8 +224,20 @@ module Fluent
207
224
  end
208
225
 
209
226
  def handle_broken_files(path, mode, e)
210
- log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
211
- # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
227
+ log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
228
+ unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
229
+ backup(unique_id) { |f|
230
+ File.open(path, 'rb') { |chunk|
231
+ chunk.set_encoding(Encoding::ASCII_8BIT)
232
+ chunk.sync = true
233
+ chunk.binmode
234
+ IO.copy_stream(chunk, f)
235
+ }
236
+ }
237
+ rescue => error
238
+ log.error "backup failed. Delete corresponding files.", :err_msg => error.message
239
+ ensure
240
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
212
241
  File.unlink(path) rescue nil
213
242
  end
214
243
 
@@ -204,7 +204,7 @@ module Fluent
204
204
  end
205
205
  end
206
206
 
207
- # used only for queued v0.12 buffer path
207
+ # used only for queued v0.12 buffer path or broken files
208
208
  def self.unique_id_from_path(path)
209
209
  if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
210
210
  return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')
@@ -66,6 +66,9 @@ module Fluent
66
66
  desc 'Compress buffered data.'
67
67
  config_param :compress, :enum, list: [:text, :gzip], default: :text
68
68
 
69
+ desc 'If true, chunks are thrown away when unrecoverable error happens'
70
+ config_param :disable_chunk_backup, :bool, default: false
71
+
69
72
  Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
70
73
  def initialize(timekey, tag, variables)
71
74
  super(timekey, tag, variables, 0)
@@ -903,6 +906,24 @@ module Fluent
903
906
  { 'buffer' => stats }
904
907
  end
905
908
 
909
+ def backup(chunk_unique_id)
910
+ unique_id = dump_unique_id_hex(chunk_unique_id)
911
+
912
+ if @disable_chunk_backup
913
+ log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
914
+ return
915
+ end
916
+
917
+ safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
918
+ backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
919
+ backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
920
+ backup_dir = File.dirname(backup_file)
921
+
922
+ log.warn "bad chunk is moved to #{backup_file}"
923
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
924
+ File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
925
+ end
926
+
906
927
  private
907
928
 
908
929
  def optimistic_queued?(metadata = nil)
@@ -316,7 +316,7 @@ module Fluent::Plugin
316
316
  end
317
317
 
318
318
  (Object.instance_methods).each do |m|
319
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
319
+ undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
320
320
  end
321
321
  end
322
322
  end
@@ -430,7 +430,7 @@ module Fluent::Plugin
430
430
  end
431
431
  _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
432
432
 
433
- node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
433
+ node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
434
434
  if !node && !@security.allow_anonymous_source
435
435
  log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
436
436
  return false, "anonymous source host '#{remote_addr}' denied", nil
@@ -428,7 +428,7 @@ module Fluent::Plugin
428
428
  @content_type = ""
429
429
  @content_encoding = ""
430
430
  headers.each_pair {|k,v|
431
- @env["HTTP_#{k.gsub('-','_').upcase}"] = v
431
+ @env["HTTP_#{k.tr('-','_').upcase}"] = v
432
432
  case k
433
433
  when /\AExpect\z/i
434
434
  expect = v
@@ -439,9 +439,9 @@ module Fluent::Plugin
439
439
  when /\AContent-Encoding\Z/i
440
440
  @content_encoding = v
441
441
  when /\AConnection\Z/i
442
- if v =~ /close/i
442
+ if /close/i.match?(v)
443
443
  @keep_alive = false
444
- elsif v =~ /Keep-alive/i
444
+ elsif /Keep-alive/i.match?(v)
445
445
  @keep_alive = true
446
446
  end
447
447
  when /\AOrigin\Z/i
@@ -566,16 +566,16 @@ module Fluent::Plugin
566
566
 
567
567
  if @format_name != 'default'
568
568
  params[EVENT_RECORD_PARAMETER] = @body
569
- elsif @content_type =~ /^application\/x-www-form-urlencoded/
569
+ elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
570
570
  params.update WEBrick::HTTPUtils.parse_query(@body)
571
571
  elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
572
572
  boundary = WEBrick::HTTPUtils.dequote($1)
573
573
  params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
574
- elsif @content_type =~ /^application\/json/
574
+ elsif /^application\/json/.match?(@content_type)
575
575
  params['json'] = @body
576
- elsif @content_type =~ /^application\/msgpack/
576
+ elsif /^application\/msgpack/.match?(@content_type)
577
577
  params['msgpack'] = @body
578
- elsif @content_type =~ /^application\/x-ndjson/
578
+ elsif /^application\/x-ndjson/.match?(@content_type)
579
579
  params['ndjson'] = @body
580
580
  end
581
581
  path_info = uri.path
@@ -585,7 +585,7 @@ module Fluent::Plugin
585
585
  query_params = WEBrick::HTTPUtils.parse_query(uri.query)
586
586
 
587
587
  query_params.each_pair {|k,v|
588
- params["QUERY_#{k.gsub('-','_').upcase}"] = v
588
+ params["QUERY_#{k.tr('-','_').upcase}"] = v
589
589
  }
590
590
  end
591
591
 
@@ -64,7 +64,7 @@ module Fluent::Plugin
64
64
  def configure(conf)
65
65
  super
66
66
  @sample_index = 0
67
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
+ config = conf.elements.find{|e| e.name == 'storage' }
68
68
  @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
69
  end
70
70