fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +151 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/command/ctl.rb +2 -2
  15. data/lib/fluent/command/fluentd.rb +55 -53
  16. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  17. data/lib/fluent/config/dsl.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +2 -2
  19. data/lib/fluent/counter/server.rb +1 -1
  20. data/lib/fluent/counter/validator.rb +3 -3
  21. data/lib/fluent/daemon.rb +2 -4
  22. data/lib/fluent/engine.rb +1 -1
  23. data/lib/fluent/event.rb +8 -4
  24. data/lib/fluent/log/console_adapter.rb +66 -0
  25. data/lib/fluent/log.rb +44 -5
  26. data/lib/fluent/match.rb +1 -1
  27. data/lib/fluent/msgpack_factory.rb +6 -1
  28. data/lib/fluent/plugin/base.rb +6 -8
  29. data/lib/fluent/plugin/buf_file.rb +32 -3
  30. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  32. data/lib/fluent/plugin/buffer.rb +21 -0
  33. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  34. data/lib/fluent/plugin/in_forward.rb +1 -1
  35. data/lib/fluent/plugin/in_http.rb +8 -8
  36. data/lib/fluent/plugin/in_sample.rb +1 -1
  37. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  38. data/lib/fluent/plugin/in_tail.rb +58 -24
  39. data/lib/fluent/plugin/in_tcp.rb +47 -2
  40. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  41. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  42. data/lib/fluent/plugin/out_forward.rb +2 -2
  43. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  44. data/lib/fluent/plugin/output.rb +50 -13
  45. data/lib/fluent/plugin/parser_json.rb +1 -1
  46. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  47. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  48. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  49. data/lib/fluent/plugin_helper/server.rb +8 -0
  50. data/lib/fluent/plugin_helper/thread.rb +3 -3
  51. data/lib/fluent/plugin_id.rb +1 -1
  52. data/lib/fluent/supervisor.rb +157 -251
  53. data/lib/fluent/test/driver/base.rb +11 -5
  54. data/lib/fluent/test/driver/filter.rb +4 -0
  55. data/lib/fluent/test/startup_shutdown.rb +6 -8
  56. data/lib/fluent/version.rb +1 -1
  57. data/templates/new_gem/test/helper.rb.erb +0 -1
  58. data/test/command/test_ctl.rb +1 -1
  59. data/test/command/test_fluentd.rb +137 -6
  60. data/test/command/test_plugin_config_formatter.rb +0 -1
  61. data/test/compat/test_parser.rb +5 -5
  62. data/test/config/test_system_config.rb +0 -8
  63. data/test/log/test_console_adapter.rb +110 -0
  64. data/test/plugin/in_tail/test_position_file.rb +31 -1
  65. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  66. data/test/plugin/test_base.rb +99 -1
  67. data/test/plugin/test_buf_file.rb +62 -23
  68. data/test/plugin/test_buf_file_single.rb +65 -0
  69. data/test/plugin/test_buffer_chunk.rb +11 -0
  70. data/test/plugin/test_in_forward.rb +9 -9
  71. data/test/plugin/test_in_http.rb +2 -3
  72. data/test/plugin/test_in_monitor_agent.rb +2 -3
  73. data/test/plugin/test_in_tail.rb +379 -0
  74. data/test/plugin/test_in_tcp.rb +87 -2
  75. data/test/plugin/test_in_udp.rb +28 -0
  76. data/test/plugin/test_in_unix.rb +2 -2
  77. data/test/plugin/test_multi_output.rb +1 -1
  78. data/test/plugin/test_out_exec_filter.rb +2 -2
  79. data/test/plugin/test_out_file.rb +2 -2
  80. data/test/plugin/test_out_forward.rb +14 -18
  81. data/test/plugin/test_out_http.rb +1 -0
  82. data/test/plugin/test_output.rb +281 -12
  83. data/test/plugin/test_output_as_buffered.rb +44 -44
  84. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  85. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  86. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  87. data/test/plugin/test_parser_regexp.rb +1 -6
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  90. data/test/plugin_helper/test_server.rb +60 -6
  91. data/test/test_config.rb +0 -21
  92. data/test/test_formatter.rb +23 -20
  93. data/test/test_log.rb +108 -36
  94. data/test/test_msgpack_factory.rb +32 -0
  95. data/test/test_supervisor.rb +287 -279
  96. metadata +15 -21
  97. data/.drone.yml +0 -35
  98. data/.gitlab-ci.yml +0 -103
  99. data/test/test_logger_initializer.rb +0 -46
@@ -110,7 +110,7 @@ module Fluent
110
110
 
111
111
  def include(*args)
112
112
  ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
113
- if args.first =~ /\.rb$/
113
+ if /\.rb$/.match?(args.first)
114
114
  path = File.expand_path(args.first)
115
115
  data = File.read(path)
116
116
  self.instance_eval(data, path)
@@ -150,8 +150,8 @@ module Fluent
150
150
  def eval_include(attrs, elems, uri)
151
151
  # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
152
152
  # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
153
- u = URI.parse(uri.gsub(/ /, '+'))
154
- if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
153
+ u = URI.parse(uri.tr(' ', '+'))
154
+ if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
155
155
  # When the Windows absolute path then u.scheme.length == 1
156
156
  # e.g. C:
157
157
  path = URI.decode_www_form_component(u.path)
@@ -27,7 +27,7 @@ module Fluent
27
27
  DEFAULT_PORT = 24321
28
28
 
29
29
  def initialize(name, opt = {})
30
- raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
30
+ raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
31
31
  @name = name
32
32
  @opt = opt
33
33
  @addr = @opt[:addr] || DEFAULT_ADDR
@@ -82,7 +82,7 @@ module Fluent
82
82
  raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
83
83
  end
84
84
 
85
- unless VALID_NAME =~ name
85
+ unless VALID_NAME.match?(name)
86
86
  raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
87
87
  end
88
88
  end
@@ -92,7 +92,7 @@ module Fluent
92
92
  raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
93
93
  end
94
94
 
95
- unless VALID_SCOPE_NAME =~ name
95
+ unless VALID_SCOPE_NAME.match?(name)
96
96
  raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
97
97
  end
98
98
  end
@@ -109,7 +109,7 @@ module Fluent
109
109
  raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
110
110
  end
111
111
 
112
- unless VALID_NAME =~ name
112
+ unless VALID_NAME.match?(name)
113
113
  raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
114
114
  end
115
115
  end
data/lib/fluent/daemon.rb CHANGED
@@ -9,7 +9,5 @@ require 'fluent/supervisor'
9
9
 
10
10
  server_module = Fluent.const_get(ARGV[0])
11
11
  worker_module = Fluent.const_get(ARGV[1])
12
- # it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
13
- config_path = ARGV[2]
14
- params = JSON.parse(ARGV[3])
15
- ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
12
+ params = JSON.parse(ARGV[2])
13
+ ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }
data/lib/fluent/engine.rb CHANGED
@@ -68,7 +68,7 @@ module Fluent
68
68
  end
69
69
 
70
70
  def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
71
- if fname =~ /\.rb$/
71
+ if /\.rb$/.match?(fname)
72
72
  require 'fluent/config/dsl'
73
73
  Config::DSL::Parser.parse(io, File.join(basepath, fname))
74
74
  else
data/lib/fluent/event.rb CHANGED
@@ -49,7 +49,7 @@ module Fluent
49
49
  raise NotImplementedError, "DO NOT USE THIS CLASS directly."
50
50
  end
51
51
 
52
- def each(unapcker: nil, &block)
52
+ def each(unpacker: nil, &block)
53
53
  raise NotImplementedError, "DO NOT USE THIS CLASS directly."
54
54
  end
55
55
 
@@ -294,7 +294,7 @@ module Fluent
294
294
  super
295
295
  end
296
296
 
297
- def to_compressed_msgpack_stream(time_int: false)
297
+ def to_compressed_msgpack_stream(time_int: false, packer: nil)
298
298
  # time_int is always ignored because @data is always packed binary in this class
299
299
  @compressed_data
300
300
  end
@@ -308,11 +308,15 @@ module Fluent
308
308
  end
309
309
 
310
310
  module ChunkMessagePackEventStreamer
311
- # chunk.extend(ChunkEventStreamer)
311
+ # chunk.extend(ChunkMessagePackEventStreamer)
312
312
  # => chunk.each{|time, record| ... }
313
313
  def each(unpacker: nil, &block)
314
+ # Note: If need to use `unpacker`, then implement it,
315
+ # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
316
+ raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
317
+
314
318
  open do |io|
315
- (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
319
+ Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
316
320
  end
317
321
  nil
318
322
  end
@@ -0,0 +1,66 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'console/terminal/logger'
18
+
19
+ module Fluent
20
+ class Log
21
+ # Async gem which is used by http_server helper switched logger mechanism to
22
+ # Console gem which isn't complatible with Ruby's standard Logger (since
23
+ # v1.17). This class adapts it to Fluentd's logger mechanism.
24
+ class ConsoleAdapter < Console::Terminal::Logger
25
+ def self.wrap(logger)
26
+ _, level = Console::Logger::LEVELS.find { |key, value|
27
+ if logger.level <= 0
28
+ key == :debug
29
+ else
30
+ value == logger.level - 1
31
+ end
32
+ }
33
+ Console::Logger.new(ConsoleAdapter.new(logger), level: level)
34
+ end
35
+
36
+ def initialize(logger)
37
+ @logger = logger
38
+ # When `verbose` is `true`, following items will be added as a prefix or
39
+ # suffix of the subject:
40
+ # * Severity
41
+ # * Object ID
42
+ # * PID
43
+ # * Time
44
+ # Severity and Time are added by Fluentd::Log too so they are redundant.
45
+ # PID is the worker's PID so it's also redundant.
46
+ # Object ID will be too verbose in usual cases.
47
+ # So set it as `false` here to suppress redundant items.
48
+ super(StringIO.new, verbose: false)
49
+ end
50
+
51
+ def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
52
+ if LEVEL_TEXT.include?(severity.to_s)
53
+ level = severity
54
+ else
55
+ @logger.warn("Unknown severity: #{severity}")
56
+ level = 'warn'
57
+ end
58
+
59
+ @io.seek(0)
60
+ @io.truncate(0)
61
+ super
62
+ @logger.send(level, @io.string.chomp)
63
+ end
64
+ end
65
+ end
66
+ end
data/lib/fluent/log.rb CHANGED
@@ -51,6 +51,8 @@ module Fluent
51
51
  LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
52
52
  LOG_ROTATE_AGE = %w(daily weekly monthly)
53
53
 
54
+ IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
55
+
54
56
  def self.str_to_level(log_level_str)
55
57
  case log_level_str.downcase
56
58
  when "trace" then LEVEL_TRACE
@@ -67,8 +69,29 @@ module Fluent
67
69
  LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
68
70
  end
69
71
 
72
+ # Create a unique path for each process.
73
+ #
74
+ # >>> per_process_path("C:/tmp/test.log", :worker, 1)
75
+ # C:/tmp/test-1.log
76
+ # >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
77
+ # C:/tmp/test-supervisor-0.log
78
+ def self.per_process_path(path, process_type, worker_id)
79
+ path = Pathname(path)
80
+ ext = path.extname
81
+
82
+ if process_type == :supervisor
83
+ suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
84
+ else
85
+ suffix = "-#{worker_id}#{ext}"
86
+ end
87
+ return path.sub_ext(suffix).to_s
88
+ end
89
+
70
90
  def initialize(logger, opts={})
71
- # overwrites logger.level= so that config reloading resets level of Fluentd::Log
91
+ # When ServerEngine changes the logger.level, the Fluentd logger level should also change.
92
+ # So overwrites logger.level= below.
93
+ # However, currently Fluentd doesn't use the ServerEngine's reloading feature,
94
+ # so maybe we don't need this overwriting anymore.
72
95
  orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
73
96
  me = self
74
97
  # The original ruby logger sets the number as each log level like below.
@@ -92,6 +115,7 @@ module Fluent
92
115
  # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
93
116
  logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
94
117
 
118
+ @path = opts[:path]
95
119
  @logger = logger
96
120
  @out = logger.instance_variable_get(:@logdev)
97
121
  @level = logger.level + 1
@@ -102,7 +126,8 @@ module Fluent
102
126
  @time_format = nil
103
127
  @formatter = nil
104
128
 
105
- self.format = :text
129
+ self.format = opts.fetch(:format, :text)
130
+ self.time_format = opts[:time_format] if opts.key?(:time_format)
106
131
  enable_color out.tty?
107
132
  # TODO: This variable name is unclear so we should change to better name.
108
133
  @threads_exclude_events = []
@@ -154,8 +179,12 @@ module Fluent
154
179
 
155
180
  attr_reader :format
156
181
  attr_reader :time_format
157
- attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
182
+ attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
158
183
  attr_accessor :out
184
+ # Strictly speaking, we should also change @logger.level when the setter of @level is called.
185
+ # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
186
+ # Since We overwrites logger.level= so that @logger.level is applied to @level,
187
+ # we need to find a good way to do this, otherwise we will end up in an endless loop.
159
188
  attr_accessor :level
160
189
  attr_accessor :optional_header, :optional_attrs
161
190
 
@@ -202,9 +231,12 @@ module Fluent
202
231
  @time_formatter = Strftime.new(@time_format) rescue nil
203
232
  end
204
233
 
234
+ def stdout?
235
+ @out == $stdout
236
+ end
237
+
205
238
  def reopen!
206
- # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
207
- @logger.reopen! if @logger
239
+ @out.reopen(@path, "a") if @path && @path != "-"
208
240
  nil
209
241
  end
210
242
 
@@ -447,6 +479,13 @@ module Fluent
447
479
  false
448
480
  end
449
481
  else
482
+ if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
483
+ cached_log.reject! do |_, cached_time|
484
+ (time - cached_time) > @ignore_same_log_interval
485
+ end
486
+ end
487
+ # If the size is still over, we have no choice but to clear it.
488
+ cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
450
489
  cached_log[message] = time
451
490
  false
452
491
  end
data/lib/fluent/match.rb CHANGED
@@ -115,7 +115,7 @@ module Fluent
115
115
  stack.last << regex.pop
116
116
  regex.push ''
117
117
 
118
- elsif c =~ /[a-zA-Z0-9_]/
118
+ elsif /[a-zA-Z0-9_]/.match?(c)
119
119
  regex.last << c
120
120
 
121
121
  else
@@ -100,7 +100,12 @@ module Fluent
100
100
  end
101
101
 
102
102
  def self.thread_local_msgpack_unpacker
103
- Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
103
+ unpacker = Thread.current[:local_msgpack_unpacker]
104
+ if unpacker.nil?
105
+ return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
106
+ end
107
+ unpacker.reset
108
+ unpacker
104
109
  end
105
110
  end
106
111
  end
@@ -53,14 +53,12 @@ module Fluent
53
53
  end
54
54
 
55
55
  def configure(conf)
56
- if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
57
- workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
58
- conf.target_worker_ids.size
59
- else
60
- 1
61
- end
62
- system_config_override(workers: workers)
56
+ raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
57
+
58
+ if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
59
+ system_config_override(workers: conf.target_worker_ids.size)
63
60
  end
61
+
64
62
  super(conf, system_config.strict_config_value)
65
63
  @_state ||= State.new(false, false, false, false, false, false, false, false, false)
66
64
  @_state.configure = true
@@ -192,7 +190,7 @@ module Fluent
192
190
  # Thread::Backtrace::Location#path returns base filename or absolute path.
193
191
  # #absolute_path returns absolute_path always.
194
192
  # https://bugs.ruby-lang.org/issues/12159
195
- if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
193
+ if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
196
194
  return true
197
195
  end
198
196
  end
@@ -139,13 +139,20 @@ module Fluent
139
139
  def resume
140
140
  stage = {}
141
141
  queue = []
142
+ exist_broken_file = false
142
143
 
143
144
  patterns = [@path]
144
145
  patterns.unshift @additional_resume_path if @additional_resume_path
145
146
  Dir.glob(escaped_patterns(patterns)) do |path|
146
147
  next unless File.file?(path)
147
148
 
148
- log.debug { "restoring buffer file: path = #{path}" }
149
+ if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
150
+ # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
151
+ # since there may be a power failure or similar failure.
152
+ log.warn { "restoring buffer file: path = #{path}" }
153
+ else
154
+ log.debug { "restoring buffer file: path = #{path}" }
155
+ end
149
156
 
150
157
  m = new_metadata() # this metadata will be overwritten by resuming .meta file content
151
158
  # so it should not added into @metadata_list for now
@@ -158,6 +165,7 @@ module Fluent
158
165
  begin
159
166
  chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
160
167
  rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
168
+ exist_broken_file = true
161
169
  handle_broken_files(path, mode, e)
162
170
  next
163
171
  end
@@ -182,6 +190,15 @@ module Fluent
182
190
 
183
191
  queue.sort_by!{ |chunk| chunk.modified_at }
184
192
 
193
+ # If one of the files is corrupted, other files may also be corrupted and be undetected.
194
+ # The time priods of each chunk are helpful to check the data.
195
+ if exist_broken_file
196
+ log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
197
+ (stage.values + queue).each { |chunk|
198
+ log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
199
+ }
200
+ end
201
+
185
202
  return stage, queue
186
203
  end
187
204
 
@@ -195,8 +212,20 @@ module Fluent
195
212
  end
196
213
 
197
214
  def handle_broken_files(path, mode, e)
198
- log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
199
- # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
215
+ log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
216
+ unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
217
+ backup(unique_id) { |f|
218
+ File.open(path, 'rb') { |chunk|
219
+ chunk.set_encoding(Encoding::ASCII_8BIT)
220
+ chunk.sync = true
221
+ chunk.binmode
222
+ IO.copy_stream(chunk, f)
223
+ }
224
+ }
225
+ rescue => error
226
+ log.error "backup failed. Delete corresponding files.", :err_msg => error.message
227
+ ensure
228
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
200
229
  File.unlink(path, path + '.meta') rescue nil
201
230
  end
202
231
 
@@ -160,13 +160,20 @@ module Fluent
160
160
  def resume
161
161
  stage = {}
162
162
  queue = []
163
+ exist_broken_file = false
163
164
 
164
165
  patterns = [@path]
165
166
  patterns.unshift @additional_resume_path if @additional_resume_path
166
167
  Dir.glob(escaped_patterns(patterns)) do |path|
167
168
  next unless File.file?(path)
168
169
 
169
- log.debug { "restoring buffer file: path = #{path}" }
170
+ if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
171
+ # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
172
+ # since there may be a power failure or similar failure.
173
+ log.warn { "restoring buffer file: path = #{path}" }
174
+ else
175
+ log.debug { "restoring buffer file: path = #{path}" }
176
+ end
170
177
 
171
178
  m = new_metadata() # this metadata will be updated in FileSingleChunk.new
172
179
  mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
@@ -179,6 +186,7 @@ module Fluent
179
186
  chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
180
187
  chunk.restore_size(@chunk_format) if @calc_num_records
181
188
  rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
189
+ exist_broken_file = true
182
190
  handle_broken_files(path, mode, e)
183
191
  next
184
192
  end
@@ -193,6 +201,15 @@ module Fluent
193
201
 
194
202
  queue.sort_by!(&:modified_at)
195
203
 
204
+ # If one of the files is corrupted, other files may also be corrupted and be undetected.
205
+ # The time priods of each chunk are helpful to check the data.
206
+ if exist_broken_file
207
+ log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
208
+ (stage.values + queue).each { |chunk|
209
+ log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
210
+ }
211
+ end
212
+
196
213
  return stage, queue
197
214
  end
198
215
 
@@ -207,8 +224,20 @@ module Fluent
207
224
  end
208
225
 
209
226
  def handle_broken_files(path, mode, e)
210
- log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
211
- # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
227
+ log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
228
+ unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
229
+ backup(unique_id) { |f|
230
+ File.open(path, 'rb') { |chunk|
231
+ chunk.set_encoding(Encoding::ASCII_8BIT)
232
+ chunk.sync = true
233
+ chunk.binmode
234
+ IO.copy_stream(chunk, f)
235
+ }
236
+ }
237
+ rescue => error
238
+ log.error "backup failed. Delete corresponding files.", :err_msg => error.message
239
+ ensure
240
+ log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
212
241
  File.unlink(path) rescue nil
213
242
  end
214
243
 
@@ -204,7 +204,7 @@ module Fluent
204
204
  end
205
205
  end
206
206
 
207
- # used only for queued v0.12 buffer path
207
+ # used only for queued v0.12 buffer path or broken files
208
208
  def self.unique_id_from_path(path)
209
209
  if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
210
210
  return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')
@@ -66,6 +66,9 @@ module Fluent
66
66
  desc 'Compress buffered data.'
67
67
  config_param :compress, :enum, list: [:text, :gzip], default: :text
68
68
 
69
+ desc 'If true, chunks are thrown away when unrecoverable error happens'
70
+ config_param :disable_chunk_backup, :bool, default: false
71
+
69
72
  Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
70
73
  def initialize(timekey, tag, variables)
71
74
  super(timekey, tag, variables, 0)
@@ -903,6 +906,24 @@ module Fluent
903
906
  { 'buffer' => stats }
904
907
  end
905
908
 
909
+ def backup(chunk_unique_id)
910
+ unique_id = dump_unique_id_hex(chunk_unique_id)
911
+
912
+ if @disable_chunk_backup
913
+ log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
914
+ return
915
+ end
916
+
917
+ safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
918
+ backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
919
+ backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
920
+ backup_dir = File.dirname(backup_file)
921
+
922
+ log.warn "bad chunk is moved to #{backup_file}"
923
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
924
+ File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
925
+ end
926
+
906
927
  private
907
928
 
908
929
  def optimistic_queued?(metadata = nil)
@@ -316,7 +316,7 @@ module Fluent::Plugin
316
316
  end
317
317
 
318
318
  (Object.instance_methods).each do |m|
319
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
319
+ undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
320
320
  end
321
321
  end
322
322
  end
@@ -430,7 +430,7 @@ module Fluent::Plugin
430
430
  end
431
431
  _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
432
432
 
433
- node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
433
+ node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
434
434
  if !node && !@security.allow_anonymous_source
435
435
  log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
436
436
  return false, "anonymous source host '#{remote_addr}' denied", nil
@@ -428,7 +428,7 @@ module Fluent::Plugin
428
428
  @content_type = ""
429
429
  @content_encoding = ""
430
430
  headers.each_pair {|k,v|
431
- @env["HTTP_#{k.gsub('-','_').upcase}"] = v
431
+ @env["HTTP_#{k.tr('-','_').upcase}"] = v
432
432
  case k
433
433
  when /\AExpect\z/i
434
434
  expect = v
@@ -439,9 +439,9 @@ module Fluent::Plugin
439
439
  when /\AContent-Encoding\Z/i
440
440
  @content_encoding = v
441
441
  when /\AConnection\Z/i
442
- if v =~ /close/i
442
+ if /close/i.match?(v)
443
443
  @keep_alive = false
444
- elsif v =~ /Keep-alive/i
444
+ elsif /Keep-alive/i.match?(v)
445
445
  @keep_alive = true
446
446
  end
447
447
  when /\AOrigin\Z/i
@@ -566,16 +566,16 @@ module Fluent::Plugin
566
566
 
567
567
  if @format_name != 'default'
568
568
  params[EVENT_RECORD_PARAMETER] = @body
569
- elsif @content_type =~ /^application\/x-www-form-urlencoded/
569
+ elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
570
570
  params.update WEBrick::HTTPUtils.parse_query(@body)
571
571
  elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
572
572
  boundary = WEBrick::HTTPUtils.dequote($1)
573
573
  params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
574
- elsif @content_type =~ /^application\/json/
574
+ elsif /^application\/json/.match?(@content_type)
575
575
  params['json'] = @body
576
- elsif @content_type =~ /^application\/msgpack/
576
+ elsif /^application\/msgpack/.match?(@content_type)
577
577
  params['msgpack'] = @body
578
- elsif @content_type =~ /^application\/x-ndjson/
578
+ elsif /^application\/x-ndjson/.match?(@content_type)
579
579
  params['ndjson'] = @body
580
580
  end
581
581
  path_info = uri.path
@@ -585,7 +585,7 @@ module Fluent::Plugin
585
585
  query_params = WEBrick::HTTPUtils.parse_query(uri.query)
586
586
 
587
587
  query_params.each_pair {|k,v|
588
- params["QUERY_#{k.gsub('-','_').upcase}"] = v
588
+ params["QUERY_#{k.tr('-','_').upcase}"] = v
589
589
  }
590
590
  end
591
591
 
@@ -64,7 +64,7 @@ module Fluent::Plugin
64
64
  def configure(conf)
65
65
  super
66
66
  @sample_index = 0
67
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
+ config = conf.elements.find{|e| e.name == 'storage' }
68
68
  @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
69
  end
70
70