fluentd 1.17.1-x64-mingw-ucrt → 1.19.0-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +116 -0
  3. data/CHANGELOG.md +293 -16
  4. data/MAINTAINERS.md +8 -2
  5. data/README.md +3 -7
  6. data/Rakefile +2 -0
  7. data/SECURITY.md +5 -3
  8. data/lib/fluent/command/cap_ctl.rb +2 -2
  9. data/lib/fluent/command/fluentd.rb +13 -3
  10. data/lib/fluent/compat/formatter.rb +6 -0
  11. data/lib/fluent/compat/socket_util.rb +2 -2
  12. data/lib/fluent/config/configure_proxy.rb +1 -1
  13. data/lib/fluent/config/element.rb +2 -2
  14. data/lib/fluent/config/literal_parser.rb +12 -5
  15. data/lib/fluent/config/parser.rb +15 -3
  16. data/lib/fluent/config/section.rb +2 -2
  17. data/lib/fluent/config/types.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +3 -3
  19. data/lib/fluent/counter/store.rb +1 -1
  20. data/lib/fluent/engine.rb +50 -34
  21. data/lib/fluent/env.rb +6 -2
  22. data/lib/fluent/event.rb +7 -6
  23. data/lib/fluent/event_router.rb +2 -2
  24. data/lib/fluent/log/console_adapter.rb +5 -7
  25. data/lib/fluent/log.rb +23 -0
  26. data/lib/fluent/plugin/bare_output.rb +0 -16
  27. data/lib/fluent/plugin/base.rb +2 -2
  28. data/lib/fluent/plugin/buf_file.rb +15 -1
  29. data/lib/fluent/plugin/buf_file_single.rb +15 -1
  30. data/lib/fluent/plugin/buffer/chunk.rb +74 -10
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +9 -5
  32. data/lib/fluent/plugin/buffer/file_single_chunk.rb +3 -3
  33. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -2
  34. data/lib/fluent/plugin/buffer.rb +34 -6
  35. data/lib/fluent/plugin/compressable.rb +68 -22
  36. data/lib/fluent/plugin/filter.rb +0 -8
  37. data/lib/fluent/plugin/filter_parser.rb +27 -51
  38. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  39. data/lib/fluent/plugin/formatter_csv.rb +18 -4
  40. data/lib/fluent/plugin/formatter_json.rb +7 -4
  41. data/lib/fluent/plugin/formatter_out_file.rb +5 -2
  42. data/lib/fluent/plugin/in_forward.rb +9 -5
  43. data/lib/fluent/plugin/in_http.rb +14 -4
  44. data/lib/fluent/plugin/in_monitor_agent.rb +4 -8
  45. data/lib/fluent/plugin/in_syslog.rb +4 -0
  46. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  47. data/lib/fluent/plugin/in_tail.rb +80 -57
  48. data/lib/fluent/plugin/in_tcp.rb +6 -2
  49. data/lib/fluent/plugin/in_udp.rb +11 -2
  50. data/lib/fluent/plugin/input.rb +4 -8
  51. data/lib/fluent/plugin/multi_output.rb +1 -17
  52. data/lib/fluent/plugin/out_buffer.rb +40 -0
  53. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  54. data/lib/fluent/plugin/out_file.rb +37 -30
  55. data/lib/fluent/plugin/out_forward/connection_manager.rb +2 -2
  56. data/lib/fluent/plugin/out_forward.rb +23 -13
  57. data/lib/fluent/plugin/out_http.rb +1 -1
  58. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  59. data/lib/fluent/plugin/out_stdout.rb +10 -3
  60. data/lib/fluent/plugin/out_stream.rb +3 -3
  61. data/lib/fluent/plugin/output.rb +26 -35
  62. data/lib/fluent/plugin/owned_by_mixin.rb +2 -2
  63. data/lib/fluent/plugin/parser.rb +3 -3
  64. data/lib/fluent/plugin/parser_json.rb +3 -3
  65. data/lib/fluent/plugin/sd_file.rb +2 -2
  66. data/lib/fluent/plugin/storage_local.rb +8 -4
  67. data/lib/fluent/plugin.rb +1 -1
  68. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  69. data/lib/fluent/plugin_helper/child_process.rb +2 -2
  70. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  71. data/lib/fluent/plugin_helper/http_server/request.rb +13 -2
  72. data/lib/fluent/plugin_helper/http_server/server.rb +14 -8
  73. data/lib/fluent/plugin_helper/http_server.rb +1 -8
  74. data/lib/fluent/plugin_helper/metrics.rb +7 -0
  75. data/lib/fluent/plugin_helper/server.rb +13 -1
  76. data/lib/fluent/plugin_helper/service_discovery.rb +1 -1
  77. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  78. data/lib/fluent/plugin_helper/storage.rb +1 -1
  79. data/lib/fluent/plugin_id.rb +3 -3
  80. data/lib/fluent/root_agent.rb +117 -21
  81. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  82. data/lib/fluent/static_config_analysis.rb +3 -2
  83. data/lib/fluent/supervisor.rb +258 -39
  84. data/lib/fluent/system_config.rb +27 -6
  85. data/lib/fluent/test/base.rb +1 -1
  86. data/lib/fluent/test/driver/base.rb +2 -2
  87. data/lib/fluent/test/filter_test.rb +2 -2
  88. data/lib/fluent/test/formatter_test.rb +1 -1
  89. data/lib/fluent/test/helpers.rb +4 -0
  90. data/lib/fluent/test/input_test.rb +2 -2
  91. data/lib/fluent/test/output_test.rb +4 -4
  92. data/lib/fluent/test/parser_test.rb +1 -1
  93. data/lib/fluent/tls.rb +24 -0
  94. data/lib/fluent/variable_store.rb +1 -1
  95. data/lib/fluent/version.rb +1 -1
  96. data/lib/fluent/winsvc.rb +38 -8
  97. metadata +99 -28
  98. data/lib/fluent/plugin_helper/http_server/compat/server.rb +0 -92
  99. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +0 -52
  100. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +0 -58
@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
46
46
  }
47
47
 
48
48
  op.on('-p', '--plugin DIR', "add plugin directory") {|s|
49
- (cmd_opts[:plugin_dirs] ||= []) << s
49
+ (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
50
50
  }
51
51
 
52
52
  op.on('-I PATH', "add library path") {|s|
@@ -127,7 +127,13 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
127
127
  cmd_opts[:without_source] = b
128
128
  }
129
129
 
130
- op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
130
+ unless Fluent.windows?
131
+ op.on('--with-source-only', "Invoke a fluentd only with input plugins. The data is stored in a temporary buffer. Send SIGWINCH to cancel this mode and process the data (Not supported on Windows).", TrueClass) {|b|
132
+ cmd_opts[:with_source_only] = b
133
+ }
134
+ end
135
+
136
+ op.on('--config-file-type VALUE', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
131
137
  if (s == 'yaml') || (s == 'yml')
132
138
  cmd_opts[:config_file_type] = s.to_sym
133
139
  elsif (s == 'guess')
@@ -149,7 +155,11 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
149
155
  cmd_opts[:strict_config_value] = b
150
156
  }
151
157
 
152
- op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
158
+ op.on('--enable-input-metrics', "[DEPRECATED] Enable input plugin metrics on fluentd", TrueClass) {|b|
159
+ cmd_opts[:enable_input_metrics] = b
160
+ }
161
+
162
+ op.on('--disable-input-metrics', "Disable input plugin metrics on fluentd", FalseClass) {|b|
153
163
  cmd_opts[:enable_input_metrics] = b
154
164
  }
155
165
 
@@ -101,6 +101,12 @@ module Fluent
101
101
 
102
102
  class CsvFormatter < Fluent::Plugin::CsvFormatter
103
103
  # TODO: warn when deprecated
104
+
105
+ # Do not cache because it is hard to consider the thread key correctly.
106
+ # (We can try, but it would be low priority.)
107
+ def csv_cacheable?
108
+ false
109
+ end
104
110
  end
105
111
 
106
112
  class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter
@@ -129,7 +129,7 @@ module Fluent
129
129
 
130
130
  def shutdown
131
131
  @loop.watchers.each { |w| w.detach }
132
- @loop.stop if @loop.instance_variable_get("@running")
132
+ @loop.stop if @loop.instance_variable_get(:@running)
133
133
  @handler.close
134
134
  @thread.join
135
135
 
@@ -148,7 +148,7 @@ module Fluent
148
148
  def on_message(msg, addr)
149
149
  @parser.parse(msg) { |time, record|
150
150
  unless time && record
151
- log.warn "pattern not matched: #{msg.inspect}"
151
+ log.warn { "pattern not matched: #{msg.inspect}" }
152
152
  return
153
153
  end
154
154
 
@@ -47,7 +47,7 @@ module Fluent
47
47
  # should override "@name".
48
48
  @root_section = root
49
49
 
50
- @param_name = param_name && param_name.to_sym
50
+ @param_name = param_name&.to_sym
51
51
  @init = init
52
52
  @required = required
53
53
  @multi = multi
@@ -31,7 +31,7 @@ module Fluent
31
31
  @unused = unused || attrs.keys
32
32
  @v1_config = false
33
33
  @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
34
- @unused_in = nil # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
34
+ @unused_in = nil # if this element is not used in plugins, corresponding plugin name and parent element name is set, e.g. [source, plugin class].
35
35
 
36
36
  # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
37
37
  @logger = defined?($log) ? $log : nil
@@ -73,7 +73,7 @@ module Fluent
73
73
 
74
74
  def inspect
75
75
  attrs = super
76
- "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
76
+ "<name:#{@name}, arg:#{@arg}, attrs:#{attrs}, elements:#{@elements.inspect}>"
77
77
  end
78
78
 
79
79
  # Used by PP and Pry
@@ -98,11 +98,11 @@ module Fluent
98
98
  else
99
99
  return string.join
100
100
  end
101
- elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
102
- if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
101
+ elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
102
+ if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
103
103
  string << s
104
104
  end
105
- skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
105
+ skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
106
106
  elsif s = scan(/\\./)
107
107
  string << eval_escape_char(s[1,1])
108
108
  elsif skip(/\#\{/)
@@ -254,8 +254,15 @@ EOM
254
254
  buffer << line_buffer + "\n"
255
255
  line_buffer = ""
256
256
  else
257
- # '#' is a char in json string
258
- line_buffer << char
257
+ if @ss.exist?(/^\{[^}]+\}/)
258
+ # if it's interpolated string
259
+ skip(/\{/)
260
+ line_buffer << eval_embedded_code(scan_embedded_code)
261
+ skip(/\}/)
262
+ else
263
+ # '#' is a char in json string
264
+ line_buffer << char
265
+ end
259
266
  end
260
267
 
261
268
  next # This char '#' MUST NOT terminate json object.
@@ -92,10 +92,23 @@ module Fluent
92
92
  else
93
93
  basepath = '/'
94
94
  fname = path
95
- require 'open-uri'
96
- URI.open(uri) {|f|
95
+ parser_proc = ->(f) {
97
96
  Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
98
97
  }
98
+
99
+ case u.scheme
100
+ when 'http', 'https', 'ftp'
101
+ # URI#open can be able to handle URIs for http, https and ftp.
102
+ require 'open-uri'
103
+ u.open(&parser_proc)
104
+ else
105
+ # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
106
+ # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
107
+ # - uri is 'c:/path/to/file'
108
+ # - u.path is '/path/to/file' and u.scheme is 'c'
109
+ # Therefore, the condition of the if statement above is not met and it is handled here.
110
+ File.open(uri, &parser_proc)
111
+ end
99
112
  end
100
113
 
101
114
  rescue SystemCallError => e
@@ -104,4 +117,3 @@ module Fluent
104
117
  end
105
118
  end
106
119
  end
107
-
@@ -150,7 +150,7 @@ module Fluent
150
150
  end
151
151
  end
152
152
  unless section_params.has_key?(proxy.argument.first)
153
- logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
153
+ logger.error "config error in:\n#{conf}" if logger # logger should exist, but sometimes it's nil (e.g, in tests)
154
154
  raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
155
155
  end
156
156
  # argument should NOT be deprecated... (argument always has a value: '')
@@ -253,7 +253,7 @@ module Fluent
253
253
  elems = conf.respond_to?(:elements) ? conf.elements : []
254
254
  elems.each { |e|
255
255
  next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
256
- next if e.unused_in && e.unused_in.empty? # the section is used at least once
256
+ next if e.unused_in&.empty? # the section is used at least once
257
257
 
258
258
  if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
259
259
  e.unused_in = []
@@ -71,7 +71,7 @@ module Fluent
71
71
  else
72
72
  # Current parser passes comment without actual values, e.g. "param #foo".
73
73
  # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
74
- if str.respond_to?('start_with?') && str.start_with?('#')
74
+ if str.respond_to?(:start_with?) && str.start_with?('#')
75
75
  true
76
76
  elsif opts[:strict]
77
77
  raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
@@ -83,7 +83,7 @@ module Fluent
83
83
  elsif skip(/\</)
84
84
  e_name = scan(ELEMENT_NAME)
85
85
  spacing
86
- e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
86
+ e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/o)
87
87
  spacing
88
88
  unless skip(/\>/)
89
89
  parse_error! "expected '>'"
@@ -98,7 +98,7 @@ module Fluent
98
98
  new_e.v1_config = true
99
99
  elems << new_e
100
100
 
101
- elsif root_element && skip(/(\@include|include)#{SPACING}/)
101
+ elsif root_element && skip(/(\@include|include)#{SPACING}/o)
102
102
  if !prev_match.start_with?('@')
103
103
  @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
104
104
  end
@@ -172,7 +172,7 @@ module Fluent
172
172
  require 'open-uri'
173
173
  basepath = '/'
174
174
  fname = path
175
- data = URI.open(uri) { |f| f.read }
175
+ data = u.open { |f| f.read }
176
176
  data.force_encoding('UTF-8')
177
177
  ss = StringScanner.new(data)
178
178
  V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
@@ -156,7 +156,7 @@ module Fluent
156
156
  }
157
157
  end
158
158
 
159
- # value is Hash. value requires these fileds.
159
+ # value is Hash. value requires these fields.
160
160
  # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
161
161
  def build_value(data)
162
162
  type = data['type'] || 'numeric'
data/lib/fluent/engine.rb CHANGED
@@ -43,13 +43,15 @@ module Fluent
43
43
  @system_config = SystemConfig.new
44
44
 
45
45
  @supervisor_mode = false
46
+
47
+ @root_agent_mutex = Mutex.new
46
48
  end
47
49
 
48
50
  MAINLOOP_SLEEP_INTERVAL = 0.3
49
51
 
50
52
  attr_reader :root_agent, :system_config, :supervisor_mode
51
53
 
52
- def init(system_config, supervisor_mode: false)
54
+ def init(system_config, supervisor_mode: false, start_in_parallel: false)
53
55
  @system_config = system_config
54
56
  @supervisor_mode = supervisor_mode
55
57
 
@@ -58,7 +60,7 @@ module Fluent
58
60
 
59
61
  @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
60
62
 
61
- @root_agent = RootAgent.new(log: log, system_config: @system_config)
63
+ @root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)
62
64
 
63
65
  self
64
66
  end
@@ -133,7 +135,15 @@ module Fluent
133
135
  end
134
136
 
135
137
  def flush!
136
- @root_agent.flush!
138
+ @root_agent_mutex.synchronize do
139
+ @root_agent.flush!
140
+ end
141
+ end
142
+
143
+ def cancel_source_only!
144
+ @root_agent_mutex.synchronize do
145
+ @root_agent.cancel_source_only!
146
+ end
137
147
  end
138
148
 
139
149
  def now
@@ -144,7 +154,9 @@ module Fluent
144
154
  def run
145
155
  begin
146
156
  $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
147
- start
157
+ @root_agent_mutex.synchronize do
158
+ start
159
+ end
148
160
 
149
161
  @fluent_log_event_router.start
150
162
 
@@ -158,47 +170,51 @@ module Fluent
158
170
  raise
159
171
  end
160
172
 
161
- stop_phase(@root_agent)
173
+ @root_agent_mutex.synchronize do
174
+ stop_phase(@root_agent)
175
+ end
162
176
  end
163
177
 
164
178
  # @param conf [Fluent::Config]
165
179
  # @param supervisor [Bool]
166
- # @reutrn nil
180
+ # @return nil
167
181
  def reload_config(conf, supervisor: false)
168
- # configure first to reduce down time while restarting
169
- new_agent = RootAgent.new(log: log, system_config: @system_config)
170
- ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
171
-
172
- ret.all_plugins.each do |plugin|
173
- if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
174
- raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
182
+ @root_agent_mutex.synchronize do
183
+ # configure first to reduce down time while restarting
184
+ new_agent = RootAgent.new(log: log, system_config: @system_config)
185
+ ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
186
+
187
+ ret.all_plugins.each do |plugin|
188
+ if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
189
+ raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
190
+ end
175
191
  end
176
- end
177
192
 
178
- # Assign @root_agent to new root_agent
179
- # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
180
- old_agent, @root_agent = @root_agent, new_agent
181
- begin
182
- @root_agent.configure(conf)
183
- rescue
184
- @root_agent = old_agent
185
- raise
186
- end
193
+ # Assign @root_agent to new root_agent
194
+ # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
195
+ old_agent, @root_agent = @root_agent, new_agent
196
+ begin
197
+ @root_agent.configure(conf)
198
+ rescue
199
+ @root_agent = old_agent
200
+ raise
201
+ end
187
202
 
188
- unless @suppress_config_dump
189
- $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
190
- end
203
+ unless @suppress_config_dump
204
+ $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
205
+ end
191
206
 
192
- # supervisor doesn't handle actual data. so the following code is unnecessary.
193
- if supervisor
194
- old_agent.shutdown # to close thread created in #configure
195
- return
196
- end
207
+ # supervisor doesn't handle actual data. so the following code is unnecessary.
208
+ if supervisor
209
+ old_agent.shutdown # to close thread created in #configure
210
+ return
211
+ end
197
212
 
198
- stop_phase(old_agent)
213
+ stop_phase(old_agent)
199
214
 
200
- $log.info 'restart fluentd worker', worker: worker_id
201
- start_phase(new_agent)
215
+ $log.info 'restart fluentd worker', worker: worker_id
216
+ start_phase(new_agent)
217
+ end
202
218
  end
203
219
 
204
220
  def stop
data/lib/fluent/env.rb CHANGED
@@ -14,27 +14,31 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'securerandom'
18
+
17
19
  require 'serverengine/utils'
18
20
  require 'fluent/oj_options'
19
21
 
20
22
  module Fluent
21
23
  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
24
+ DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d'
22
25
  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
23
26
  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
24
27
  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
25
28
  DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
26
29
  DEFAULT_DIR_PERMISSION = 0755
27
30
  DEFAULT_FILE_PERMISSION = 0644
31
+ INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid
28
32
 
29
33
  def self.windows?
30
34
  ServerEngine.windows?
31
35
  end
32
36
 
33
37
  def self.linux?
34
- /linux/ === RUBY_PLATFORM
38
+ RUBY_PLATFORM.include?("linux")
35
39
  end
36
40
 
37
41
  def self.macos?
38
- /darwin/ =~ RUBY_PLATFORM
42
+ RUBY_PLATFORM.include?("darwin")
39
43
  end
40
44
  end
data/lib/fluent/event.rb CHANGED
@@ -62,9 +62,9 @@ module Fluent
62
62
  out.full_pack
63
63
  end
64
64
 
65
- def to_compressed_msgpack_stream(time_int: false, packer: nil)
65
+ def to_compressed_msgpack_stream(time_int: false, packer: nil, type: :gzip)
66
66
  packed = to_msgpack_stream(time_int: time_int, packer: packer)
67
- compress(packed)
67
+ compress(packed, type: type)
68
68
  end
69
69
 
70
70
  def to_msgpack_stream_forced_integer(packer: nil)
@@ -247,7 +247,7 @@ module Fluent
247
247
  end
248
248
 
249
249
  # This method returns MultiEventStream, because there are no reason
250
- # to surve binary serialized by msgpack.
250
+ # to survey binary serialized by msgpack.
251
251
  def slice(index, num)
252
252
  ensure_unpacked!
253
253
  MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
@@ -268,10 +268,11 @@ module Fluent
268
268
  end
269
269
 
270
270
  class CompressedMessagePackEventStream < MessagePackEventStream
271
- def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
272
- super
271
+ def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip)
272
+ super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)
273
273
  @decompressed_data = nil
274
274
  @compressed_data = data
275
+ @type = compress
275
276
  end
276
277
 
277
278
  def empty?
@@ -303,7 +304,7 @@ module Fluent
303
304
 
304
305
  def ensure_decompressed!
305
306
  return if @decompressed_data
306
- @data = @decompressed_data = decompress(@data)
307
+ @data = @decompressed_data = decompress(@data, type: @type)
307
308
  end
308
309
  end
309
310
 
@@ -286,7 +286,7 @@ module Fluent
286
286
 
287
287
  def find(tag)
288
288
  pipeline = nil
289
- @match_rules.each_with_index { |rule, i|
289
+ @match_rules.each do |rule|
290
290
  if rule.match?(tag)
291
291
  if rule.collector.is_a?(Plugin::Filter)
292
292
  pipeline ||= Pipeline.new
@@ -301,7 +301,7 @@ module Fluent
301
301
  return pipeline
302
302
  end
303
303
  end
304
- }
304
+ end
305
305
 
306
306
  if pipeline
307
307
  # filter is matched but no match
@@ -19,11 +19,9 @@ require 'console'
19
19
  module Fluent
20
20
  class Log
21
21
  # Async gem which is used by http_server helper switched logger mechanism to
22
- # Console gem which isn't complatible with Ruby's standard Logger (since
22
+ # Console gem which isn't compatible with Ruby's standard Logger (since
23
23
  # v1.17). This class adapts it to Fluentd's logger mechanism.
24
- class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
25
- Console::Output::Terminal : Console::Terminal::Logger
26
-
24
+ class ConsoleAdapter < Console::Output::Terminal
27
25
  def self.wrap(logger)
28
26
  _, level = Console::Logger::LEVELS.find { |key, value|
29
27
  if logger.level <= 0
@@ -58,10 +56,10 @@ module Fluent
58
56
  level = 'warn'
59
57
  end
60
58
 
61
- @io.seek(0)
62
- @io.truncate(0)
59
+ @stream.seek(0)
60
+ @stream.truncate(0)
63
61
  super
64
- @logger.send(level, @io.string.chomp)
62
+ @logger.send(level, @stream.string.chomp)
65
63
  end
66
64
  end
67
65
  end
data/lib/fluent/log.rb CHANGED
@@ -138,6 +138,7 @@ module Fluent
138
138
  @optional_attrs = nil
139
139
 
140
140
  @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
141
+ @forced_stacktrace_level = nil
141
142
  @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
142
143
  @ignore_same_log_interval = opts[:ignore_same_log_interval]
143
144
 
@@ -173,6 +174,7 @@ module Fluent
173
174
  clone.format = @format
174
175
  clone.time_format = @time_format
175
176
  clone.log_event_enabled = @log_event_enabled
177
+ clone.force_stacktrace_level(@forced_stacktrace_level)
176
178
  # optional headers/attrs are not copied, because new PluginLogger should have another one of it
177
179
  clone
178
180
  end
@@ -240,6 +242,14 @@ module Fluent
240
242
  nil
241
243
  end
242
244
 
245
+ def force_stacktrace_level?
246
+ not @forced_stacktrace_level.nil?
247
+ end
248
+
249
+ def force_stacktrace_level(level)
250
+ @forced_stacktrace_level = level
251
+ end
252
+
243
253
  def enable_debug(b=true)
244
254
  @debug_mode = b
245
255
  self
@@ -500,6 +510,16 @@ module Fluent
500
510
  def dump_stacktrace(type, backtrace, level)
501
511
  return if @level > level
502
512
 
513
+ dump_stacktrace_internal(
514
+ type,
515
+ backtrace,
516
+ force_stacktrace_level? ? @forced_stacktrace_level : level,
517
+ )
518
+ end
519
+
520
+ def dump_stacktrace_internal(type, backtrace, level)
521
+ return if @level > level
522
+
503
523
  time = Time.now
504
524
 
505
525
  if @format == :text
@@ -633,6 +653,9 @@ module Fluent
633
653
  if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
634
654
  @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
635
655
  end
656
+ if logger.instance_variable_defined?(:@forced_stacktrace_level)
657
+ @forced_stacktrace_level = logger.instance_variable_get(:@forced_stacktrace_level)
658
+ end
636
659
  if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
637
660
  @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
638
661
  end
@@ -40,22 +40,6 @@ module Fluent
40
40
  raise NotImplementedError, "BUG: output plugins MUST implement this method"
41
41
  end
42
42
 
43
- def num_errors
44
- @num_errors_metrics.get
45
- end
46
-
47
- def emit_count
48
- @emit_count_metrics.get
49
- end
50
-
51
- def emit_size
52
- @emit_size_metrics.get
53
- end
54
-
55
- def emit_records
56
- @emit_records_metrics.get
57
- end
58
-
59
43
  def initialize
60
44
  super
61
45
  @counter_mutex = Mutex.new
@@ -84,7 +84,7 @@ module Fluent
84
84
  yield
85
85
  end
86
86
  # Update access time to prevent tmpwatch from deleting a lock file.
87
- FileUtils.touch(lock_path);
87
+ FileUtils.touch(lock_path)
88
88
  end
89
89
 
90
90
  def string_safe_encoding(str)
@@ -206,7 +206,7 @@ module Fluent
206
206
  end
207
207
 
208
208
  def reloadable_plugin?
209
- # Engine can't capture all class variables. so it's forbbiden to use class variables in each plugins if enabling reload.
209
+ # Engine can't capture all class variables. so it's forbidden to use class variables in each plugins if enabling reload.
210
210
  self.class.class_variables.empty?
211
211
  end
212
212
  end
@@ -191,7 +191,7 @@ module Fluent
191
191
  queue.sort_by!{ |chunk| chunk.modified_at }
192
192
 
193
193
  # If one of the files is corrupted, other files may also be corrupted and be undetected.
194
- # The time priods of each chunk are helpful to check the data.
194
+ # The time periods of each chunk are helpful to check the data.
195
195
  if exist_broken_file
196
196
  log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
197
197
  (stage.values + queue).each { |chunk|
@@ -229,6 +229,20 @@ module Fluent
229
229
  File.unlink(path, path + '.meta') rescue nil
230
230
  end
231
231
 
232
+ def evacuate_chunk(chunk)
233
+ unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk)
234
+ raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}."
235
+ end
236
+
237
+ backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
238
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
239
+
240
+ FileUtils.copy([chunk.path, chunk.meta_path], backup_dir)
241
+ log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
242
+ rescue => e
243
+ log.error "unexpected error while evacuating chunk files.", error: e
244
+ end
245
+
232
246
  private
233
247
 
234
248
  def escaped_patterns(patterns)
@@ -202,7 +202,7 @@ module Fluent
202
202
  queue.sort_by!(&:modified_at)
203
203
 
204
204
  # If one of the files is corrupted, other files may also be corrupted and be undetected.
205
- # The time priods of each chunk are helpful to check the data.
205
+ # The time periods of each chunk are helpful to check the data.
206
206
  if exist_broken_file
207
207
  log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
208
208
  (stage.values + queue).each { |chunk|
@@ -241,6 +241,20 @@ module Fluent
241
241
  File.unlink(path) rescue nil
242
242
  end
243
243
 
244
+ def evacuate_chunk(chunk)
245
+ unless chunk.is_a?(Fluent::Plugin::Buffer::FileSingleChunk)
246
+ raise ArgumentError, "The chunk must be FileSingleChunk, but it was #{chunk.class}."
247
+ end
248
+
249
+ backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
250
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
251
+
252
+ FileUtils.copy(chunk.path, backup_dir)
253
+ log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
254
+ rescue => e
255
+ log.error "unexpected error while evacuating chunk files.", error: e
256
+ end
257
+
244
258
  private
245
259
 
246
260
  def escaped_patterns(patterns)