fluentd 1.17.1-x86-mingw32 → 1.19.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +116 -0
  3. data/CHANGELOG.md +293 -16
  4. data/MAINTAINERS.md +8 -2
  5. data/README.md +3 -7
  6. data/Rakefile +2 -0
  7. data/SECURITY.md +5 -3
  8. data/lib/fluent/command/cap_ctl.rb +2 -2
  9. data/lib/fluent/command/fluentd.rb +13 -3
  10. data/lib/fluent/compat/formatter.rb +6 -0
  11. data/lib/fluent/compat/socket_util.rb +2 -2
  12. data/lib/fluent/config/configure_proxy.rb +1 -1
  13. data/lib/fluent/config/element.rb +2 -2
  14. data/lib/fluent/config/literal_parser.rb +12 -5
  15. data/lib/fluent/config/parser.rb +15 -3
  16. data/lib/fluent/config/section.rb +2 -2
  17. data/lib/fluent/config/types.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +3 -3
  19. data/lib/fluent/counter/store.rb +1 -1
  20. data/lib/fluent/engine.rb +50 -34
  21. data/lib/fluent/env.rb +6 -2
  22. data/lib/fluent/event.rb +7 -6
  23. data/lib/fluent/event_router.rb +2 -2
  24. data/lib/fluent/log/console_adapter.rb +5 -7
  25. data/lib/fluent/log.rb +23 -0
  26. data/lib/fluent/plugin/bare_output.rb +0 -16
  27. data/lib/fluent/plugin/base.rb +2 -2
  28. data/lib/fluent/plugin/buf_file.rb +15 -1
  29. data/lib/fluent/plugin/buf_file_single.rb +15 -1
  30. data/lib/fluent/plugin/buffer/chunk.rb +74 -10
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +9 -5
  32. data/lib/fluent/plugin/buffer/file_single_chunk.rb +3 -3
  33. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -2
  34. data/lib/fluent/plugin/buffer.rb +34 -6
  35. data/lib/fluent/plugin/compressable.rb +68 -22
  36. data/lib/fluent/plugin/filter.rb +0 -8
  37. data/lib/fluent/plugin/filter_parser.rb +27 -51
  38. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  39. data/lib/fluent/plugin/formatter_csv.rb +18 -4
  40. data/lib/fluent/plugin/formatter_json.rb +7 -4
  41. data/lib/fluent/plugin/formatter_out_file.rb +5 -2
  42. data/lib/fluent/plugin/in_forward.rb +9 -5
  43. data/lib/fluent/plugin/in_http.rb +14 -4
  44. data/lib/fluent/plugin/in_monitor_agent.rb +4 -8
  45. data/lib/fluent/plugin/in_syslog.rb +4 -0
  46. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  47. data/lib/fluent/plugin/in_tail.rb +80 -57
  48. data/lib/fluent/plugin/in_tcp.rb +6 -2
  49. data/lib/fluent/plugin/in_udp.rb +11 -2
  50. data/lib/fluent/plugin/input.rb +4 -8
  51. data/lib/fluent/plugin/multi_output.rb +1 -17
  52. data/lib/fluent/plugin/out_buffer.rb +40 -0
  53. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  54. data/lib/fluent/plugin/out_file.rb +37 -30
  55. data/lib/fluent/plugin/out_forward/connection_manager.rb +2 -2
  56. data/lib/fluent/plugin/out_forward.rb +23 -13
  57. data/lib/fluent/plugin/out_http.rb +1 -1
  58. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  59. data/lib/fluent/plugin/out_stdout.rb +10 -3
  60. data/lib/fluent/plugin/out_stream.rb +3 -3
  61. data/lib/fluent/plugin/output.rb +26 -35
  62. data/lib/fluent/plugin/owned_by_mixin.rb +2 -2
  63. data/lib/fluent/plugin/parser.rb +3 -3
  64. data/lib/fluent/plugin/parser_json.rb +3 -3
  65. data/lib/fluent/plugin/sd_file.rb +2 -2
  66. data/lib/fluent/plugin/storage_local.rb +8 -4
  67. data/lib/fluent/plugin.rb +1 -1
  68. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  69. data/lib/fluent/plugin_helper/child_process.rb +2 -2
  70. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  71. data/lib/fluent/plugin_helper/http_server/request.rb +13 -2
  72. data/lib/fluent/plugin_helper/http_server/server.rb +14 -8
  73. data/lib/fluent/plugin_helper/http_server.rb +1 -8
  74. data/lib/fluent/plugin_helper/metrics.rb +7 -0
  75. data/lib/fluent/plugin_helper/server.rb +13 -1
  76. data/lib/fluent/plugin_helper/service_discovery.rb +1 -1
  77. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  78. data/lib/fluent/plugin_helper/storage.rb +1 -1
  79. data/lib/fluent/plugin_id.rb +3 -3
  80. data/lib/fluent/root_agent.rb +117 -21
  81. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  82. data/lib/fluent/static_config_analysis.rb +3 -2
  83. data/lib/fluent/supervisor.rb +258 -39
  84. data/lib/fluent/system_config.rb +27 -6
  85. data/lib/fluent/test/base.rb +1 -1
  86. data/lib/fluent/test/driver/base.rb +2 -2
  87. data/lib/fluent/test/filter_test.rb +2 -2
  88. data/lib/fluent/test/formatter_test.rb +1 -1
  89. data/lib/fluent/test/helpers.rb +4 -0
  90. data/lib/fluent/test/input_test.rb +2 -2
  91. data/lib/fluent/test/output_test.rb +4 -4
  92. data/lib/fluent/test/parser_test.rb +1 -1
  93. data/lib/fluent/tls.rb +24 -0
  94. data/lib/fluent/variable_store.rb +1 -1
  95. data/lib/fluent/version.rb +1 -1
  96. data/lib/fluent/winsvc.rb +38 -8
  97. metadata +99 -28
  98. data/lib/fluent/plugin_helper/http_server/compat/server.rb +0 -92
  99. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +0 -52
  100. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +0 -58
@@ -56,22 +56,6 @@ module Fluent
56
56
  @enable_size_metrics = false
57
57
  end
58
58
 
59
- def num_errors
60
- @num_errors_metrics.get
61
- end
62
-
63
- def emit_count
64
- @emit_count_metrics.get
65
- end
66
-
67
- def emit_size
68
- @emit_size_metrics.get
69
- end
70
-
71
- def emit_records
72
- @emit_records_metrics.get
73
- end
74
-
75
59
  def statistics
76
60
  stats = {
77
61
  'num_errors' => @num_errors_metrics.get,
@@ -91,7 +75,7 @@ module Fluent
91
75
  super
92
76
 
93
77
  @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
94
- @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
78
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_count", help_text: "Number of count emits")
95
79
  @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
96
80
  @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
97
81
  @enable_size_metrics = !!system_config.enable_size_metrics
@@ -0,0 +1,40 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class BufferOutput < Output
21
+ Fluent::Plugin.register_output("buffer", self)
22
+ helpers :event_emitter
23
+
24
+ config_section :buffer do
25
+ config_set_default :@type, "file"
26
+ config_set_default :chunk_keys, ["tag"]
27
+ config_set_default :flush_mode, :interval
28
+ config_set_default :flush_interval, 10
29
+ end
30
+
31
+ def multi_workers_ready?
32
+ true
33
+ end
34
+
35
+ def write(chunk)
36
+ return if chunk.empty?
37
+ router.emit_stream(chunk.metadata.tag, Fluent::MessagePackEventStream.new(chunk.read))
38
+ end
39
+ end
40
+ end
@@ -193,8 +193,8 @@ module Fluent::Plugin
193
193
  log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
194
194
  end
195
195
  c.mutex.synchronize do
196
- (c.writeio && c.writeio.close) rescue nil
197
- (c.readio && c.readio.close) rescue nil
196
+ c.writeio&.close rescue nil
197
+ c.readio&.close rescue nil
198
198
  c.pid = c.readio = c.writeio = nil
199
199
  end
200
200
  end
@@ -17,6 +17,7 @@
17
17
  require 'fileutils'
18
18
  require 'zlib'
19
19
  require 'time'
20
+ require 'pathname'
20
21
 
21
22
  require 'fluent/plugin/output'
22
23
  require 'fluent/config/error'
@@ -29,11 +30,12 @@ module Fluent::Plugin
29
30
 
30
31
  helpers :formatter, :inject, :compat_parameters
31
32
 
32
- SUPPORTED_COMPRESS = [:text, :gz, :gzip]
33
+ SUPPORTED_COMPRESS = [:text, :gz, :gzip, :zstd]
33
34
  SUPPORTED_COMPRESS_MAP = {
34
35
  text: nil,
35
36
  gz: :gzip,
36
37
  gzip: :gzip,
38
+ zstd: :zstd,
37
39
  }
38
40
 
39
41
  DEFAULT_TIMEKEY = 60 * 60 * 24
@@ -53,6 +55,8 @@ module Fluent::Plugin
53
55
  config_param :recompress, :bool, default: false
54
56
  desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
55
57
  config_param :symlink_path, :string, default: nil
58
+ desc "Use relative path for symlink target (default: false)"
59
+ config_param :symlink_path_use_relative, :bool, default: false
56
60
 
57
61
  config_section :format do
58
62
  config_set_default :@type, 'out_file'
@@ -96,7 +100,12 @@ module Fluent::Plugin
96
100
  if chunk.metadata == @latest_metadata
97
101
  sym_path = @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk)
98
102
  FileUtils.mkdir_p(File.dirname(sym_path), mode: @_output_plugin_for_symlink.dir_perm)
99
- FileUtils.ln_sf(chunk.path, sym_path)
103
+ if @_output_plugin_for_symlink.symlink_path_use_relative
104
+ relative_path = Pathname.new(chunk.path).relative_path_from(Pathname.new(File.dirname(sym_path)))
105
+ FileUtils.ln_sf(relative_path, sym_path)
106
+ else
107
+ FileUtils.ln_sf(chunk.path, sym_path)
108
+ end
100
109
  end
101
110
  chunk
102
111
  end
@@ -184,18 +193,15 @@ module Fluent::Plugin
184
193
  @buffer.symlink_path = @symlink_path
185
194
  @buffer.output_plugin_for_symlink = self
186
195
  end
196
+
197
+ if @compress != :text && @buffer.compress != :text && @buffer.compress != @compress_method
198
+ raise Fluent::ConfigError, "You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})"
199
+ end
187
200
  end
188
201
 
189
202
  @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
190
203
  @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
191
204
  @need_lock = system_config.workers > 1
192
-
193
- # https://github.com/fluent/fluentd/issues/3569
194
- @need_ruby_on_macos_workaround = false
195
- if @append && Fluent.macos?
196
- condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
197
- @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
198
- end
199
205
  end
200
206
 
201
207
  def multi_workers_ready?
@@ -212,17 +218,17 @@ module Fluent::Plugin
212
218
  FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
213
219
 
214
220
  writer = case
215
- when @compress_method.nil?
216
- method(:write_without_compression)
217
- when @compress_method == :gzip
218
- if @buffer.compress != :gzip || @recompress
219
- method(:write_gzip_with_compression)
220
- else
221
- method(:write_gzip_from_gzipped_chunk)
222
- end
223
- else
224
- raise "BUG: unknown compression method #{@compress_method}"
225
- end
221
+ when @compress_method.nil?
222
+ method(:write_without_compression)
223
+ when @compress_method != :text
224
+ if @buffer.compress == :text || @recompress
225
+ method(:write_with_compression).curry.call(@compress_method)
226
+ else
227
+ method(:write_from_compressed_chunk).curry.call(@compress_method)
228
+ end
229
+ else
230
+ raise "BUG: unknown compression method #{@compress_method}"
231
+ end
226
232
 
227
233
  if @append
228
234
  if @need_lock
@@ -244,26 +250,26 @@ module Fluent::Plugin
244
250
 
245
251
  def write_without_compression(path, chunk)
246
252
  File.open(path, "ab", @file_perm) do |f|
247
- if @need_ruby_on_macos_workaround
248
- content = chunk.read()
249
- f.puts content
250
- else
251
- chunk.write_to(f)
252
- end
253
+ chunk.write_to(f)
253
254
  end
254
255
  end
255
256
 
256
- def write_gzip_with_compression(path, chunk)
257
+ def write_with_compression(type, path, chunk)
257
258
  File.open(path, "ab", @file_perm) do |f|
258
- gz = Zlib::GzipWriter.new(f)
259
+ gz = nil
260
+ if type == :gzip
261
+ gz = Zlib::GzipWriter.new(f)
262
+ elsif type == :zstd
263
+ gz = Zstd::StreamWriter.new(f)
264
+ end
259
265
  chunk.write_to(gz, compressed: :text)
260
266
  gz.close
261
267
  end
262
268
  end
263
269
 
264
- def write_gzip_from_gzipped_chunk(path, chunk)
270
+ def write_from_compressed_chunk(type, path, chunk)
265
271
  File.open(path, "ab", @file_perm) do |f|
266
- chunk.write_to(f, compressed: :gzip)
272
+ chunk.write_to(f, compressed: type)
267
273
  end
268
274
  end
269
275
 
@@ -280,6 +286,7 @@ module Fluent::Plugin
280
286
  def compression_suffix(compress)
281
287
  case compress
282
288
  when :gzip then '.gz'
289
+ when :zstd then '.zstd'
283
290
  when nil then ''
284
291
  else
285
292
  raise ArgumentError, "unknown compression type #{compress}"
@@ -24,7 +24,7 @@ module Fluent::Plugin
24
24
  # @param log [Logger]
25
25
  # @param secure [Boolean]
26
26
  # @param connection_factory [Proc]
27
- # @param SocketCache [Fluent::ForwardOutput::SocketCache]
27
+ # @param socket_cache [Fluent::ForwardOutput::SocketCache]
28
28
  def initialize(log:, secure:, connection_factory:, socket_cache:)
29
29
  @log = log
30
30
  @secure = secure
@@ -36,7 +36,7 @@ module Fluent::Plugin
36
36
  @socket_cache && @socket_cache.clear
37
37
  end
38
38
 
39
- # @param ack [Fluent::Plugin::ForwardOutput::AckHander::Ack|nil]
39
+ # @param ack [Fluent::Plugin::ForwardOutput::AckHandler::Ack|nil]
40
40
  def connect(host:, port:, hostname:, ack: nil, &block)
41
41
  if @socket_cache
42
42
  return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)
@@ -87,7 +87,7 @@ module Fluent::Plugin
87
87
  config_param :verify_connection_at_startup, :bool, default: false
88
88
 
89
89
  desc 'Compress buffered data.'
90
- config_param :compress, :enum, list: [:text, :gzip], default: :text
90
+ config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
91
91
 
92
92
  desc 'The default version of TLS transport.'
93
93
  config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
@@ -251,10 +251,14 @@ module Fluent::Plugin
251
251
  end
252
252
 
253
253
  unless @as_secondary
254
- if @compress == :gzip && @buffer.compress == :text
255
- @buffer.compress = :gzip
256
- elsif @compress == :text && @buffer.compress == :gzip
257
- log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
254
+ if @buffer.compress == :text
255
+ @buffer.compress = @compress
256
+ else
257
+ if @compress == :text
258
+ log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
259
+ elsif @compress != @buffer.compress
260
+ raise Fluent::ConfigError, "You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})"
261
+ end
258
262
  end
259
263
  end
260
264
 
@@ -267,9 +271,15 @@ module Fluent::Plugin
267
271
  end
268
272
 
269
273
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
274
+
275
+ if @compress == :zstd
276
+ log.warn "zstd compression feature is an experimental new feature supported since v1.19.0." +
277
+ " Please make sure that the destination server also supports this feature before using it." +
278
+ " in_forward plugin for Fluentd supports it since v1.19.0."
279
+ end
280
+
270
281
  @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
271
282
  @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
272
-
273
283
  end
274
284
 
275
285
  def multi_workers_ready?
@@ -295,7 +305,7 @@ module Fluent::Plugin
295
305
  unless @heartbeat_type == :none
296
306
  if @heartbeat_type == :udp
297
307
  @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)
298
- server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
308
+ server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heartbeat_response_recv))
299
309
  end
300
310
  timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
301
311
  end
@@ -481,7 +491,7 @@ module Fluent::Plugin
481
491
  end
482
492
  end
483
493
 
484
- def on_udp_heatbeat_response_recv(data, sock)
494
+ def on_udp_heartbeat_response_recv(data, sock)
485
495
  sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
486
496
  if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
487
497
  # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
@@ -567,8 +577,8 @@ module Fluent::Plugin
567
577
 
568
578
  @handshake = HandshakeProtocol.new(
569
579
  log: @log,
570
- hostname: sender.security && sender.security.self_hostname,
571
- shared_key: server.shared_key || (sender.security && sender.security.shared_key) || '',
580
+ hostname: sender.security&.self_hostname,
581
+ shared_key: server.shared_key || sender.security&.shared_key || '',
572
582
  password: server.password || '',
573
583
  username: server.username || '',
574
584
  )
@@ -584,7 +594,7 @@ module Fluent::Plugin
584
594
  attr_accessor :usock
585
595
 
586
596
  attr_reader :state
587
- attr_reader :sockaddr # used by on_udp_heatbeat_response_recv
597
+ attr_reader :sockaddr # used by on_udp_heartbeat_response_recv
588
598
  attr_reader :failure # for test
589
599
 
590
600
  def validate_host_resolution!
@@ -711,7 +721,7 @@ module Fluent::Plugin
711
721
  end
712
722
  when :udp
713
723
  @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, dest_addr)
714
- # response is going to receive at on_udp_heatbeat_response_recv
724
+ # response is going to receive at on_udp_heartbeat_response_recv
715
725
  false
716
726
  when :none # :none doesn't use this class
717
727
  raise "BUG: heartbeat_type none must not use Node"
@@ -744,7 +754,7 @@ module Fluent::Plugin
744
754
  def resolve_dns!
745
755
  addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)
746
756
  addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
747
- @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heatbeat_response_recv
757
+ @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heartbeat_response_recv
748
758
  addrinfo[3]
749
759
  end
750
760
  private :resolve_dns!
@@ -270,7 +270,7 @@ module Fluent::Plugin
270
270
  OpenSSL::SSL::VERIFY_PEER
271
271
  end
272
272
  opt[:ciphers] = @tls_ciphers
273
- opt[:ssl_version] = @tls_version
273
+ opt = Fluent::TLS.set_version_to_options(opt, @tls_version, nil, nil)
274
274
  end
275
275
 
276
276
  opt
@@ -98,11 +98,11 @@ module Fluent::Plugin
98
98
  raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
99
99
  end
100
100
 
101
- if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
101
+ if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match?(/tag(\[\d+\])?/) })
102
102
  raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
103
103
  end
104
104
 
105
- vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
105
+ vars = placeholders.reject { |placeholder| placeholder.match?(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
106
106
 
107
107
  if ph = vars.find { |v| !@chunk_keys.include?(v) }
108
108
  raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
@@ -25,6 +25,9 @@ module Fluent::Plugin
25
25
  DEFAULT_LINE_FORMAT_TYPE = 'stdout'
26
26
  DEFAULT_FORMAT_TYPE = 'json'
27
27
 
28
+ desc "If Fluentd logger outputs logs to a file (with -o option), this plugin outputs events to the file as well."
29
+ config_param :use_logger, :bool, default: true
30
+
28
31
  config_section :buffer do
29
32
  config_set_default :chunk_keys, ['tag']
30
33
  config_set_default :flush_at_shutdown, true
@@ -44,6 +47,10 @@ module Fluent::Plugin
44
47
  true
45
48
  end
46
49
 
50
+ def dest_io
51
+ @use_logger ? $log : $stdout
52
+ end
53
+
47
54
  attr_accessor :formatter
48
55
 
49
56
  def configure(conf)
@@ -57,9 +64,9 @@ module Fluent::Plugin
57
64
  def process(tag, es)
58
65
  es = inject_values_to_event_stream(tag, es)
59
66
  es.each {|time,record|
60
- $log.write(format(tag, time, record))
67
+ dest_io.write(format(tag, time, record))
61
68
  }
62
- $log.flush
69
+ dest_io.flush
63
70
  end
64
71
 
65
72
  def format(tag, time, record)
@@ -68,7 +75,7 @@ module Fluent::Plugin
68
75
  end
69
76
 
70
77
  def write(chunk)
71
- chunk.write_to($log)
78
+ chunk.write_to(dest_io)
72
79
  end
73
80
  end
74
81
  end
@@ -92,8 +92,8 @@ module Fluent
92
92
 
93
93
  def initialize
94
94
  super
95
- $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
96
- $log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
95
+ log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
96
+ log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
97
97
  end
98
98
 
99
99
  config_param :port, :integer, default: LISTEN_PORT
@@ -114,7 +114,7 @@ module Fluent
114
114
 
115
115
  def initialize
116
116
  super
117
- $log.warn "'unix' output is obsoleted and will be removed."
117
+ log.warn "'unix' output is obsoleted and will be removed."
118
118
  end
119
119
 
120
120
  config_param :path, :string
@@ -171,30 +171,6 @@ module Fluent
171
171
  # output_enqueue_thread_waiting: for test of output.rb itself
172
172
  attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
173
173
 
174
- def num_errors
175
- @num_errors_metrics.get
176
- end
177
-
178
- def emit_count
179
- @emit_count_metrics.get
180
- end
181
-
182
- def emit_size
183
- @emit_size_metrics.get
184
- end
185
-
186
- def emit_records
187
- @emit_records_metrics.get
188
- end
189
-
190
- def write_count
191
- @write_count_metrics.get
192
- end
193
-
194
- def rollback_count
195
- @rollback_count_metrics.get
196
- end
197
-
198
174
  def initialize
199
175
  super
200
176
  @counter_mutex = Mutex.new
@@ -210,9 +186,11 @@ module Fluent
210
186
  @emit_records_metrics = nil
211
187
  @emit_size_metrics = nil
212
188
  @write_count_metrics = nil
189
+ @write_secondary_count_metrics = nil
213
190
  @rollback_count_metrics = nil
214
191
  @flush_time_count_metrics = nil
215
192
  @slow_flush_count_metrics = nil
193
+ @drop_oldest_chunk_count_metrics = nil
216
194
  @enable_size_metrics = false
217
195
 
218
196
  # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
@@ -278,9 +256,11 @@ module Fluent
278
256
  @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
279
257
  @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
280
258
  @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
259
+ @write_secondary_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_secondary_count", help_text: "Number of writing events in secondary")
281
260
  @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
282
261
  @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
283
262
  @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
263
+ @drop_oldest_chunk_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "drop_oldest_chunk_count", help_text: "Number of count that old chunk were discarded with drop_oldest_chunk")
284
264
 
285
265
  if has_buffer_section
286
266
  unless implement?(:buffered) || implement?(:delayed_commit)
@@ -572,7 +552,7 @@ module Fluent
572
552
  @output_flush_threads.each do |state|
573
553
  # to wakeup thread and make it to stop by itself
574
554
  state.mutex.synchronize {
575
- if state.thread && state.thread.status
555
+ if state.thread&.status
576
556
  state.next_clock = 0
577
557
  state.cond_var.signal
578
558
  end
@@ -999,6 +979,7 @@ module Fluent
999
979
  if oldest
1000
980
  log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
1001
981
  @buffer.purge_chunk(oldest.unique_id)
982
+ @drop_oldest_chunk_count_metrics.inc
1002
983
  else
1003
984
  log.error "no queued chunks to be dropped for drop_oldest_chunk"
1004
985
  end
@@ -1014,13 +995,17 @@ module Fluent
1014
995
  end
1015
996
 
1016
997
  FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
1017
- FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
998
+ FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
999
+ FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }
1018
1000
  FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
1019
- FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
1001
+ FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
1002
+ FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }
1020
1003
 
1021
1004
  def generate_format_proc
1022
1005
  if @buffer && @buffer.compress == :gzip
1023
- @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
1006
+ @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP : FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP
1007
+ elsif @buffer && @buffer.compress == :zstd
1008
+ @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD : FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD
1024
1009
  else
1025
1010
  @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
1026
1011
  end
@@ -1036,17 +1021,17 @@ module Fluent
1036
1021
  # iteration of event stream, and it should be done just once even if total event stream size
1037
1022
  # is bigger than chunk_limit_size because of performance.
1038
1023
  def handle_stream_with_custom_format(tag, es, enqueue: false)
1039
- meta_and_data = {}
1024
+ meta_and_data = Hash.new { |h, k| h[k] = [] }
1040
1025
  records = 0
1041
1026
  es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
1042
1027
  meta = metadata(tag, time, record)
1043
- meta_and_data[meta] ||= []
1044
1028
  res = format(tag, time, record)
1045
1029
  if res
1046
1030
  meta_and_data[meta] << res
1047
1031
  records += 1
1048
1032
  end
1049
1033
  end
1034
+ meta_and_data.default_proc = nil
1050
1035
  write_guard do
1051
1036
  @buffer.write(meta_and_data, enqueue: enqueue)
1052
1037
  end
@@ -1057,14 +1042,14 @@ module Fluent
1057
1042
 
1058
1043
  def handle_stream_with_standard_format(tag, es, enqueue: false)
1059
1044
  format_proc = generate_format_proc
1060
- meta_and_data = {}
1045
+ meta_and_data = Hash.new { |h, k| h[k] = MultiEventStream.new }
1061
1046
  records = 0
1062
1047
  es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
1063
1048
  meta = metadata(tag, time, record)
1064
- meta_and_data[meta] ||= MultiEventStream.new
1065
1049
  meta_and_data[meta].add(time, record)
1066
1050
  records += 1
1067
1051
  end
1052
+ meta_and_data.default_proc = nil
1068
1053
  write_guard do
1069
1054
  @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
1070
1055
  end
@@ -1146,7 +1131,7 @@ module Fluent
1146
1131
 
1147
1132
  def try_rollback_write
1148
1133
  @dequeued_chunks_mutex.synchronize do
1149
- while @dequeued_chunks.first && @dequeued_chunks.first.expired?
1134
+ while @dequeued_chunks.first&.expired?
1150
1135
  info = @dequeued_chunks.shift
1151
1136
  if @buffer.takeback_chunk(info.chunk_id)
1152
1137
  @rollback_count_metrics.inc
@@ -1208,6 +1193,7 @@ module Fluent
1208
1193
  if output.delayed_commit
1209
1194
  log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
1210
1195
  @write_count_metrics.inc
1196
+ @write_secondary_count_metrics.inc if using_secondary
1211
1197
  @dequeued_chunks_mutex.synchronize do
1212
1198
  # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
1213
1199
  @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1220,6 +1206,7 @@ module Fluent
1220
1206
  dump_chunk_id = dump_unique_id_hex(chunk_id)
1221
1207
  log.trace "adding write count", instance: self.object_id
1222
1208
  @write_count_metrics.inc
1209
+ @write_secondary_count_metrics.inc if using_secondary
1223
1210
  log.trace "executing sync write", chunk: dump_chunk_id
1224
1211
 
1225
1212
  output.write(chunk)
@@ -1384,11 +1371,12 @@ module Fluent
1384
1371
  end
1385
1372
 
1386
1373
  def submit_flush_once
1374
+ return unless @buffer_config.flush_thread_count > 0
1387
1375
  # Without locks: it is rough but enough to select "next" writer selection
1388
1376
  @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
1389
1377
  state = @output_flush_threads[@output_flush_thread_current_position]
1390
1378
  state.mutex.synchronize {
1391
- if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
1379
+ if state.thread&.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
1392
1380
  state.next_clock = 0
1393
1381
  state.cond_var.signal
1394
1382
  else
@@ -1406,6 +1394,7 @@ module Fluent
1406
1394
  end
1407
1395
 
1408
1396
  def submit_flush_all
1397
+ return unless @buffer_config.flush_thread_count > 0
1409
1398
  while !@retry && @buffer.queued?
1410
1399
  submit_flush_once
1411
1400
  sleep @buffer_config.flush_thread_burst_interval
@@ -1433,7 +1422,7 @@ module Fluent
1433
1422
  def flush_thread_wakeup
1434
1423
  @output_flush_threads.each do |state|
1435
1424
  state.mutex.synchronize {
1436
- if state.thread && state.thread.status
1425
+ if state.thread&.status
1437
1426
  state.next_clock = 0
1438
1427
  state.cond_var.signal
1439
1428
  end
@@ -1585,9 +1574,11 @@ module Fluent
1585
1574
  'retry_count' => @num_errors_metrics.get,
1586
1575
  'emit_count' => @emit_count_metrics.get,
1587
1576
  'write_count' => @write_count_metrics.get,
1577
+ 'write_secondary_count' => @write_secondary_count_metrics.get,
1588
1578
  'rollback_count' => @rollback_count_metrics.get,
1589
1579
  'slow_flush_count' => @slow_flush_count_metrics.get,
1590
1580
  'flush_time_count' => @flush_time_count_metrics.get,
1581
+ 'drop_oldest_chunk_count' => @drop_oldest_chunk_count_metrics.get,
1591
1582
  }
1592
1583
 
1593
1584
  if @buffer && @buffer.respond_to?(:statistics)
@@ -26,13 +26,13 @@ module Fluent
26
26
  end
27
27
 
28
28
  def owner
29
- if instance_variable_defined?("@_owner")
29
+ if instance_variable_defined?(:@_owner)
30
30
  @_owner
31
31
  end
32
32
  end
33
33
 
34
34
  def log
35
- if instance_variable_defined?("@log")
35
+ if instance_variable_defined?(:@log)
36
36
  @log
37
37
  end
38
38
  end