fluentd 1.17.1-x64-mingw-ucrt → 1.19.0-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +116 -0
  3. data/CHANGELOG.md +293 -16
  4. data/MAINTAINERS.md +8 -2
  5. data/README.md +3 -7
  6. data/Rakefile +2 -0
  7. data/SECURITY.md +5 -3
  8. data/lib/fluent/command/cap_ctl.rb +2 -2
  9. data/lib/fluent/command/fluentd.rb +13 -3
  10. data/lib/fluent/compat/formatter.rb +6 -0
  11. data/lib/fluent/compat/socket_util.rb +2 -2
  12. data/lib/fluent/config/configure_proxy.rb +1 -1
  13. data/lib/fluent/config/element.rb +2 -2
  14. data/lib/fluent/config/literal_parser.rb +12 -5
  15. data/lib/fluent/config/parser.rb +15 -3
  16. data/lib/fluent/config/section.rb +2 -2
  17. data/lib/fluent/config/types.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +3 -3
  19. data/lib/fluent/counter/store.rb +1 -1
  20. data/lib/fluent/engine.rb +50 -34
  21. data/lib/fluent/env.rb +6 -2
  22. data/lib/fluent/event.rb +7 -6
  23. data/lib/fluent/event_router.rb +2 -2
  24. data/lib/fluent/log/console_adapter.rb +5 -7
  25. data/lib/fluent/log.rb +23 -0
  26. data/lib/fluent/plugin/bare_output.rb +0 -16
  27. data/lib/fluent/plugin/base.rb +2 -2
  28. data/lib/fluent/plugin/buf_file.rb +15 -1
  29. data/lib/fluent/plugin/buf_file_single.rb +15 -1
  30. data/lib/fluent/plugin/buffer/chunk.rb +74 -10
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +9 -5
  32. data/lib/fluent/plugin/buffer/file_single_chunk.rb +3 -3
  33. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -2
  34. data/lib/fluent/plugin/buffer.rb +34 -6
  35. data/lib/fluent/plugin/compressable.rb +68 -22
  36. data/lib/fluent/plugin/filter.rb +0 -8
  37. data/lib/fluent/plugin/filter_parser.rb +27 -51
  38. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  39. data/lib/fluent/plugin/formatter_csv.rb +18 -4
  40. data/lib/fluent/plugin/formatter_json.rb +7 -4
  41. data/lib/fluent/plugin/formatter_out_file.rb +5 -2
  42. data/lib/fluent/plugin/in_forward.rb +9 -5
  43. data/lib/fluent/plugin/in_http.rb +14 -4
  44. data/lib/fluent/plugin/in_monitor_agent.rb +4 -8
  45. data/lib/fluent/plugin/in_syslog.rb +4 -0
  46. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  47. data/lib/fluent/plugin/in_tail.rb +80 -57
  48. data/lib/fluent/plugin/in_tcp.rb +6 -2
  49. data/lib/fluent/plugin/in_udp.rb +11 -2
  50. data/lib/fluent/plugin/input.rb +4 -8
  51. data/lib/fluent/plugin/multi_output.rb +1 -17
  52. data/lib/fluent/plugin/out_buffer.rb +40 -0
  53. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  54. data/lib/fluent/plugin/out_file.rb +37 -30
  55. data/lib/fluent/plugin/out_forward/connection_manager.rb +2 -2
  56. data/lib/fluent/plugin/out_forward.rb +23 -13
  57. data/lib/fluent/plugin/out_http.rb +1 -1
  58. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  59. data/lib/fluent/plugin/out_stdout.rb +10 -3
  60. data/lib/fluent/plugin/out_stream.rb +3 -3
  61. data/lib/fluent/plugin/output.rb +26 -35
  62. data/lib/fluent/plugin/owned_by_mixin.rb +2 -2
  63. data/lib/fluent/plugin/parser.rb +3 -3
  64. data/lib/fluent/plugin/parser_json.rb +3 -3
  65. data/lib/fluent/plugin/sd_file.rb +2 -2
  66. data/lib/fluent/plugin/storage_local.rb +8 -4
  67. data/lib/fluent/plugin.rb +1 -1
  68. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  69. data/lib/fluent/plugin_helper/child_process.rb +2 -2
  70. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  71. data/lib/fluent/plugin_helper/http_server/request.rb +13 -2
  72. data/lib/fluent/plugin_helper/http_server/server.rb +14 -8
  73. data/lib/fluent/plugin_helper/http_server.rb +1 -8
  74. data/lib/fluent/plugin_helper/metrics.rb +7 -0
  75. data/lib/fluent/plugin_helper/server.rb +13 -1
  76. data/lib/fluent/plugin_helper/service_discovery.rb +1 -1
  77. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  78. data/lib/fluent/plugin_helper/storage.rb +1 -1
  79. data/lib/fluent/plugin_id.rb +3 -3
  80. data/lib/fluent/root_agent.rb +117 -21
  81. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  82. data/lib/fluent/static_config_analysis.rb +3 -2
  83. data/lib/fluent/supervisor.rb +258 -39
  84. data/lib/fluent/system_config.rb +27 -6
  85. data/lib/fluent/test/base.rb +1 -1
  86. data/lib/fluent/test/driver/base.rb +2 -2
  87. data/lib/fluent/test/filter_test.rb +2 -2
  88. data/lib/fluent/test/formatter_test.rb +1 -1
  89. data/lib/fluent/test/helpers.rb +4 -0
  90. data/lib/fluent/test/input_test.rb +2 -2
  91. data/lib/fluent/test/output_test.rb +4 -4
  92. data/lib/fluent/test/parser_test.rb +1 -1
  93. data/lib/fluent/tls.rb +24 -0
  94. data/lib/fluent/variable_store.rb +1 -1
  95. data/lib/fluent/version.rb +1 -1
  96. data/lib/fluent/winsvc.rb +38 -8
  97. metadata +99 -28
  98. data/lib/fluent/plugin_helper/http_server/compat/server.rb +0 -92
  99. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +0 -52
  100. data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +0 -58
@@ -59,8 +59,11 @@ module Fluent
59
59
  @size = 0
60
60
  @created_at = Fluent::Clock.real_now
61
61
  @modified_at = Fluent::Clock.real_now
62
-
63
- extend Decompressable if compress == :gzip
62
+ if compress == :gzip
63
+ extend GzipDecompressable
64
+ elsif compress == :zstd
65
+ extend ZstdDecompressable
66
+ end
64
67
  end
65
68
 
66
69
  attr_reader :unique_id, :metadata, :state
@@ -85,10 +88,17 @@ module Fluent
85
88
 
86
89
  # data is array of formatted record string
87
90
  def append(data, **kwargs)
88
- raise ArgumentError, '`compress: gzip` can be used for Compressable module' if kwargs[:compress] == :gzip
89
- adding = ''.b
90
- data.each do |d|
91
- adding << d.b
91
+ raise ArgumentError, "`compress: #{kwargs[:compress]}` can be used for Compressable module" if kwargs[:compress] == :gzip || kwargs[:compress] == :zstd
92
+ begin
93
+ adding = data.join.force_encoding(Encoding::ASCII_8BIT)
94
+ rescue
95
+ # Fallback
96
+ # Array#join throws an exception if data contains strings with a different encoding.
97
+ # Although such cases may be rare, it should be considered as a safety precaution.
98
+ adding = ''.force_encoding(Encoding::ASCII_8BIT)
99
+ data.each do |d|
100
+ adding << d.b
101
+ end
92
102
  end
93
103
  concat(adding, data.size)
94
104
  end
@@ -165,23 +175,23 @@ module Fluent
165
175
  end
166
176
 
167
177
  def read(**kwargs)
168
- raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
178
+ raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
169
179
  raise NotImplementedError, "Implement this method in child class"
170
180
  end
171
181
 
172
182
  def open(**kwargs, &block)
173
- raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
183
+ raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
174
184
  raise NotImplementedError, "Implement this method in child class"
175
185
  end
176
186
 
177
187
  def write_to(io, **kwargs)
178
- raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
188
+ raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
179
189
  open do |i|
180
190
  IO.copy_stream(i, io)
181
191
  end
182
192
  end
183
193
 
184
- module Decompressable
194
+ module GzipDecompressable
185
195
  include Fluent::Plugin::Compressable
186
196
 
187
197
  def append(data, **kwargs)
@@ -234,6 +244,60 @@ module Fluent
234
244
  end
235
245
  end
236
246
  end
247
+
248
+ module ZstdDecompressable
249
+ include Fluent::Plugin::Compressable
250
+
251
+ def append(data, **kwargs)
252
+ if kwargs[:compress] == :zstd
253
+ io = StringIO.new
254
+ stream = Zstd::StreamWriter.new(io)
255
+ data.each do |d|
256
+ stream.write(d)
257
+ end
258
+ stream.finish
259
+ concat(io.string, data.size)
260
+ else
261
+ super
262
+ end
263
+ end
264
+
265
+ def open(**kwargs, &block)
266
+ if kwargs[:compressed] == :zstd
267
+ super
268
+ else
269
+ super(**kwargs) do |chunk_io|
270
+ output_io = if chunk_io.is_a?(StringIO)
271
+ StringIO.new
272
+ else
273
+ Tempfile.new('decompressed-data')
274
+ end
275
+ output_io.binmode if output_io.is_a?(Tempfile)
276
+ decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
277
+ output_io.seek(0, IO::SEEK_SET)
278
+ yield output_io
279
+ end
280
+ end
281
+ end
282
+
283
+ def read(**kwargs)
284
+ if kwargs[:compressed] == :zstd
285
+ super
286
+ else
287
+ decompress(super,type: :zstd)
288
+ end
289
+ end
290
+
291
+ def write_to(io, **kwargs)
292
+ open(compressed: :zstd) do |chunk_io|
293
+ if kwargs[:compressed] == :zstd
294
+ IO.copy_stream(chunk_io, io)
295
+ else
296
+ decompress(input_io: chunk_io, output_io: io, type: :zstd)
297
+ end
298
+ end
299
+ end
300
+ end
237
301
  end
238
302
  end
239
303
  end
@@ -37,7 +37,7 @@ module Fluent
37
37
  # path_prefix: path prefix string, ended with '.'
38
38
  # path_suffix: path suffix string, like '.log' (or any other user specified)
39
39
 
40
- attr_reader :path, :permission
40
+ attr_reader :path, :meta_path, :permission
41
41
 
42
42
  def initialize(metadata, path, mode, perm: nil, compress: :text)
43
43
  super(metadata, compress: compress)
@@ -219,13 +219,17 @@ module Fluent
219
219
  # old type of restore
220
220
  data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
221
221
  end
222
+ raise FileChunkError, "invalid meta data" if data.nil? || !data.is_a?(Hash)
223
+ raise FileChunkError, "invalid unique_id" unless data[:id]
224
+ raise FileChunkError, "invalid created_at" unless data[:c].to_i > 0
225
+ raise FileChunkError, "invalid modified_at" unless data[:m].to_i > 0
222
226
 
223
227
  now = Fluent::Clock.real_now
224
228
 
225
- @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
229
+ @unique_id = data[:id]
226
230
  @size = data[:s] || 0
227
- @created_at = data.fetch(:c, now.to_i)
228
- @modified_at = data.fetch(:m, now.to_i)
231
+ @created_at = data[:c]
232
+ @modified_at = data[:m]
229
233
 
230
234
  @metadata.timekey = data[:timekey]
231
235
  @metadata.tag = data[:tag]
@@ -285,7 +289,7 @@ module Fluent
285
289
  @chunk.binmode
286
290
  rescue => e
287
291
  # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
288
- # If other cases are possible, we will change erorr handling with proper classes.
292
+ # If other cases are possible, we will change error handling with proper classes.
289
293
  raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
290
294
  end
291
295
  begin
@@ -243,11 +243,11 @@ module Fluent
243
243
  def encode_key(metadata)
244
244
  k = @key ? metadata.variables[@key] : metadata.tag
245
245
  k ||= ''
246
- URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
246
+ URI::RFC2396_PARSER.escape(k, ESCAPE_REGEXP)
247
247
  end
248
248
 
249
249
  def decode_key(key)
250
- URI::DEFAULT_PARSER.unescape(key)
250
+ URI::RFC2396_PARSER.unescape(key)
251
251
  end
252
252
 
253
253
  def create_new_chunk(path, metadata, perm)
@@ -259,7 +259,7 @@ module Fluent
259
259
  @chunk.binmode
260
260
  rescue => e
261
261
  # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
262
- # If other cases are possible, we will change erorr handling with proper classes.
262
+ # If other cases are possible, we will change error handling with proper classes.
263
263
  raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
264
264
  end
265
265
 
@@ -68,13 +68,13 @@ module Fluent
68
68
 
69
69
  def purge
70
70
  super
71
- @chunk = ''.force_encoding("ASCII-8BIT")
71
+ @chunk.clear
72
72
  @chunk_bytes = @size = @adding_bytes = @adding_size = 0
73
73
  true
74
74
  end
75
75
 
76
76
  def read(**kwargs)
77
- @chunk
77
+ @chunk.dup
78
78
  end
79
79
 
80
80
  def open(**kwargs, &block)
@@ -64,7 +64,7 @@ module Fluent
64
64
  config_param :queued_chunks_limit_size, :integer, default: nil
65
65
 
66
66
  desc 'Compress buffered data.'
67
- config_param :compress, :enum, list: [:text, :gzip], default: :text
67
+ config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
68
68
 
69
69
  desc 'If true, chunks are thrown away when unrecoverable error happens'
70
70
  config_param :disable_chunk_backup, :bool, default: false
@@ -196,6 +196,8 @@ module Fluent
196
196
  @mutex = Mutex.new
197
197
  end
198
198
 
199
+ # The metrics_create method defines getter methods named stage_byte_size and queue_byte_size.
200
+ # For compatibility, stage_size, stage_size=, queue_size, and queue_size= are still available.
199
201
  def stage_size
200
202
  @stage_size_metrics.get
201
203
  end
@@ -385,7 +387,7 @@ module Fluent
385
387
  end
386
388
 
387
389
  errors = []
388
- # Buffer plugin estimates there's no serious error cause: will commit for all chunks eigher way
390
+ # Buffer plugin estimates there's no serious error cause: will commit for all chunks either way
389
391
  operated_chunks.each do |chunk|
390
392
  begin
391
393
  chunk.commit
@@ -523,7 +525,7 @@ module Fluent
523
525
  chunks = @stage.values
524
526
  chunks.concat(@queue)
525
527
  @timekeys = chunks.each_with_object({}) do |chunk, keys|
526
- if chunk.metadata && chunk.metadata.timekey
528
+ if chunk.metadata&.timekey
527
529
  t = chunk.metadata.timekey
528
530
  keys[t] = keys.fetch(t, 0) + 1
529
531
  end
@@ -623,6 +625,7 @@ module Fluent
623
625
  until @queue.empty?
624
626
  begin
625
627
  q = @queue.shift
628
+ evacuate_chunk(q)
626
629
  log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
627
630
  q.purge
628
631
  rescue => e
@@ -634,6 +637,25 @@ module Fluent
634
637
  end
635
638
  end
636
639
 
640
+ def evacuate_chunk(chunk)
641
+ # Overwrite this on demand.
642
+ #
643
+ # Note: Difference from the `backup` feature.
644
+ # The `backup` feature is for unrecoverable errors, mainly for bad chunks.
645
+ # On the other hand, this feature is for normal chunks.
646
+ # The main motivation for this feature is to enable recovery by evacuating buffer files
647
+ # when the retry limit is reached due to external factors such as network issues.
648
+ #
649
+ # Note: Difference from the `secondary` feature.
650
+ # The `secondary` feature is not suitable for recovery.
651
+ # It can be difficult to recover files made by `out_secondary_file` because the metadata
652
+ # is lost.
653
+ # For file buffers, the easiest way for recovery is to evacuate the chunk files as is.
654
+ # Once the issue is recovered, we can put back the chunk files, and restart Fluentd to
655
+ # load them.
656
+ # This feature enables it.
657
+ end
658
+
637
659
  def chunk_size_over?(chunk)
638
660
  chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)
639
661
  end
@@ -923,8 +945,6 @@ module Fluent
923
945
  return
924
946
  end
925
947
 
926
- safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
927
- backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
928
948
  backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
929
949
  backup_dir = File.dirname(backup_file)
930
950
 
@@ -938,11 +958,19 @@ module Fluent
938
958
  def optimistic_queued?(metadata = nil)
939
959
  if metadata
940
960
  n = @queued_num[metadata]
941
- n && n.nonzero?
961
+ n&.nonzero?
942
962
  else
943
963
  !@queue.empty?
944
964
  end
945
965
  end
966
+
967
+ def safe_owner_id
968
+ owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
969
+ end
970
+
971
+ def backup_base_dir
972
+ system_config.root_dir || DEFAULT_BACKUP_DIR
973
+ end
946
974
  end
947
975
  end
948
976
  end
@@ -16,29 +16,35 @@
16
16
 
17
17
  require 'stringio'
18
18
  require 'zlib'
19
+ require 'zstd-ruby'
19
20
 
20
21
  module Fluent
21
22
  module Plugin
22
23
  module Compressable
23
- def compress(data, **kwargs)
24
+ def compress(data, type: :gzip, **kwargs)
24
25
  output_io = kwargs[:output_io]
25
26
  io = output_io || StringIO.new
26
- Zlib::GzipWriter.wrap(io) do |gz|
27
- gz.write data
27
+ if type == :gzip
28
+ writer = Zlib::GzipWriter.new(io)
29
+ elsif type == :zstd
30
+ writer = Zstd::StreamWriter.new(io)
31
+ else
32
+ raise ArgumentError, "Unknown compression type: #{type}"
28
33
  end
29
-
34
+ writer.write(data)
35
+ writer.finish
30
36
  output_io || io.string
31
37
  end
32
38
 
33
39
  # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`
34
40
  # https://www.ruby-forum.com/topic/971591#979503
35
- def decompress(compressed_data = nil, output_io: nil, input_io: nil)
41
+ def decompress(compressed_data = nil, output_io: nil, input_io: nil, type: :gzip)
36
42
  case
37
43
  when input_io && output_io
38
- io_decompress(input_io, output_io)
44
+ io_decompress(input_io, output_io, type)
39
45
  when input_io
40
46
  output_io = StringIO.new
41
- io = io_decompress(input_io, output_io)
47
+ io = io_decompress(input_io, output_io, type)
42
48
  io.string
43
49
  when compressed_data.nil? || compressed_data.empty?
44
50
  # check compressed_data(String) is 0 length
@@ -46,51 +52,91 @@ module Fluent
46
52
  when output_io
47
53
  # execute after checking compressed_data is empty or not
48
54
  io = StringIO.new(compressed_data)
49
- io_decompress(io, output_io)
55
+ io_decompress(io, output_io, type)
50
56
  else
51
- string_decompress(compressed_data)
57
+ string_decompress(compressed_data, type)
52
58
  end
53
59
  end
54
60
 
55
61
  private
56
62
 
57
- def string_decompress(compressed_data)
63
+ def string_decompress_gzip(compressed_data)
58
64
  io = StringIO.new(compressed_data)
59
-
60
65
  out = ''
61
66
  loop do
62
- gz = Zlib::GzipReader.new(io)
63
- out << gz.read
64
- unused = gz.unused
65
- gz.finish
66
-
67
+ reader = Zlib::GzipReader.new(io)
68
+ out << reader.read
69
+ unused = reader.unused
70
+ reader.finish
67
71
  unless unused.nil?
68
72
  adjust = unused.length
69
73
  io.pos -= adjust
70
74
  end
71
75
  break if io.eof?
72
76
  end
77
+ out
78
+ end
73
79
 
80
+ def string_decompress_zstd(compressed_data)
81
+ io = StringIO.new(compressed_data)
82
+ reader = Zstd::StreamReader.new(io)
83
+ out = ''
84
+ loop do
85
+ # Zstd::StreamReader needs to specify the size of the buffer
86
+ out << reader.read(1024)
87
+ # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
88
+ break if io.eof?
89
+ end
74
90
  out
75
91
  end
76
92
 
77
- def io_decompress(input, output)
93
+ def string_decompress(compressed_data, type = :gzip)
94
+ if type == :gzip
95
+ string_decompress_gzip(compressed_data)
96
+ elsif type == :zstd
97
+ string_decompress_zstd(compressed_data)
98
+ else
99
+ raise ArgumentError, "Unknown compression type: #{type}"
100
+ end
101
+ end
102
+
103
+ def io_decompress_gzip(input, output)
78
104
  loop do
79
- gz = Zlib::GzipReader.new(input)
80
- v = gz.read
105
+ reader = Zlib::GzipReader.new(input)
106
+ v = reader.read
81
107
  output.write(v)
82
- unused = gz.unused
83
- gz.finish
84
-
108
+ unused = reader.unused
109
+ reader.finish
85
110
  unless unused.nil?
86
111
  adjust = unused.length
87
112
  input.pos -= adjust
88
113
  end
89
114
  break if input.eof?
90
115
  end
116
+ output
117
+ end
91
118
 
119
+ def io_decompress_zstd(input, output)
120
+ reader = Zstd::StreamReader.new(input)
121
+ loop do
122
+ # Zstd::StreamReader needs to specify the size of the buffer
123
+ v = reader.read(1024)
124
+ output.write(v)
125
+ # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
126
+ break if input.eof?
127
+ end
92
128
  output
93
129
  end
130
+
131
+ def io_decompress(input, output, type = :gzip)
132
+ if type == :gzip
133
+ io_decompress_gzip(input, output)
134
+ elsif type == :zstd
135
+ io_decompress_zstd(input, output)
136
+ else
137
+ raise ArgumentError, "Unknown compression type: #{type}"
138
+ end
139
+ end
94
140
  end
95
141
  end
96
142
  end
@@ -41,14 +41,6 @@ module Fluent
41
41
  @enable_size_metrics = false
42
42
  end
43
43
 
44
- def emit_records
45
- @emit_records_metrics.get
46
- end
47
-
48
- def emit_size
49
- @emit_size_metrics.get
50
- end
51
-
52
44
  def configure(conf)
53
45
  super
54
46
 
@@ -54,29 +54,32 @@ module Fluent::Plugin
54
54
  @parser = parser_create
55
55
  end
56
56
 
57
- FAILED_RESULT = [nil, nil].freeze # reduce allocation cost
58
57
  REPLACE_CHAR = '?'.freeze
59
58
 
60
- def filter_with_time(tag, time, record)
61
- raw_value = @accessor.call(record)
62
- if raw_value.nil?
63
- if @emit_invalid_record_to_error
64
- router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
65
- end
66
- if @reserve_data
67
- return time, handle_parsed(tag, record, time, {})
68
- else
69
- return FAILED_RESULT
59
+ def filter_stream(tag, es)
60
+ new_es = Fluent::MultiEventStream.new
61
+ es.each do |time, record|
62
+ begin
63
+ raw_value = @accessor.call(record)
64
+ if raw_value.nil?
65
+ new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
66
+ raise ArgumentError, "#{@key_name} does not exist"
67
+ else
68
+ filter_one_record(tag, time, record, raw_value) do |result_time, result_record|
69
+ new_es.add(result_time, result_record)
70
+ end
71
+ end
72
+ rescue => e
73
+ router.emit_error_event(tag, time, record, e) if @emit_invalid_record_to_error
70
74
  end
71
75
  end
72
- begin
73
- # Note: https://github.com/fluent/fluentd/issues/4100
74
- # If the parser returns multiple records from one raw_value,
75
- # this returns only the first one record.
76
- # This should be fixed in the future version.
77
- result_time = nil
78
- result_record = nil
76
+ new_es
77
+ end
78
+
79
+ private
79
80
 
81
+ def filter_one_record(tag, time, record, raw_value)
82
+ begin
80
83
  @parser.parse(raw_value) do |t, values|
81
84
  if values
82
85
  t = if @reserve_time
@@ -85,38 +88,17 @@ module Fluent::Plugin
85
88
  t.nil? ? time : t
86
89
  end
87
90
  @accessor.delete(record) if @remove_key_name_field
88
- r = handle_parsed(tag, record, t, values)
89
-
90
- if result_record.nil?
91
- result_time = t
92
- result_record = r
93
- else
94
- if @emit_invalid_record_to_error
95
- router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
96
- "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
97
- ))
98
- end
99
- end
100
91
  else
101
- if @emit_invalid_record_to_error
102
- router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
103
- end
104
-
92
+ router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'")) if @emit_invalid_record_to_error
105
93
  next unless @reserve_data
106
- next unless result_record.nil?
107
-
108
- result_time = time
109
- result_record = handle_parsed(tag, record, time, {})
94
+ t = time
95
+ values = {}
110
96
  end
97
+ yield(t, handle_parsed(tag, record, t, values))
111
98
  end
112
99
 
113
- return result_time, result_record
114
100
  rescue Fluent::Plugin::Parser::ParserError => e
115
- if @emit_invalid_record_to_error
116
- raise e
117
- else
118
- return FAILED_RESULT
119
- end
101
+ raise e
120
102
  rescue ArgumentError => e
121
103
  raise unless @replace_invalid_sequence
122
104
  raise unless e.message.index("invalid byte sequence in") == 0
@@ -124,16 +106,10 @@ module Fluent::Plugin
124
106
  raw_value = raw_value.scrub(REPLACE_CHAR)
125
107
  retry
126
108
  rescue => e
127
- if @emit_invalid_record_to_error
128
- raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
129
- else
130
- return FAILED_RESULT
131
- end
109
+ raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
132
110
  end
133
111
  end
134
112
 
135
- private
136
-
137
113
  def handle_parsed(tag, record, t, values)
138
114
  if values && @inject_key_prefix
139
115
  values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]
@@ -207,7 +207,7 @@ module Fluent::Plugin
207
207
  value.each do |k, v|
208
208
  placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
209
209
  end
210
- else # string, interger, float, and others?
210
+ else # string, integer, float, and others?
211
211
  placeholders.store("${#{key}}", value)
212
212
  end
213
213
  end
@@ -35,6 +35,22 @@ module Fluent
35
35
  config_param :fields, :array, value_type: :string
36
36
  config_param :add_newline, :bool, default: true
37
37
 
38
+ def csv_cacheable?
39
+ !!owner
40
+ end
41
+
42
+ def csv_thread_key
43
+ csv_cacheable? ? "#{owner.plugin_id}_csv_formatter_#{@usage}_csv" : nil
44
+ end
45
+
46
+ def csv_for_thread
47
+ if csv_cacheable?
48
+ Thread.current[csv_thread_key] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
49
+ else
50
+ CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
51
+ end
52
+ end
53
+
38
54
  def configure(conf)
39
55
  super
40
56
 
@@ -51,12 +67,10 @@ module Fluent
51
67
 
52
68
  @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
53
69
  row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
54
- # Cache CSV object per thread to avoid internal state sharing
55
- @cache = {}
56
70
  end
57
71
 
58
72
  def format(tag, time, record)
59
- csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
73
+ csv = csv_for_thread
60
74
  line = (csv << record).string.dup
61
75
  # Need manual cleanup because CSV writer doesn't provide such method.
62
76
  csv.rewind
@@ -65,7 +79,7 @@ module Fluent
65
79
  end
66
80
 
67
81
  def format_with_nested_fields(tag, time, record)
68
- csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
82
+ csv = csv_for_thread
69
83
  values = @accessors.map { |a| a.call(record) }
70
84
  line = (csv << values).string.dup
71
85
  # Need manual cleanup because CSV writer doesn't provide such method.
@@ -34,11 +34,11 @@ module Fluent
34
34
  if Fluent::OjOptions.available?
35
35
  @dump_proc = Oj.method(:dump)
36
36
  else
37
- log.info "Oj isn't installed, fallback to Yajl as json parser"
38
- @dump_proc = Yajl.method(:dump)
37
+ log.info "Oj isn't installed, fallback to JSON as json parser"
38
+ @dump_proc = JSON.method(:generate)
39
39
  end
40
40
  else
41
- @dump_proc = Yajl.method(:dump)
41
+ @dump_proc = JSON.method(:generate)
42
42
  end
43
43
 
44
44
  # format json is used on various highload environment, so re-define method to skip if check
@@ -48,7 +48,10 @@ module Fluent
48
48
  end
49
49
 
50
50
  def format(tag, time, record)
51
- "#{@dump_proc.call(record)}#{@newline}"
51
+ json_str = @dump_proc.call(record)
52
+ "#{json_str}#{@newline}"
53
+ ensure
54
+ json_str&.clear
52
55
  end
53
56
 
54
57
  def format_without_nl(tag, time, record)