fluentd 1.19.4 → 1.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/AGENTS.md +46 -0
  3. data/CHANGELOG.md +83 -0
  4. data/GithubWorkflow.md +1 -1
  5. data/MAINTAINERS.md +2 -2
  6. data/README.md +3 -2
  7. data/Rakefile +2 -1
  8. data/SECURITY.md +26 -1
  9. data/lib/fluent/command/cat.rb +2 -2
  10. data/lib/fluent/compat/exec_util.rb +20 -5
  11. data/lib/fluent/config/literal_parser.rb +4 -3
  12. data/lib/fluent/config/yaml_parser/loader.rb +12 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +14 -13
  14. data/lib/fluent/event.rb +37 -0
  15. data/lib/fluent/load.rb +0 -1
  16. data/lib/fluent/log.rb +2 -2
  17. data/lib/fluent/plugin/buf_file.rb +2 -2
  18. data/lib/fluent/plugin/buf_file_single.rb +6 -2
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -10
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +23 -10
  21. data/lib/fluent/plugin/buffer.rb +102 -36
  22. data/lib/fluent/plugin/formatter_out_file.rb +3 -4
  23. data/lib/fluent/plugin/in_exec.rb +6 -4
  24. data/lib/fluent/plugin/in_forward.rb +39 -29
  25. data/lib/fluent/plugin/in_object_space.rb +2 -2
  26. data/lib/fluent/plugin/in_syslog.rb +2 -2
  27. data/lib/fluent/plugin/in_tail/compatibility.rb +111 -0
  28. data/lib/fluent/plugin/in_tail/dir_watcher.rb +713 -0
  29. data/lib/fluent/plugin/in_tail/group_watch.rb +13 -13
  30. data/lib/fluent/plugin/in_tail/io_handler.rb +391 -0
  31. data/lib/fluent/plugin/in_tail/line_feeder.rb +230 -0
  32. data/lib/fluent/plugin/in_tail/stat_watcher.rb +52 -0
  33. data/lib/fluent/plugin/in_tail/tail_watcher.rb +276 -0
  34. data/lib/fluent/plugin/in_tail/worker_pool.rb +298 -0
  35. data/lib/fluent/plugin/in_tail.rb +324 -815
  36. data/lib/fluent/plugin/in_unix.rb +5 -3
  37. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  38. data/lib/fluent/plugin/out_file.rb +10 -4
  39. data/lib/fluent/plugin/out_forward.rb +8 -0
  40. data/lib/fluent/plugin/out_http.rb +0 -10
  41. data/lib/fluent/plugin/output.rb +49 -19
  42. data/lib/fluent/plugin/parser_csv.rb +7 -2
  43. data/lib/fluent/plugin/parser_json.rb +20 -12
  44. data/lib/fluent/plugin/parser_ltsv.rb +2 -4
  45. data/lib/fluent/plugin/parser_msgpack.rb +7 -6
  46. data/lib/fluent/plugin/parser_syslog.rb +52 -5
  47. data/lib/fluent/plugin/parser_tsv.rb +6 -1
  48. data/lib/fluent/plugin_helper/http_server/server.rb +29 -13
  49. data/lib/fluent/plugin_helper/http_server.rb +2 -1
  50. data/lib/fluent/root_agent.rb +1 -1
  51. data/lib/fluent/supervisor.rb +24 -5
  52. data/lib/fluent/system_config.rb +5 -2
  53. data/lib/fluent/time.rb +8 -7
  54. data/lib/fluent/version.rb +1 -1
  55. data/templates/new_gem/.gitignore +8 -0
  56. metadata +25 -16
@@ -19,7 +19,7 @@ require 'fluent/plugin/input'
19
19
  require 'fluent/msgpack_factory'
20
20
 
21
21
  require 'cool.io'
22
- require 'yajl'
22
+ require 'json'
23
23
  require 'fileutils'
24
24
  require 'socket'
25
25
 
@@ -158,8 +158,7 @@ module Fluent::Plugin
158
158
  first = data[0]
159
159
  if first == '{'.freeze || first == '['.freeze
160
160
  m = method(:on_read_json)
161
- @parser = Yajl::Parser.new
162
- @parser.on_parse_complete = @on_message
161
+ @parser = JSON::ResumableParser.new(**Fluent::DEFAULT_JSON_PARSE_OPTIONS)
163
162
  else
164
163
  m = method(:on_read_msgpack)
165
164
  @parser = Fluent::MessagePackFactory.msgpack_unpacker
@@ -173,6 +172,9 @@ module Fluent::Plugin
173
172
 
174
173
  def on_read_json(data)
175
174
  @parser << data
175
+ while @parser.parse
176
+ @on_message.call(@parser.value)
177
+ end
176
178
  rescue => e
177
179
  @log.error "unexpected error in json payload", error: e.to_s
178
180
  @log.error_backtrace
@@ -17,7 +17,7 @@ require 'fluent/plugin/output'
17
17
  require 'fluent/env'
18
18
  require 'fluent/config/error'
19
19
 
20
- require 'yajl'
20
+ require 'json'
21
21
 
22
22
  module Fluent::Plugin
23
23
  class ExecFilterOutput < Output
@@ -309,7 +309,7 @@ module Fluent::Plugin
309
309
  router.emit(tag, time, record)
310
310
  rescue => e
311
311
  if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
312
- log.error "exec_filter failed to emit", record: Yajl.dump(record), error: e
312
+ log.error "exec_filter failed to emit", record: JSON.generate(record), error: e
313
313
  log.error_backtrace e.backtrace
314
314
  @next_log_time = Time.now.to_i + @suppress_error_log_interval
315
315
  end
@@ -75,9 +75,8 @@ module Fluent::Plugin
75
75
  def metadata(timekey: nil, tag: nil, variables: nil)
76
76
  metadata = super
77
77
 
78
- @latest_metadata ||= new_metadata(timekey: 0)
79
- if metadata.timekey && (metadata.timekey >= @latest_metadata.timekey)
80
- @latest_metadata = metadata
78
+ if newer_timekey?(metadata.timekey)
79
+ synchronize { @latest_timekey = metadata.timekey if newer_timekey?(metadata.timekey) }
81
80
  end
82
81
 
83
82
  metadata
@@ -97,7 +96,8 @@ module Fluent::Plugin
97
96
  # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
98
97
  # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
99
98
  # These chunks will be enqueued immediately, and will be flushed soon.
100
- if chunk.metadata == @latest_metadata
99
+ # Split chunks (seq > 0) are generated outside the buffer lock, so ln_sf would race between threads.
100
+ if chunk.metadata.timekey && chunk.metadata.seq == 0 && chunk.metadata.timekey == @latest_timekey
101
101
  sym_path = @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk)
102
102
  FileUtils.mkdir_p(File.dirname(sym_path), mode: @_output_plugin_for_symlink.dir_perm)
103
103
  if @_output_plugin_for_symlink.symlink_path_use_relative
@@ -109,6 +109,12 @@ module Fluent::Plugin
109
109
  end
110
110
  chunk
111
111
  end
112
+
113
+ private
114
+
115
+ def newer_timekey?(timekey)
116
+ timekey && (@latest_timekey.nil? || timekey > @latest_timekey)
117
+ end
112
118
  end
113
119
 
114
120
  def configure(conf)
@@ -119,6 +119,8 @@ module Fluent::Plugin
119
119
  config_param :keepalive, :bool, default: false
120
120
  desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
121
121
  config_param :keepalive_timeout, :time, default: nil
122
+ desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
123
+ config_param :send_keepalive_packet, :bool, default: false
122
124
 
123
125
  config_section :security, required: false, multi: false do
124
126
  desc 'The hostname'
@@ -270,6 +272,10 @@ module Fluent::Plugin
270
272
  log.warn('The value of keepalive_timeout is ignored. if you want to use keepalive, please add `keepalive true` to your conf.')
271
273
  end
272
274
 
275
+ if @send_keepalive_packet && !@keepalive
276
+ raise Fluent::ConfigError, "'send_keepalive_packet' is enabled but 'keepalive' is not. Enable 'keepalive' to use TCP keepalive."
277
+ end
278
+
273
279
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
274
280
 
275
281
  if @compress == :zstd
@@ -414,6 +420,7 @@ module Fluent::Plugin
414
420
  send_timeout: @send_timeout,
415
421
  recv_timeout: @ack_response_timeout,
416
422
  connect_timeout: @connect_timeout,
423
+ send_keepalive_packet: @send_keepalive_packet,
417
424
  &block
418
425
  )
419
426
  when :tcp
@@ -423,6 +430,7 @@ module Fluent::Plugin
423
430
  send_timeout: @send_timeout,
424
431
  recv_timeout: @ack_response_timeout,
425
432
  connect_timeout: @connect_timeout,
433
+ send_keepalive_packet: @send_keepalive_packet,
426
434
  &block
427
435
  )
428
436
  else
@@ -22,16 +22,6 @@ require 'fluent/tls'
22
22
  require 'fluent/plugin/output'
23
23
  require 'fluent/plugin_helper/socket'
24
24
 
25
- # patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
26
- # see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
27
- unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
28
- class Net::HTTP
29
- SSL_IVNAMES << :@extra_chain_cert
30
- SSL_ATTRIBUTES << :extra_chain_cert
31
- attr_accessor :extra_chain_cert
32
- end
33
- end
34
-
35
25
  module Fluent::Plugin
36
26
  class HTTPOutput < Output
37
27
  Fluent::Plugin.register_output('http', self)
@@ -958,39 +958,47 @@ module Fluent
958
958
  return Struct.new(:timekey, :tag, :variables).new
959
959
  end
960
960
 
961
+ timekey = @chunk_key_time ? calculate_timekey(time) : nil
962
+ @_metadata_cache ||= MetadataCache.new
963
+
961
964
  # timekey is int from epoch, and `timekey - timekey % 60` is assumed to mach with 0s of each minutes.
962
965
  # it's wrong if timezone is configured as one which supports leap second, but it's very rare and
963
966
  # we can ignore it (especially in production systems).
964
967
  if @chunk_keys.empty?
965
- if !@chunk_key_time && !@chunk_key_tag
966
- @buffer.metadata()
967
- elsif @chunk_key_time && @chunk_key_tag
968
- timekey = calculate_timekey(time)
969
- @buffer.metadata(timekey: timekey, tag: tag)
970
- elsif @chunk_key_time
971
- timekey = calculate_timekey(time)
972
- @buffer.metadata(timekey: timekey)
973
- else
974
- @buffer.metadata(tag: tag)
975
- end
968
+ return @_metadata_cache.metadata if @_metadata_cache.cached?(timekey: timekey, tag: tag)
969
+
970
+ meta = if @chunk_key_time && @chunk_key_tag
971
+ @buffer.metadata(timekey: timekey, tag: tag)
972
+ elsif @chunk_key_time
973
+ @buffer.metadata(timekey: timekey)
974
+ elsif @chunk_key_tag
975
+ @buffer.metadata(tag: tag)
976
+ else
977
+ @buffer.metadata()
978
+ end
979
+
980
+ @_metadata_cache.update(timekey: timekey, tag: tag, metadata: meta)
981
+ meta
976
982
  else
977
- timekey = if @chunk_key_time
978
- calculate_timekey(time)
979
- else
980
- nil
981
- end
982
983
  pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
983
- @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
984
+ return @_metadata_cache.metadata if @_metadata_cache.cached?(timekey: timekey, tag: tag, variables: pairs)
985
+
986
+ meta = @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
987
+ @_metadata_cache.update(timekey: timekey, tag: tag, variables: pairs, metadata: meta)
988
+ meta
984
989
  end
985
990
  end
986
991
 
987
992
  def calculate_timekey(time)
988
993
  time_int = time.to_i
994
+ return @calculate_timekey_last_timekey if @calculate_timekey_last_time_int == time_int
995
+
996
+ @calculate_timekey_last_time_int = time_int
989
997
  if @timekey_use_utc
990
- (time_int - (time_int % @timekey)).to_i
998
+ @calculate_timekey_last_timekey = (time_int - (time_int % @timekey)).to_i
991
999
  else
992
1000
  offset = @calculate_offset ? @calculate_offset.call(time) : @offset
993
- (time_int - ((time_int + offset)% @timekey)).to_i
1001
+ @calculate_timekey_last_timekey = (time_int - ((time_int + offset) % @timekey)).to_i
994
1002
  end
995
1003
  end
996
1004
 
@@ -1654,5 +1662,27 @@ module Fluent
1654
1662
  { 'output' => stats }
1655
1663
  end
1656
1664
  end
1665
+
1666
+ class MetadataCache
1667
+ attr_reader :metadata
1668
+
1669
+ def initialize
1670
+ @timekey = nil
1671
+ @tag = nil
1672
+ @variables = nil
1673
+ @metadata = nil
1674
+ end
1675
+
1676
+ def cached?(timekey:, tag:, variables: nil)
1677
+ @metadata && @timekey == timekey && @tag == tag && @variables == variables
1678
+ end
1679
+
1680
+ def update(timekey:, tag:, variables: nil, metadata:)
1681
+ @timekey = timekey
1682
+ @tag = tag
1683
+ @variables = variables
1684
+ @metadata = metadata
1685
+ end
1686
+ end
1657
1687
  end
1658
1688
  end
@@ -32,7 +32,7 @@ module Fluent
32
32
 
33
33
  def configure(conf)
34
34
  super
35
-
35
+ @key_num = @keys.length
36
36
 
37
37
  if @parser_engine == :fast
38
38
  @quote_char = '"'
@@ -46,13 +46,18 @@ module Fluent
46
46
  end
47
47
 
48
48
  def parse(text, &block)
49
+ r = {}
50
+ i = 0
49
51
  values = CSV.parse_line(text, col_sep: @delimiter)
50
52
  unless values
51
53
  yield nil, nil
52
54
  return
53
55
  end
54
56
 
55
- r = Hash[@keys.zip(values)]
57
+ while i < @key_num
58
+ r[@keys[i]] = values[i]
59
+ i += 1
60
+ end
56
61
  time, record = convert_values(parse_time(r), r)
57
62
  yield time, record
58
63
  end
@@ -19,7 +19,6 @@ require 'fluent/plugin/parser'
19
19
  require 'fluent/time'
20
20
  require 'fluent/oj_options'
21
21
 
22
- require 'yajl'
23
22
  require 'json'
24
23
 
25
24
  module Fluent
@@ -29,12 +28,10 @@ module Fluent
29
28
 
30
29
  config_set_default :time_key, 'time'
31
30
  desc 'Set JSON parser'
31
+ # NOTE: Contains yajl for backward compatibility
32
32
  config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
33
33
 
34
- # The Yajl library defines a default buffer size of 8KiB when parsing
35
- # from IO streams, so maintain this for backwards-compatibility.
36
- # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
37
- desc 'Set the buffer size that Yajl will use when parsing streaming input'
34
+ desc 'Set the buffer size that JSON parser will use when parsing streaming input'
38
35
  config_param :stream_buffer_size, :integer, default: 8192
39
36
 
40
37
  config_set_default :time_type, :float
@@ -59,8 +56,8 @@ module Fluent
59
56
 
60
57
  log&.info "Oj is not installed, and failing back to JSON for json parser"
61
58
  configure_json_parser(:json)
62
- when :json then [JSON_PARSE_PROC, JSON::ParserError]
63
- when :yajl then [Yajl.method(:load), Yajl::ParseError]
59
+ when :yajl, :json # NOTE: Fallback yajl to json for backward compatibility
60
+ [JSON_PARSE_PROC, JSON::ParserError]
64
61
  else
65
62
  raise "BUG: unknown json parser specified: #{name}"
66
63
  end
@@ -99,11 +96,22 @@ module Fluent
99
96
  end
100
97
 
101
98
  def parse_io(io, &block)
102
- y = Yajl::Parser.new
103
- y.on_parse_complete = ->(record){
104
- block.call(parse_time(record), record)
105
- }
106
- y.parse(io, @stream_buffer_size)
99
+ parser = JSON::ResumableParser.new(**Fluent::DEFAULT_JSON_PARSE_OPTIONS)
100
+ begin
101
+ chunk = +"".b
102
+ while io.readpartial(@stream_buffer_size, chunk)
103
+ parser << chunk
104
+ while parser.parse
105
+ record = parser.value
106
+ block.call(parse_time(record), record)
107
+ end
108
+ end
109
+ rescue EOFError
110
+ unless parser.empty?
111
+ log&.warn "JSON stream ended in the middle of a document; " \
112
+ "discarding incomplete data"
113
+ end
114
+ end
107
115
  end
108
116
  end
109
117
  end
@@ -38,10 +38,8 @@ module Fluent
38
38
  def parse(text)
39
39
  r = {}
40
40
  text.split(@delimiter).each do |pair|
41
- if pair.include? @label_delimiter
42
- key, value = pair.split(@label_delimiter, 2)
43
- r[key] = value
44
- end
41
+ key, value = pair.split(@label_delimiter, 2)
42
+ r[key] = value if value
45
43
  end
46
44
  time, record = convert_values(parse_time(r), r)
47
45
  yield time, record
@@ -22,21 +22,22 @@ module Fluent
22
22
  class MessagePackParser < Parser
23
23
  Plugin.register_parser('msgpack', self)
24
24
 
25
- def configure(conf)
26
- super
27
- @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
28
- end
29
-
30
25
  def parser_type
31
26
  :binary
32
27
  end
33
28
 
34
29
  def parse(data, &block)
30
+ Fluent::MessagePackFactory.engine_factory.unpacker.feed_each(data) do |obj|
31
+ parse_unpacked_data(obj, &block)
32
+ end
33
+ end
34
+
35
+ def parse_partial_data(data, &block)
36
+ @unpacker ||= Fluent::MessagePackFactory.engine_factory.unpacker
35
37
  @unpacker.feed_each(data) do |obj|
36
38
  parse_unpacked_data(obj, &block)
37
39
  end
38
40
  end
39
- alias parse_partial_data parse
40
41
 
41
42
  def parse_io(io, &block)
42
43
  u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
@@ -71,6 +71,7 @@ module Fluent
71
71
  @time_parser_rfc5424 = nil
72
72
  @space_count_rfc3164 = nil
73
73
  @space_count_rfc5424 = nil
74
+ @time_format_starts_with_space_rfc3164 = false
74
75
  @skip_space_count_rfc3164 = false
75
76
  @skip_space_count_rfc5424 = false
76
77
  @time_parser_rfc5424_without_subseconds = nil
@@ -123,10 +124,12 @@ module Fluent
123
124
 
124
125
  def setup_time_parser_3164(time_fmt)
125
126
  @time_parser_rfc3164 = time_parser_create(format: time_fmt)
127
+ @time_format_starts_with_space_rfc3164 = time_fmt.start_with?(SPLIT_CHAR)
126
128
  if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt)
127
129
  @skip_space_count_rfc3164 = true
128
130
  end
129
131
  @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1
132
+ @space_count_rfc3164 -= 1 if @time_format_starts_with_space_rfc3164
130
133
  end
131
134
 
132
135
  def setup_time_parser_5424(time_fmt)
@@ -162,6 +165,8 @@ module Fluent
162
165
  end
163
166
 
164
167
  SPLIT_CHAR = ' '.freeze
168
+ DOT_CHAR = '.'.freeze
169
+ RFC3164_PRI_DIGITS_REGEXP = /\A[0-9]{1,3}\z/
165
170
 
166
171
  def parse_rfc3164_regex(text, &block)
167
172
  idx = 0
@@ -180,13 +185,26 @@ module Fluent
180
185
 
181
186
  i = idx - 1
182
187
  sq = false
188
+ first_time_field = true
183
189
  @space_count_rfc3164.times do
184
190
  while text[i + 1] == SPLIT_CHAR
191
+ if first_time_field
192
+ unless @time_format_starts_with_space_rfc3164
193
+ idx += 1
194
+ i += 1
195
+ break
196
+ end
197
+ end
185
198
  sq = true
186
199
  i += 1
187
200
  end
188
201
 
202
+ first_time_field = false
189
203
  i = text.index(SPLIT_CHAR, i + 1)
204
+ unless i
205
+ yield nil, nil
206
+ return
207
+ end
190
208
  end
191
209
 
192
210
  time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)
@@ -287,16 +305,30 @@ module Fluent
287
305
  end
288
306
  end
289
307
 
308
+ if text[cursor] == SPLIT_CHAR
309
+ cursor = rfc3164_space_cursor(text, cursor)
310
+ unless cursor
311
+ yield nil, nil
312
+ return
313
+ end
314
+ end
315
+
290
316
  if @skip_space_count_rfc3164
291
317
  # header part
292
318
  time_size = 15 # skip Mmm dd hh:mm:ss
293
- time_end = text[cursor + time_size]
319
+ time_end_index = cursor + time_size
320
+ time_end = text[time_end_index]
321
+
294
322
  if time_end == SPLIT_CHAR
295
323
  time_str = text.slice(cursor, time_size)
296
324
  cursor += 16 # time + ' '
297
- elsif time_end == '.'.freeze
325
+ elsif time_end == DOT_CHAR
298
326
  # support subsecond time
299
- i = text.index(SPLIT_CHAR, time_size)
327
+ i = text.index(SPLIT_CHAR, time_end_index)
328
+ unless i
329
+ yield nil, nil
330
+ return
331
+ end
300
332
  time_str = text.slice(cursor, i - cursor)
301
333
  cursor = i + 1
302
334
  else
@@ -312,6 +344,10 @@ module Fluent
312
344
  i += 1
313
345
  end
314
346
  i = text.index(SPLIT_CHAR, i + 1)
347
+ unless i
348
+ yield nil, nil
349
+ return
350
+ end
315
351
  end
316
352
 
317
353
  time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
@@ -319,7 +355,7 @@ module Fluent
319
355
  end
320
356
 
321
357
  i = text.index(SPLIT_CHAR, cursor)
322
- if i.nil?
358
+ unless i
323
359
  yield nil, nil
324
360
  return
325
361
  end
@@ -363,12 +399,23 @@ module Fluent
363
399
  msg.chomp!
364
400
  record['message'] = msg
365
401
 
366
- time = @time_parser_rfc3164.parse(time_str)
402
+ begin
403
+ time = @time_parser_rfc3164.parse(time_str)
404
+ rescue Fluent::TimeParser::TimeParseError
405
+ yield nil, nil
406
+ return
407
+ end
367
408
  record['time'] = time_str if @keep_time_key
368
409
 
369
410
  yield time, record
370
411
  end
371
412
 
413
+ def rfc3164_space_cursor(text, cursor)
414
+ return if @with_priority && !RFC3164_PRI_DIGITS_REGEXP.match?(text.slice(1, cursor - 2))
415
+
416
+ @time_format_starts_with_space_rfc3164 ? cursor : cursor + 1
417
+ end
418
+
372
419
  NILVALUE = '-'.freeze
373
420
 
374
421
  def parse_rfc5424(text, &block)
@@ -32,8 +32,13 @@ module Fluent
32
32
  end
33
33
 
34
34
  def parse(text)
35
+ r = {}
36
+ i = 0
35
37
  values = text.split(@delimiter, @key_num)
36
- r = Hash[@keys.zip(values)]
38
+ while i < @key_num
39
+ r[@keys[i]] = values[i]
40
+ i += 1
41
+ end
37
42
  time, record = convert_values(parse_time(r), r)
38
43
  yield time, record
39
44
  end
@@ -31,15 +31,28 @@ module Fluent
31
31
  # @param default_app [Object] This method must have #call.
32
32
  # @param tls_context [OpenSSL::SSL::SSLContext]
33
33
  def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
34
- @addr = addr
34
+ # Normalize the address: strip brackets if present
35
+ # Brackets are only for URI formatting, not for socket binding
36
+ @addr = if addr.start_with?('[') && addr.end_with?(']')
37
+ addr[1..-2] # Remove surrounding brackets
38
+ else
39
+ addr
40
+ end
35
41
  @port = port
36
42
  @logger = logger
37
43
 
38
44
  # TODO: support http2
39
45
  scheme = tls_context ? 'https' : 'http'
40
- @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
46
+ # Handle IPv6 addresses properly in URI construction per RFC 3986
47
+ # Add brackets to IPv6 addresses for URI compliance (ip-literal per RFC 3986)
48
+ ip_literal = if @addr.include?(":")
49
+ "[#{@addr}]" # IPv6 address - add brackets
50
+ else
51
+ @addr # IPv4 or hostname - use directly
52
+ end
53
+ @uri = URI("#{scheme}://#{ip_literal}:#{@port}").to_s
41
54
  @router = Router.new(default_app)
42
- @server_task = nil
55
+ @server_task_queue = ::Thread::Queue.new
43
56
  Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
44
57
 
45
58
  opts = if tls_context
@@ -60,17 +73,20 @@ module Fluent
60
73
 
61
74
  Async do |task|
62
75
  Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
63
- @server_task = task.async do
64
- Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
65
- @server.run
66
- end
67
- if notify
68
- notify.push(:ready)
69
- end
76
+ begin
77
+ @server.endpoint.accept(&@server.method(:accept))
78
+ if notify
79
+ notify.push(:ready)
80
+ end
70
81
 
71
- @server_task_queue = ::Thread::Queue.new
72
- @server_task_queue.pop
73
- @server_task&.stop
82
+ @server_task_queue.pop
83
+ rescue => e
84
+ @logger.error("Failed to run HTTP server listening #{@uri}: #{e.class}: #{e.message}")
85
+ raise unless notify
86
+ notify.push(e)
87
+ ensure
88
+ task.children&.each(&:stop)
89
+ end
74
90
  end
75
91
 
76
92
  @logger.debug('Finished HTTP server')
@@ -121,7 +121,8 @@ module Fluent
121
121
  def _block_until_http_server_start
122
122
  que = Queue.new
123
123
  yield(que)
124
- que.pop
124
+ result = que.pop
125
+ raise result if result.is_a?(Exception)
125
126
  end
126
127
  end
127
128
  end
@@ -415,7 +415,7 @@ module Fluent
415
415
  input.configure(conf)
416
416
  input.event_emitter_apply_source_only if @source_only_mode.enabled?
417
417
  if @enable_input_metrics
418
- @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
418
+ input.event_emitter_router(conf['@label']).add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
419
419
  end
420
420
  @inputs << input
421
421
 
@@ -373,6 +373,7 @@ module Fluent
373
373
  encoding: config[:conf_encoding],
374
374
  additional_config: config[:inline_config],
375
375
  use_v1_config: config[:use_v1_config],
376
+ type: config[:config_file_type],
376
377
  )
377
378
 
378
379
  Fluent::VariableStore.try_to_reset do
@@ -611,6 +612,8 @@ module Fluent
611
612
  fluentd_conf: params['fluentd_conf'],
612
613
  conf_encoding: params['conf_encoding'],
613
614
  inline_config: params['inline_config'],
615
+ use_v1_config: params['use_v1_config'],
616
+ config_file_type: params['config_file_type']&.to_sym,
614
617
  main_cmd: params['main_cmd'],
615
618
  signame: params['signame'],
616
619
  disable_shared_socket: params['disable_shared_socket'],
@@ -787,6 +790,7 @@ module Fluent
787
790
  MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
788
791
  Fluent::Engine.init(@system_config, start_in_parallel: ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"))
789
792
  Fluent::Engine.run_configure(@conf)
793
+ enable_ruby_jit if @system_config.enable_jit
790
794
  Fluent::Engine.run
791
795
  self.class.cleanup_socketmanager_path if @standalone_worker
792
796
  exit 0
@@ -815,6 +819,12 @@ module Fluent
815
819
  )
816
820
  @system_config = build_system_config(@conf)
817
821
 
822
+ # `<system> umask` sets the process umask, just like the `--umask` command
823
+ # line option. The command line option takes precedence when both are given.
824
+ if @system_config.umask && !@cl_opt.key?(:chumask)
825
+ @chumask = @system_config.umask
826
+ end
827
+
818
828
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
819
829
 
820
830
  build_additional_configurations(parsed_files) do |additional_conf|
@@ -971,6 +981,7 @@ module Fluent
971
981
  'fluentd_conf_path' => @config_path,
972
982
  'fluentd_conf' => @conf.to_s,
973
983
  'use_v1_config' => @use_v1_config,
984
+ 'config_file_type' => @config_file_type,
974
985
  'conf_encoding' => @conf_encoding,
975
986
  'signame' => @signame,
976
987
 
@@ -1215,6 +1226,19 @@ module Fluent
1215
1226
  exit!(unrecoverable_error ? 2 : 1)
1216
1227
  end
1217
1228
 
1229
+ def enable_ruby_jit
1230
+ unless defined?(RubyVM::YJIT) && RubyVM::YJIT.respond_to?(:enable)
1231
+ $log.info "Ruby JIT is not available on this Ruby"
1232
+ return
1233
+ end
1234
+
1235
+ if RubyVM::YJIT.enable
1236
+ $log.info "enabled Ruby JIT"
1237
+ else
1238
+ $log.warn "failed to enable Ruby JIT"
1239
+ end
1240
+ end
1241
+
1218
1242
  def build_system_config(conf)
1219
1243
  system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
1220
1244
  # Prefer the options explicitly specified in the command line
@@ -1282,11 +1306,6 @@ module Fluent
1282
1306
  fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
1283
1307
  end
1284
1308
 
1285
- if @system_config.enable_jit
1286
- $log.info "enable Ruby JIT for workers (--jit)"
1287
- fluentd_spawn_cmd << '--jit'
1288
- end
1289
-
1290
1309
  # Adding `-h` so that it can avoid ruby's command blocking
1291
1310
  # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
1292
1311
  _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")