fluentd 1.13.3-x86-mingw32 → 1.14.3-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +6 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +126 -0
  6. data/README.md +2 -0
  7. data/SECURITY.md +18 -0
  8. data/fluentd.gemspec +3 -3
  9. data/lib/fluent/command/cat.rb +13 -3
  10. data/lib/fluent/command/fluentd.rb +8 -0
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/event_router.rb +28 -1
  15. data/lib/fluent/plugin/bare_output.rb +49 -8
  16. data/lib/fluent/plugin/buf_file.rb +2 -2
  17. data/lib/fluent/plugin/buffer.rb +123 -27
  18. data/lib/fluent/plugin/filter.rb +35 -1
  19. data/lib/fluent/plugin/in_http.rb +21 -2
  20. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  21. data/lib/fluent/plugin/in_syslog.rb +13 -1
  22. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  23. data/lib/fluent/plugin/in_tail.rb +47 -8
  24. data/lib/fluent/plugin/input.rb +39 -1
  25. data/lib/fluent/plugin/metrics.rb +119 -0
  26. data/lib/fluent/plugin/metrics_local.rb +96 -0
  27. data/lib/fluent/plugin/multi_output.rb +43 -6
  28. data/lib/fluent/plugin/out_copy.rb +1 -1
  29. data/lib/fluent/plugin/out_forward.rb +15 -7
  30. data/lib/fluent/plugin/output.rb +80 -38
  31. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  32. data/lib/fluent/plugin/storage_local.rb +3 -5
  33. data/lib/fluent/plugin.rb +10 -1
  34. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  35. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  36. data/lib/fluent/plugin_helper/server.rb +4 -2
  37. data/lib/fluent/plugin_helper.rb +1 -0
  38. data/lib/fluent/plugin_id.rb +2 -1
  39. data/lib/fluent/root_agent.rb +6 -0
  40. data/lib/fluent/supervisor.rb +4 -2
  41. data/lib/fluent/system_config.rb +9 -1
  42. data/lib/fluent/time.rb +21 -20
  43. data/lib/fluent/version.rb +1 -1
  44. data/test/command/test_cat.rb +31 -2
  45. data/test/config/test_system_config.rb +6 -0
  46. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  47. data/test/plugin/in_tail/test_position_file.rb +26 -4
  48. data/test/plugin/test_bare_output.rb +13 -0
  49. data/test/plugin/test_buffer.rb +80 -3
  50. data/test/plugin/test_filter.rb +11 -0
  51. data/test/plugin/test_in_http.rb +40 -0
  52. data/test/plugin/test_in_monitor_agent.rb +214 -8
  53. data/test/plugin/test_in_syslog.rb +35 -0
  54. data/test/plugin/test_in_tail.rb +83 -35
  55. data/test/plugin/test_input.rb +11 -0
  56. data/test/plugin/test_metrics.rb +294 -0
  57. data/test/plugin/test_metrics_local.rb +96 -0
  58. data/test/plugin/test_multi_output.rb +25 -1
  59. data/test/plugin/test_out_exec_filter.rb +4 -0
  60. data/test/plugin/test_output.rb +16 -0
  61. data/test/plugin_helper/test_child_process.rb +9 -9
  62. data/test/plugin_helper/test_event_emitter.rb +29 -0
  63. data/test/plugin_helper/test_metrics.rb +137 -0
  64. data/test/test_plugin_classes.rb +102 -0
  65. data/test/test_root_agent.rb +30 -1
  66. data/test/test_time_parser.rb +22 -0
  67. metadata +18 -8
@@ -16,6 +16,8 @@
16
16
 
17
17
  require 'fluent/plugin/base'
18
18
  require 'fluent/plugin/owned_by_mixin'
19
+ require 'fluent/plugin_id'
20
+ require 'fluent/plugin_helper'
19
21
  require 'fluent/unique_id'
20
22
  require 'fluent/ext_monitor_require'
21
23
 
@@ -24,7 +26,9 @@ module Fluent
24
26
  class Buffer < Base
25
27
  include OwnedByMixin
26
28
  include UniqueId::Mixin
29
+ include PluginId
27
30
  include MonitorMixin
31
+ include PluginHelper::Mixin # for metrics
28
32
 
29
33
  class BufferError < StandardError; end
30
34
  class BufferOverflowError < BufferError; end
@@ -39,6 +43,8 @@ module Fluent
39
43
 
40
44
  configured_in :buffer
41
45
 
46
+ helpers_internal :metrics
47
+
42
48
  # TODO: system total buffer limit size in bytes by SystemConfig
43
49
 
44
50
  config_param :chunk_limit_size, :size, default: DEFAULT_CHUNK_LIMIT_SIZE
@@ -153,8 +159,11 @@ module Fluent
153
159
  end
154
160
  end
155
161
 
162
+ # for metrics
163
+ attr_reader :stage_size_metrics, :stage_length_metrics, :queue_size_metrics, :queue_length_metrics
164
+ attr_reader :available_buffer_space_ratios_metrics, :total_queued_size_metrics
165
+ attr_reader :newest_timekey_metrics, :oldest_timekey_metrics
156
166
  # for tests
157
- attr_accessor :stage_size, :queue_size
158
167
  attr_reader :stage, :queue, :dequeued, :queued_num
159
168
 
160
169
  def initialize
@@ -171,12 +180,35 @@ module Fluent
171
180
  @queued_num = {} # metadata => int (number of queued chunks)
172
181
  @dequeued_num = {} # metadata => int (number of dequeued chunks)
173
182
 
174
- @stage_size = @queue_size = 0
183
+ @stage_length_metrics = nil
184
+ @stage_size_metrics = nil
185
+ @queue_length_metrics = nil
186
+ @queue_size_metrics = nil
187
+ @available_buffer_space_ratios_metrics = nil
188
+ @total_queued_size_metrics = nil
189
+ @newest_timekey_metrics = nil
190
+ @oldest_timekey_metrics = nil
175
191
  @timekeys = Hash.new(0)
176
192
  @enable_update_timekeys = false
177
193
  @mutex = Mutex.new
178
194
  end
179
195
 
196
+ def stage_size
197
+ @stage_size_metrics.get
198
+ end
199
+
200
+ def stage_size=(value)
201
+ @stage_size_metrics.set(value)
202
+ end
203
+
204
+ def queue_size
205
+ @queue_size_metrics.get
206
+ end
207
+
208
+ def queue_size=(value)
209
+ @queue_size_metrics.set(value)
210
+ end
211
+
180
212
  def persistent?
181
213
  false
182
214
  end
@@ -187,6 +219,28 @@ module Fluent
187
219
  unless @queue_limit_length.nil?
188
220
  @total_limit_size = @chunk_limit_size * @queue_limit_length
189
221
  end
222
+ @stage_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_length",
223
+ help_text: 'Length of stage buffers', prefer_gauge: true)
224
+ @stage_length_metrics.set(0)
225
+ @stage_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_byte_size",
226
+ help_text: 'Total size of stage buffers', prefer_gauge: true)
227
+ @stage_size_metrics.set(0) # Ensure zero.
228
+ @queue_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "queue_length",
229
+ help_text: 'Length of queue buffers', prefer_gauge: true)
230
+ @queue_length_metrics.set(0)
231
+ @queue_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "queue_byte_size",
232
+ help_text: 'Total size of queue buffers', prefer_gauge: true)
233
+ @queue_size_metrics.set(0) # Ensure zero.
234
+ @available_buffer_space_ratios_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "available_buffer_space_ratios",
235
+ help_text: 'Ratio of available space in buffer', prefer_gauge: true)
236
+ @available_buffer_space_ratios_metrics.set(100) # Default is 100%.
237
+ @total_queued_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "total_queued_size",
238
+ help_text: 'Total size of stage and queue buffers', prefer_gauge: true)
239
+ @total_queued_size_metrics.set(0)
240
+ @newest_timekey_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "newest_timekey",
241
+ help_text: 'Newest timekey in buffer', prefer_gauge: true)
242
+ @oldest_timekey_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "oldest_timekey",
243
+ help_text: 'Oldest timekey in buffer', prefer_gauge: true)
190
244
  end
191
245
 
192
246
  def enable_update_timekeys
@@ -198,15 +252,15 @@ module Fluent
198
252
 
199
253
  @stage, @queue = resume
200
254
  @stage.each_pair do |metadata, chunk|
201
- @stage_size += chunk.bytesize
255
+ @stage_size_metrics.add(chunk.bytesize)
202
256
  end
203
257
  @queue.each do |chunk|
204
258
  @queued_num[chunk.metadata] ||= 0
205
259
  @queued_num[chunk.metadata] += 1
206
- @queue_size += chunk.bytesize
260
+ @queue_size_metrics.add(chunk.bytesize)
207
261
  end
208
262
  update_timekeys
209
- log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
263
+ log.debug "buffer started", instance: self.object_id, stage_size: @stage_size_metrics.get, queue_size: @queue_size_metrics.get
210
264
  end
211
265
 
212
266
  def close
@@ -228,17 +282,19 @@ module Fluent
228
282
  def terminate
229
283
  super
230
284
  @dequeued = @stage = @queue = @queued_num = nil
231
- @stage_size = @queue_size = 0
285
+ @stage_length_metrics = @stage_size_metrics = @queue_length_metrics = @queue_size_metrics = nil
286
+ @available_buffer_space_ratios_metrics = @total_queued_size_metrics = nil
287
+ @newest_timekey_metrics = @oldest_timekey_metrics = nil
232
288
  @timekeys.clear
233
289
  end
234
290
 
235
291
  def storable?
236
- @total_limit_size > @stage_size + @queue_size
292
+ @total_limit_size > @stage_size_metrics.get + @queue_size_metrics.get
237
293
  end
238
294
 
239
295
  ## TODO: for back pressure feature
240
296
  # def used?(ratio)
241
- # @total_limit_size * ratio > @stage_size + @queue_size
297
+ # @total_limit_size * ratio > @stage_size_metrics.get + @queue_size_metrics.get
242
298
  # end
243
299
 
244
300
  def resume
@@ -276,12 +332,14 @@ module Fluent
276
332
  unstaged_chunks = {} # metadata => [chunk, chunk, ...]
277
333
  chunks_to_enqueue = []
278
334
  staged_bytesizes_by_chunk = {}
335
+ # track internal BufferChunkOverflowError in write_step_by_step
336
+ buffer_chunk_overflow_errors = []
279
337
 
280
338
  begin
281
339
  # sort metadata to get lock of chunks in same order with other threads
282
340
  metadata_and_data.keys.sort.each do |metadata|
283
341
  data = metadata_and_data[metadata]
284
- write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
342
+ write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize, error|
285
343
  chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
286
344
  operated_chunks << chunk
287
345
  if chunk.staged?
@@ -296,6 +354,9 @@ module Fluent
296
354
  unstaged_chunks[metadata] ||= []
297
355
  unstaged_chunks[metadata] << chunk
298
356
  end
357
+ if error && !error.empty?
358
+ buffer_chunk_overflow_errors << error
359
+ end
299
360
  end
300
361
  end
301
362
 
@@ -344,7 +405,7 @@ module Fluent
344
405
  #
345
406
  staged_bytesizes_by_chunk.each do |chunk, bytesize|
346
407
  chunk.synchronize do
347
- synchronize { @stage_size += bytesize }
408
+ synchronize { @stage_size_metrics.add(bytesize) }
348
409
  log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
349
410
  end
350
411
  end
@@ -361,7 +422,7 @@ module Fluent
361
422
  u.metadata.seq = 0
362
423
  synchronize {
363
424
  @stage[m] = u.staged!
364
- @stage_size += u.bytesize
425
+ @stage_size_metrics.add(u.bytesize)
365
426
  }
366
427
  end
367
428
  end
@@ -388,6 +449,10 @@ module Fluent
388
449
  end
389
450
  chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
390
451
  end
452
+ unless buffer_chunk_overflow_errors.empty?
453
+ # Notify delayed BufferChunkOverflowError here
454
+ raise BufferChunkOverflowError, buffer_chunk_overflow_errors.join(", ")
455
+ end
391
456
  end
392
457
  end
393
458
 
@@ -428,8 +493,8 @@ module Fluent
428
493
  chunk.enqueued!
429
494
  end
430
495
  bytesize = chunk.bytesize
431
- @stage_size -= bytesize
432
- @queue_size += bytesize
496
+ @stage_size_metrics.sub(bytesize)
497
+ @queue_size_metrics.add(bytesize)
433
498
  end
434
499
  end
435
500
  nil
@@ -446,7 +511,7 @@ module Fluent
446
511
  @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
447
512
  chunk.enqueued!
448
513
  end
449
- @queue_size += chunk.bytesize
514
+ @queue_size_metrics.add(chunk.bytesize)
450
515
  end
451
516
  end
452
517
 
@@ -531,7 +596,7 @@ module Fluent
531
596
  begin
532
597
  bytesize = chunk.bytesize
533
598
  chunk.purge
534
- @queue_size -= bytesize
599
+ @queue_size_metrics.sub(bytesize)
535
600
  rescue => e
536
601
  log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
537
602
  log.error_backtrace
@@ -562,7 +627,7 @@ module Fluent
562
627
  log.error_backtrace
563
628
  end
564
629
  end
565
- @queue_size = 0
630
+ @queue_size_metrics.set(0)
566
631
  end
567
632
  end
568
633
 
@@ -660,6 +725,7 @@ module Fluent
660
725
 
661
726
  def write_step_by_step(metadata, data, format, splits_count, &block)
662
727
  splits = []
728
+ errors = []
663
729
  if splits_count > data.size
664
730
  splits_count = data.size
665
731
  end
@@ -705,18 +771,41 @@ module Fluent
705
771
  begin
706
772
  while writing_splits_index < splits.size
707
773
  split = splits[writing_splits_index]
774
+ formatted_split = format ? format.call(split) : split.first
775
+ if split.size == 1 && original_bytesize == 0
776
+ if format == nil && @compress != :text
777
+ # The actual size of chunk is not determined until after chunk.append.
778
+ # so, keep already processed 'split' content here.
779
+ # (allow performance regression a bit)
780
+ chunk.commit
781
+ else
782
+ big_record_size = formatted_split.bytesize
783
+ if chunk.bytesize + big_record_size > @chunk_limit_size
784
+ errors << "a #{big_record_size} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
785
+ writing_splits_index += 1
786
+ next
787
+ end
788
+ end
789
+ end
790
+
708
791
  if format
709
- chunk.concat(format.call(split), split.size)
792
+ chunk.concat(formatted_split, split.size)
710
793
  else
711
794
  chunk.append(split, compress: @compress)
712
795
  end
713
796
 
714
797
  if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
798
+ adding_bytes = chunk.instance_eval { @adding_bytes } || "N/A" # 3rd party might not have 'adding_bytes'
715
799
  chunk.rollback
716
800
 
717
801
  if split.size == 1 && original_bytesize == 0
718
- big_record_size = format ? format.call(split).bytesize : split.first.bytesize
719
- raise BufferChunkOverflowError, "a #{big_record_size}bytes record is larger than buffer chunk limit size"
802
+ # It is obviously case that BufferChunkOverflowError should be raised here,
803
+ # but if it raises here, already processed 'split' or
804
+ # the proceeding 'split' will be lost completely.
805
+ # so it is a last resort to delay raising such a exception
806
+ errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
807
+ writing_splits_index += 1
808
+ next
720
809
  end
721
810
 
722
811
  if chunk_size_full?(chunk) || split.size == 1
@@ -739,7 +828,8 @@ module Fluent
739
828
  raise
740
829
  end
741
830
 
742
- block.call(chunk, chunk.bytesize - original_bytesize)
831
+ block.call(chunk, chunk.bytesize - original_bytesize, errors)
832
+ errors = []
743
833
  end
744
834
  end
745
835
  rescue ShouldRetry
@@ -765,23 +855,29 @@ module Fluent
765
855
  ]
766
856
 
767
857
  def statistics
768
- stage_size, queue_size = @stage_size, @queue_size
858
+ stage_size, queue_size = @stage_size_metrics.get, @queue_size_metrics.get
769
859
  buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)
860
+ @stage_length_metrics.set(@stage.size)
861
+ @queue_length_metrics.set(@queue.size)
862
+ @available_buffer_space_ratios_metrics.set(buffer_space * 100)
863
+ @total_queued_size_metrics.set(stage_size + queue_size)
770
864
  stats = {
771
- 'stage_length' => @stage.size,
865
+ 'stage_length' => @stage_length_metrics.get,
772
866
  'stage_byte_size' => stage_size,
773
- 'queue_length' => @queue.size,
867
+ 'queue_length' => @queue_length_metrics.get,
774
868
  'queue_byte_size' => queue_size,
775
- 'available_buffer_space_ratios' => (buffer_space * 100).round(1),
776
- 'total_queued_size' => stage_size + queue_size,
869
+ 'available_buffer_space_ratios' => @available_buffer_space_ratios_metrics.get.round(1),
870
+ 'total_queued_size' => @total_queued_size_metrics.get,
777
871
  }
778
872
 
779
873
  tkeys = timekeys
780
874
  if (m = tkeys.min)
781
- stats['oldest_timekey'] = m
875
+ @oldest_timekey_metrics.set(m)
876
+ stats['oldest_timekey'] = @oldest_timekey_metrics.get
782
877
  end
783
878
  if (m = tkeys.max)
784
- stats['newest_timekey'] = m
879
+ @newest_timekey_metrics.set(m)
880
+ stats['newest_timekey'] = @newest_timekey_metrics.get
785
881
  end
786
882
 
787
883
  { 'buffer' => stats }
@@ -28,13 +28,47 @@ module Fluent
28
28
  include PluginLoggerMixin
29
29
  include PluginHelper::Mixin
30
30
 
31
- helpers_internal :event_emitter
31
+ helpers_internal :event_emitter, :metrics
32
32
 
33
33
  attr_reader :has_filter_with_time
34
34
 
35
35
  def initialize
36
36
  super
37
37
  @has_filter_with_time = has_filter_with_time?
38
+ @emit_records_metrics = nil
39
+ @emit_size_metrics = nil
40
+ @counter_mutex = Mutex.new
41
+ @enable_size_metrics = false
42
+ end
43
+
44
+ def emit_records
45
+ @emit_records_metrics.get
46
+ end
47
+
48
+ def emit_size
49
+ @emit_size_metrics.get
50
+ end
51
+
52
+ def configure(conf)
53
+ super
54
+
55
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_records", help_text: "Number of count emit records")
56
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_size", help_text: "Total size of emit events")
57
+ @enable_size_metrics = !!system_config.enable_size_metrics
58
+ end
59
+
60
+ def statistics
61
+ stats = {
62
+ 'emit_records' => @emit_records_metrics.get,
63
+ 'emit_size' => @emit_size_metrics.get,
64
+ }
65
+
66
+ { 'filter' => stats }
67
+ end
68
+
69
+ def measure_metrics(es)
70
+ @emit_records_metrics.add(es.size)
71
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
38
72
  end
39
73
 
40
74
  def filter(tag, time, record)
@@ -74,6 +74,8 @@ module Fluent::Plugin
74
74
  config_param :blocking_timeout, :time, default: 0.5
75
75
  desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
76
76
  config_param :cors_allow_origins, :array, default: nil
77
+ desc 'Tells browsers whether to expose the response to frontend when the credentials mode is "include".'
78
+ config_param :cors_allow_credentials, :bool, default: false
77
79
  desc 'Respond with empty gif image of 1x1 pixel.'
78
80
  config_param :respond_with_empty_img, :bool, default: false
79
81
  desc 'Respond status code with 204.'
@@ -112,6 +114,12 @@ module Fluent::Plugin
112
114
 
113
115
  super
114
116
 
117
+ if @cors_allow_credentials
118
+ if @cors_allow_origins.nil? || @cors_allow_origins.include?('*')
119
+ raise Fluent::ConfigError, "Cannot enable cors_allow_credentials without specific origins"
120
+ end
121
+ end
122
+
115
123
  m = if @parser_configs.first['@type'] == 'in_http'
116
124
  @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
117
125
  @parser_msgpack.time_key = nil
@@ -279,7 +287,10 @@ module Fluent::Plugin
279
287
  private
280
288
 
281
289
  def on_server_connect(conn)
282
- handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
290
+ handler = Handler.new(conn, @km, method(:on_request),
291
+ @body_size_limit, @format_name, log,
292
+ @cors_allow_origins, @cors_allow_credentials,
293
+ @add_query_params)
283
294
 
284
295
  conn.on(:data) do |data|
285
296
  handler.on_read(data)
@@ -356,7 +367,8 @@ module Fluent::Plugin
356
367
  class Handler
357
368
  attr_reader :content_type
358
369
 
359
- def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
370
+ def initialize(io, km, callback, body_size_limit, format_name, log,
371
+ cors_allow_origins, cors_allow_credentials, add_query_params)
360
372
  @io = io
361
373
  @km = km
362
374
  @callback = callback
@@ -365,6 +377,7 @@ module Fluent::Plugin
365
377
  @format_name = format_name
366
378
  @log = log
367
379
  @cors_allow_origins = cors_allow_origins
380
+ @cors_allow_credentials = cors_allow_credentials
368
381
  @idle = 0
369
382
  @add_query_params = add_query_params
370
383
  @km.add(self)
@@ -491,6 +504,9 @@ module Fluent::Plugin
491
504
  send_response_and_close(RES_200_STATUS, header, "")
492
505
  elsif include_cors_allow_origin
493
506
  header["Access-Control-Allow-Origin"] = @origin
507
+ if @cors_allow_credentials
508
+ header["Access-Control-Allow-Credentials"] = true
509
+ end
494
510
  send_response_and_close(RES_200_STATUS, header, "")
495
511
  else
496
512
  send_response_and_close(RES_403_STATUS, {}, "")
@@ -576,6 +592,9 @@ module Fluent::Plugin
576
592
  header['Access-Control-Allow-Origin'] = '*'
577
593
  elsif include_cors_allow_origin
578
594
  header['Access-Control-Allow-Origin'] = @origin
595
+ if @cors_allow_credentials
596
+ header["Access-Control-Allow-Credentials"] = true
597
+ end
579
598
  end
580
599
  end
581
600
 
@@ -238,7 +238,7 @@ module Fluent::Plugin
238
238
  'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
239
239
  'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
240
240
  'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
241
- 'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
241
+ 'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },
242
242
  }
243
243
 
244
244
  def all_plugins
@@ -335,7 +335,9 @@ module Fluent::Plugin
335
335
  }
336
336
 
337
337
  if pe.respond_to?(:statistics)
338
- obj.merge!(pe.statistics['output'] || {})
338
+ obj.merge!(pe.statistics.dig('output') || {})
339
+ obj.merge!(pe.statistics.dig('filter') || {})
340
+ obj.merge!(pe.statistics.dig('input') || {})
339
341
  end
340
342
 
341
343
  obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
@@ -89,6 +89,8 @@ module Fluent::Plugin
89
89
  config_param :source_hostname_key, :string, default: nil
90
90
  desc 'Try to resolve hostname from IP addresses or not.'
91
91
  config_param :resolve_hostname, :bool, default: nil
92
+ desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
93
+ config_param :send_keepalive_packet, :bool, default: false
92
94
  desc 'The field name of source address of sender.'
93
95
  config_param :source_address_key, :string, default: nil
94
96
  desc 'The field name of the severity.'
@@ -143,6 +145,11 @@ module Fluent::Plugin
143
145
  end
144
146
 
145
147
  @_event_loop_run_timeout = @blocking_timeout
148
+
149
+ protocol = @protocol_type || @transport_config.protocol
150
+ if @send_keepalive_packet && protocol == :udp
151
+ raise Fluent::ConfigError, "send_keepalive_packet is available for tcp/tls"
152
+ end
146
153
  end
147
154
 
148
155
  def multi_workers_ready?
@@ -173,7 +180,12 @@ module Fluent::Plugin
173
180
 
174
181
  delimiter = octet_count_frame ? " " : @delimiter
175
182
  delimiter_size = delimiter.size
176
- server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
183
+ server_create_connection(
184
+ tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port,
185
+ bind: @bind,
186
+ resolve_name: @resolve_hostname,
187
+ send_keepalive_packet: @send_keepalive_packet
188
+ ) do |conn|
177
189
  conn.data do |data|
178
190
  buffer = conn.buffer
179
191
  buffer << data
@@ -158,7 +158,7 @@ module Fluent::Plugin
158
158
  end
159
159
  end
160
160
 
161
- entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
161
+ entries = remove_deleted_files_entries(entries, existing_targets)
162
162
  entries
163
163
  end
164
164