fluentd 1.14.0-x86-mingw32 → 1.14.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 585487ebe4fdf640b517d8673d446dcc683acae55d6a8b58a3255e3e523e6824
4
- data.tar.gz: e4ce59e08623ad835cf979e870d46f98fcc89a43eac7d180ace8e88e4bda7e64
3
+ metadata.gz: d5f5c1c3b6598d0eb25f9372beda5e5b08932b8facda51aa7e3f9ef9a98849cd
4
+ data.tar.gz: 6569c24837680722e497009210e7bf64b1b3a50c9fe4b4ef4d6a89db8cbb0faf
5
5
  SHA512:
6
- metadata.gz: 14a8b359210ce14d816009b9afecbd7f749b68c96655fad2d6ea7700991bba771ca3834ecc4916c8edb26d69ad5f01006b4674a1b21fb4640c390457bd9835dc
7
- data.tar.gz: a7dd58d022d2388fd4c2de96ce454405eb7bdba9cb2aba87379849567958e2da47591def7d5baac18f09977a9fde4ec356aea9ca409f7c204e73fc0965f772d6
6
+ metadata.gz: 3bdfc7de632946e7e0ac8ee65c7ad4334399dee5471021da21fb15484c08471508af2ccd0b28f700850b57869aac29ced58b0e2d60bac1fc6bdc6a8373a4a41b
7
+ data.tar.gz: 31dac402718a9e78416f2481704eec0ac35c154006c14f73c60c62149dc83bf53d55d3aad1b94a57f9dd45ae158b4eac979542de4e2d90ff41339881c16175e0
data/CHANGELOG.md CHANGED
@@ -1,3 +1,34 @@
1
+ # v1.14.1
2
+
3
+ ## Release v1.14.1 - 2021/09/29
4
+
5
+ ### Enhancement
6
+
7
+ * in_tail: Added file related metrics.
8
+ These metrics should be collected same as fluent-bit's in_tail.
9
+ https://github.com/fluent/fluentd/pull/3504
10
+ * out_forward: Changed to use metrics mechanism for node statistics
11
+ https://github.com/fluent/fluentd/pull/3506
12
+
13
+ ### Bug fix
14
+
15
+ * in_tail: Fixed a crash bug that it raise undefined method of eof? error.
16
+ This error may happen only when `read_bytes_limit_per_second` was specified.
17
+ https://github.com/fluent/fluentd/pull/3500
18
+ * out_forward: Fixed a bug that node statistics information is not included correctly.
19
+ https://github.com/fluent/fluentd/pull/3503
20
+ https://github.com/fluent/fluentd/pull/3507
21
+ * Fixed a error when using `@include` directive
22
+ It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
23
+ https://github.com/fluent/fluentd/pull/3517
24
+ * out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
25
+ It didn't work even if a user set it.
26
+ https://github.com/fluent/fluentd/pull/3515
27
+ * Fixed not to output nanoseconds field of next retry time in warning log
28
+ Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
29
+ next_retry_seconds -> next_retry_time)
30
+ https://github.com/fluent/fluentd/pull/3518
31
+
1
32
  # v1.14.0
2
33
 
3
34
  ## Release v1.14.0 - 2021/08/30
@@ -93,7 +93,7 @@ module Fluent
93
93
  basepath = '/'
94
94
  fname = path
95
95
  require 'open-uri'
96
- open(uri) {|f|
96
+ URI.open(uri) {|f|
97
97
  Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
98
98
  }
99
99
  end
@@ -172,7 +172,7 @@ module Fluent
172
172
  require 'open-uri'
173
173
  basepath = '/'
174
174
  fname = path
175
- data = open(uri) { |f| f.read }
175
+ data = URI.open(uri) { |f| f.read }
176
176
  data.force_encoding('UTF-8')
177
177
  ss = StringScanner.new(data)
178
178
  V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
@@ -38,6 +38,7 @@ module Fluent::Plugin
38
38
  helpers :timer, :event_loop, :parser, :compat_parameters
39
39
 
40
40
  RESERVED_CHARS = ['/', '*', '%'].freeze
41
+ MetricsInfo = Struct.new(:opened, :closed, :rotated)
41
42
 
42
43
  class WatcherSetupError < StandardError
43
44
  def initialize(msg)
@@ -57,6 +58,7 @@ module Fluent::Plugin
57
58
  @pf = nil
58
59
  @ignore_list = []
59
60
  @shutdown_start_time = nil
61
+ @metrics = nil
60
62
  end
61
63
 
62
64
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -191,6 +193,10 @@ module Fluent::Plugin
191
193
  @read_bytes_limit_per_second = min_bytes
192
194
  end
193
195
  end
196
+ opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
197
+ closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
198
+ rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
199
+ @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
194
200
  end
195
201
 
196
202
  def configure_tag
@@ -375,7 +381,7 @@ module Fluent::Plugin
375
381
 
376
382
  def setup_watcher(target_info, pe)
377
383
  line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
378
- tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
384
+ tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
379
385
 
380
386
  if @enable_watch_timer
381
387
  tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -670,6 +676,19 @@ module Fluent::Plugin
670
676
  es
671
677
  end
672
678
 
679
+ def statistics
680
+ stats = super
681
+
682
+ stats = {
683
+ 'input' => stats["input"].merge({
684
+ 'opened_file_count' => @metrics.opened.get,
685
+ 'closed_file_count' => @metrics.closed.get,
686
+ 'rotated_file_count' => @metrics.rotated.get,
687
+ })
688
+ }
689
+ stats
690
+ end
691
+
673
692
  private
674
693
 
675
694
  def io_handler(watcher, path)
@@ -682,6 +701,7 @@ module Fluent::Plugin
682
701
  open_on_every_update: @open_on_every_update,
683
702
  from_encoding: @from_encoding,
684
703
  encoding: @encoding,
704
+ metrics: @metrics,
685
705
  &method(:receive_lines)
686
706
  )
687
707
  end
@@ -717,7 +737,7 @@ module Fluent::Plugin
717
737
  end
718
738
 
719
739
  class TailWatcher
720
- def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
740
+ def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
721
741
  @path = target_info.path
722
742
  @ino = target_info.ino
723
743
  @pe = pe || MemoryPositionEntry.new
@@ -729,6 +749,7 @@ module Fluent::Plugin
729
749
  @line_buffer_timer_flusher = line_buffer_timer_flusher
730
750
  @io_handler = nil
731
751
  @io_handler_build = io_handler_build
752
+ @metrics = metrics
732
753
  @watchers = []
733
754
  end
734
755
 
@@ -762,7 +783,7 @@ module Fluent::Plugin
762
783
  end
763
784
 
764
785
  def eof?
765
- @io_handler.eof?
786
+ @io_handler.nil? || @io_handler.eof?
766
787
  end
767
788
 
768
789
  def on_notify
@@ -855,6 +876,7 @@ module Fluent::Plugin
855
876
  @log.info "detected rotation of #{@path}"
856
877
  @io_handler = io_handler
857
878
  end
879
+ @metrics.rotated.inc
858
880
  end
859
881
  end
860
882
 
@@ -934,7 +956,7 @@ module Fluent::Plugin
934
956
 
935
957
  attr_accessor :shutdown_timeout
936
958
 
937
- def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
959
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
938
960
  @watcher = watcher
939
961
  @path = path
940
962
  @read_lines_limit = read_lines_limit
@@ -953,6 +975,7 @@ module Fluent::Plugin
953
975
  @shutdown_timeout = SHUTDOWN_TIMEOUT
954
976
  @shutdown_mutex = Mutex.new
955
977
  @eof = false
978
+ @metrics = metrics
956
979
 
957
980
  @log.info "following tail of #{@path}"
958
981
  end
@@ -972,6 +995,7 @@ module Fluent::Plugin
972
995
  if @io && !@io.closed?
973
996
  @io.close
974
997
  @io = nil
998
+ @metrics.closed.inc
975
999
  end
976
1000
  end
977
1001
 
@@ -1059,6 +1083,7 @@ module Fluent::Plugin
1059
1083
  def open
1060
1084
  io = Fluent::FileWrapper.open(@path)
1061
1085
  io.seek(@watcher.pe.read_pos + @fifo.bytesize)
1086
+ @metrics.opened.inc
1062
1087
  io
1063
1088
  rescue RangeError
1064
1089
  io.close if io
@@ -1111,6 +1136,10 @@ module Fluent::Plugin
1111
1136
  def opened?
1112
1137
  false
1113
1138
  end
1139
+
1140
+ def eof?
1141
+ true
1142
+ end
1114
1143
  end
1115
1144
 
1116
1145
  class RotateHandler
@@ -46,7 +46,7 @@ module Fluent::Plugin
46
46
  @ignore_errors << (store.arg.include?('ignore_error'))
47
47
  @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
48
48
  }
49
- if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
49
+ if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
50
50
  log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
51
51
  end
52
52
  end
@@ -167,6 +167,8 @@ module Fluent::Plugin
167
167
  @usock = nil
168
168
  @keep_alive_watcher_interval = 5 # TODO
169
169
  @suspend_flush = false
170
+ @healthy_nodes_count_metrics = nil
171
+ @registered_nodes_count_metrics = nil
170
172
  end
171
173
 
172
174
  def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
265
267
  end
266
268
 
267
269
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
270
+ @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
271
+ @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
272
+
268
273
  end
269
274
 
270
275
  def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
418
423
  def statistics
419
424
  stats = super
420
425
  services = service_discovery_services
421
- healthy_nodes_count = 0
422
- registed_nodes_count = services.size
426
+ @healthy_nodes_count_metrics.set(0)
427
+ @registered_nodes_count_metrics.set(services.size)
423
428
  services.each do |s|
424
429
  if s.available?
425
- healthy_nodes_count += 1
430
+ @healthy_nodes_count_metrics.inc
426
431
  end
427
432
  end
428
433
 
429
- stats.merge(
430
- 'healthy_nodes_count' => healthy_nodes_count,
431
- 'registered_nodes_count' => registed_nodes_count,
432
- )
434
+ stats = {
435
+ 'output' => stats["output"].merge({
436
+ 'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
437
+ 'registered_nodes_count' => @registered_nodes_count_metrics.get,
438
+ })
439
+ }
440
+ stats
433
441
  end
434
442
 
435
443
  # MessagePack FixArray length is 3
@@ -1275,7 +1275,7 @@ module Fluent
1275
1275
  unless @retry
1276
1276
  @retry = retry_state(@buffer_config.retry_randomize)
1277
1277
  if error
1278
- log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1278
+ log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1279
1279
  log.warn_backtrace error.backtrace
1280
1280
  end
1281
1281
  return
@@ -1304,11 +1304,11 @@ module Fluent
1304
1304
  if error
1305
1305
  if using_secondary
1306
1306
  msg = "failed to flush the buffer with secondary output."
1307
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1307
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1308
1308
  log.warn_backtrace error.backtrace
1309
1309
  else
1310
1310
  msg = "failed to flush the buffer."
1311
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1311
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1312
1312
  log.warn_backtrace error.backtrace
1313
1313
  end
1314
1314
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.14.0'
19
+ VERSION = '1.14.1'
20
20
 
21
21
  end
@@ -1,11 +1,19 @@
1
1
  require_relative '../../helper'
2
2
 
3
3
  require 'fluent/plugin/in_tail'
4
+ require 'fluent/plugin/metrics_local'
4
5
  require 'tempfile'
5
6
 
6
7
  class IntailIOHandlerTest < Test::Unit::TestCase
7
8
  setup do
8
9
  @file = Tempfile.new('intail_io_handler').binmode
10
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
11
+ opened_file_metrics.configure(config_element('metrics', '', {}))
12
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
13
+ closed_file_metrics.configure(config_element('metrics', '', {}))
14
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
15
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
16
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
9
17
  end
10
18
 
11
19
  teardown do
@@ -30,7 +38,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
30
38
  end
31
39
 
32
40
  returned_lines = ''
33
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
41
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
34
42
  returned_lines << lines.join
35
43
  true
36
44
  end
@@ -62,7 +70,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
62
70
  end
63
71
 
64
72
  returned_lines = ''
65
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
73
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|
66
74
  returned_lines << lines.join
67
75
  true
68
76
  end
@@ -93,7 +101,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
93
101
  end
94
102
 
95
103
  returned_lines = []
96
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
104
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
97
105
  returned_lines << lines.dup
98
106
  true
99
107
  end
@@ -119,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
119
127
  end
120
128
 
121
129
  returned_lines = []
122
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
130
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
123
131
  returned_lines << lines.dup
124
132
  true
125
133
  end
@@ -1524,11 +1524,18 @@ class TailInputTest < Test::Unit::TestCase
1524
1524
  plugin.instance_eval do
1525
1525
  @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1526
1526
  @loop = Coolio::Loop.new
1527
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1528
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1529
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1530
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1531
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1532
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1533
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1527
1534
  end
1528
1535
 
1529
1536
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1530
1537
  ex_paths.each do |target_info|
1531
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1538
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1532
1539
  end
1533
1540
 
1534
1541
  plugin.refresh_watchers
@@ -1542,7 +1549,7 @@ class TailInputTest < Test::Unit::TestCase
1542
1549
  path = "test/plugin/data/2010/01/20100102-030406.log"
1543
1550
  inode = Fluent::FileWrapper.stat(path).ino
1544
1551
  target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1545
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1552
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1546
1553
  plugin.refresh_watchers
1547
1554
 
1548
1555
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1885,13 +1892,22 @@ class TailInputTest < Test::Unit::TestCase
1885
1892
  config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1886
1893
 
1887
1894
  d = create_driver(config, false)
1895
+ d.instance.instance_eval do
1896
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1897
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1898
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1899
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1900
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1901
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1902
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1903
+ end
1888
1904
 
1889
1905
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1890
1906
  f.puts "test1"
1891
1907
  f.puts "test2"
1892
1908
  }
1893
1909
  target_info = create_target_info("#{TMP_DIR}/tail.txt")
1894
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
1910
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
1895
1911
  d.run(shutdown: false)
1896
1912
  assert d.instance.instance_variable_get(:@tails)[target_info]
1897
1913
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.14.0
4
+ version: 1.14.1
5
5
  platform: x86-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-30 00:00:00.000000000 Z
11
+ date: 2021-09-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler