fluentd 1.13.1-x64-mingw32 → 1.14.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +136 -0
  6. data/README.md +2 -2
  7. data/example/v0_12_filter.conf +2 -2
  8. data/fluentd.gemspec +1 -1
  9. data/lib/fluent/command/fluentd.rb +8 -0
  10. data/lib/fluent/command/plugin_generator.rb +15 -5
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/types.rb +15 -0
  14. data/lib/fluent/config/v1_parser.rb +4 -3
  15. data/lib/fluent/config.rb +1 -1
  16. data/lib/fluent/env.rb +2 -1
  17. data/lib/fluent/event_router.rb +28 -1
  18. data/lib/fluent/oj_options.rb +62 -0
  19. data/lib/fluent/plugin/bare_output.rb +49 -8
  20. data/lib/fluent/plugin/buffer.rb +84 -22
  21. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  22. data/lib/fluent/plugin/filter.rb +35 -1
  23. data/lib/fluent/plugin/formatter.rb +1 -0
  24. data/lib/fluent/plugin/formatter_json.rb +9 -7
  25. data/lib/fluent/plugin/in_http.rb +21 -2
  26. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  27. data/lib/fluent/plugin/in_syslog.rb +13 -1
  28. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  29. data/lib/fluent/plugin/in_tail.rb +77 -6
  30. data/lib/fluent/plugin/input.rb +39 -1
  31. data/lib/fluent/plugin/metrics.rb +119 -0
  32. data/lib/fluent/plugin/metrics_local.rb +96 -0
  33. data/lib/fluent/plugin/multi_output.rb +43 -6
  34. data/lib/fluent/plugin/out_copy.rb +1 -1
  35. data/lib/fluent/plugin/out_forward.rb +15 -7
  36. data/lib/fluent/plugin/output.rb +77 -36
  37. data/lib/fluent/plugin/parser_json.rb +2 -3
  38. data/lib/fluent/plugin.rb +10 -1
  39. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  40. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  41. data/lib/fluent/plugin_helper/server.rb +4 -2
  42. data/lib/fluent/plugin_helper.rb +1 -0
  43. data/lib/fluent/root_agent.rb +6 -0
  44. data/lib/fluent/supervisor.rb +2 -0
  45. data/lib/fluent/system_config.rb +9 -1
  46. data/lib/fluent/test/driver/storage.rb +30 -0
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  49. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  50. data/test/command/test_plugin_generator.rb +2 -1
  51. data/test/config/test_system_config.rb +6 -0
  52. data/test/config/test_types.rb +7 -0
  53. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  54. data/test/plugin/in_tail/test_position_file.rb +48 -8
  55. data/test/plugin/test_bare_output.rb +13 -0
  56. data/test/plugin/test_buffer.rb +8 -2
  57. data/test/plugin/test_file_wrapper.rb +11 -0
  58. data/test/plugin/test_filter.rb +11 -0
  59. data/test/plugin/test_in_http.rb +40 -0
  60. data/test/plugin/test_in_monitor_agent.rb +214 -8
  61. data/test/plugin/test_in_syslog.rb +35 -0
  62. data/test/plugin/test_in_tail.rb +157 -29
  63. data/test/plugin/test_input.rb +11 -0
  64. data/test/plugin/test_metrics.rb +294 -0
  65. data/test/plugin/test_metrics_local.rb +96 -0
  66. data/test/plugin/test_multi_output.rb +25 -1
  67. data/test/plugin/test_output.rb +16 -0
  68. data/test/plugin_helper/test_event_emitter.rb +29 -0
  69. data/test/plugin_helper/test_metrics.rb +137 -0
  70. data/test/test_event_time.rb +2 -2
  71. data/test/test_oj_options.rb +55 -0
  72. data/test/test_plugin_classes.rb +102 -0
  73. data/test/test_root_agent.rb +30 -1
  74. metadata +21 -6
  75. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  76. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -22,19 +22,18 @@ module Fluent::Plugin
22
22
  UNWATCHED_POSITION = 0xffffffffffffffff
23
23
  POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
24
 
25
- def self.load(file, follow_inodes, existing_paths, logger:)
26
- pf = new(file, follow_inodes, existing_paths, logger: logger)
27
- pf.load
25
+ def self.load(file, follow_inodes, existing_targets, logger:)
26
+ pf = new(file, follow_inodes, logger: logger)
27
+ pf.load(existing_targets)
28
28
  pf
29
29
  end
30
30
 
31
- def initialize(file, follow_inodes, existing_paths, logger: nil)
31
+ def initialize(file, follow_inodes, logger: nil)
32
32
  @file = file
33
33
  @logger = logger
34
34
  @file_mutex = Mutex.new
35
35
  @map = {}
36
36
  @follow_inodes = follow_inodes
37
- @existing_paths = existing_paths
38
37
  end
39
38
 
40
39
  def [](target_info)
@@ -60,8 +59,8 @@ module Fluent::Plugin
60
59
  end
61
60
  end
62
61
 
63
- def load
64
- compact
62
+ def load(existing_targets = nil)
63
+ compact(existing_targets)
65
64
 
66
65
  map = {}
67
66
  @file_mutex.synchronize do
@@ -118,9 +117,9 @@ module Fluent::Plugin
118
117
 
119
118
  private
120
119
 
121
- def compact
120
+ def compact(existing_targets = nil)
122
121
  @file_mutex.synchronize do
123
- entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
122
+ entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
124
123
 
125
124
  @file.pos = 0
126
125
  @file.truncate(0)
@@ -128,7 +127,7 @@ module Fluent::Plugin
128
127
  end
129
128
  end
130
129
 
131
- def fetch_compacted_entries
130
+ def fetch_compacted_entries(existing_targets = nil)
132
131
  entries = {}
133
132
 
134
133
  @file.pos = 0
@@ -151,23 +150,26 @@ module Fluent::Plugin
151
150
  end
152
151
 
153
152
  if @follow_inodes
154
- entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
153
+ entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
155
154
  else
156
- entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
155
+ entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
157
156
  end
158
157
  file_pos += line.size
159
158
  end
160
159
  end
161
160
 
162
- entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
161
+ entries = remove_deleted_files_entries(entries, existing_targets)
163
162
  entries
164
163
  end
165
164
 
166
- def remove_deleted_files_entries(existent_entries, existing_paths)
167
- filtered_entries = existent_entries.select {|file_entry|
168
- existing_paths.key?(file_entry)
169
- }
170
- filtered_entries
165
+ def remove_deleted_files_entries(existent_entries, existing_targets)
166
+ if existing_targets
167
+ existent_entries.select { |path_or_ino|
168
+ existing_targets.key?(path_or_ino)
169
+ }
170
+ else
171
+ existent_entries
172
+ end
171
173
  end
172
174
  end
173
175
 
@@ -38,6 +38,7 @@ module Fluent::Plugin
38
38
  helpers :timer, :event_loop, :parser, :compat_parameters
39
39
 
40
40
  RESERVED_CHARS = ['/', '*', '%'].freeze
41
+ MetricsInfo = Struct.new(:opened, :closed, :rotated)
41
42
 
42
43
  class WatcherSetupError < StandardError
43
44
  def initialize(msg)
@@ -57,6 +58,7 @@ module Fluent::Plugin
57
58
  @pf = nil
58
59
  @ignore_list = []
59
60
  @shutdown_start_time = nil
61
+ @metrics = nil
60
62
  end
61
63
 
62
64
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -191,6 +193,10 @@ module Fluent::Plugin
191
193
  @read_bytes_limit_per_second = min_bytes
192
194
  end
193
195
  end
196
+ opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
197
+ closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
198
+ rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
199
+ @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
194
200
  end
195
201
 
196
202
  def configure_tag
@@ -375,7 +381,7 @@ module Fluent::Plugin
375
381
 
376
382
  def setup_watcher(target_info, pe)
377
383
  line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
378
- tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
384
+ tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
379
385
 
380
386
  if @enable_watch_timer
381
387
  tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -426,6 +432,7 @@ module Fluent::Plugin
426
432
 
427
433
  begin
428
434
  target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
435
+ @tails.delete(target_info)
429
436
  @tails[target_info] = tw
430
437
  tw.on_notify
431
438
  rescue Errno::ENOENT, Errno::EACCES => e
@@ -491,10 +498,17 @@ module Fluent::Plugin
491
498
  new_position_entry = @pf[target_info]
492
499
 
493
500
  if new_position_entry.read_inode == 0
501
+ # When follow_inodes is true, it's not cleaned up by refresh_watcher.
502
+ # So it should be unwatched here explicitly.
503
+ rotated_tw.unwatched = true
504
+ # Make sure to delete old key, it has a different ino while the hash key is same.
505
+ @tails.delete(rotated_target_info)
494
506
  @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
495
507
  @tails[new_target_info].on_notify
496
508
  end
497
509
  else
510
+ # Make sure to delete old key, it has a different ino while the hash key is same.
511
+ @tails.delete(rotated_target_info)
498
512
  @tails[new_target_info] = setup_watcher(new_target_info, pe)
499
513
  @tails[new_target_info].on_notify
500
514
  end
@@ -522,8 +536,25 @@ module Fluent::Plugin
522
536
  def detach_watcher_after_rotate_wait(tw, ino)
523
537
  # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
524
538
  # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
525
- timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
539
+ if @open_on_every_update
540
+ # Detach now because it's already closed, waiting it doesn't make sense.
526
541
  detach_watcher(tw, ino)
542
+ elsif @read_bytes_limit_per_second < 0
543
+ # throttling isn't enabled, just wait @rotate_wait
544
+ timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
545
+ detach_watcher(tw, ino)
546
+ end
547
+ else
548
+ # When the throttling feature is enabled, it might not reach EOF yet.
549
+ # Should ensure to read all contents before closing it, with keeping throttling.
550
+ start_time_to_wait = Fluent::Clock.now
551
+ timer = timer_execute(:in_tail_close_watcher, 1, repeat: true) do
552
+ elapsed = Fluent::Clock.now - start_time_to_wait
553
+ if tw.eof? && elapsed >= @rotate_wait
554
+ timer.detach
555
+ detach_watcher(tw, ino)
556
+ end
557
+ end
527
558
  end
528
559
  end
529
560
 
@@ -645,6 +676,19 @@ module Fluent::Plugin
645
676
  es
646
677
  end
647
678
 
679
+ def statistics
680
+ stats = super
681
+
682
+ stats = {
683
+ 'input' => stats["input"].merge({
684
+ 'opened_file_count' => @metrics.opened.get,
685
+ 'closed_file_count' => @metrics.closed.get,
686
+ 'rotated_file_count' => @metrics.rotated.get,
687
+ })
688
+ }
689
+ stats
690
+ end
691
+
648
692
  private
649
693
 
650
694
  def io_handler(watcher, path)
@@ -657,6 +701,7 @@ module Fluent::Plugin
657
701
  open_on_every_update: @open_on_every_update,
658
702
  from_encoding: @from_encoding,
659
703
  encoding: @encoding,
704
+ metrics: @metrics,
660
705
  &method(:receive_lines)
661
706
  )
662
707
  end
@@ -692,7 +737,7 @@ module Fluent::Plugin
692
737
  end
693
738
 
694
739
  class TailWatcher
695
- def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
740
+ def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
696
741
  @path = target_info.path
697
742
  @ino = target_info.ino
698
743
  @pe = pe || MemoryPositionEntry.new
@@ -704,6 +749,7 @@ module Fluent::Plugin
704
749
  @line_buffer_timer_flusher = line_buffer_timer_flusher
705
750
  @io_handler = nil
706
751
  @io_handler_build = io_handler_build
752
+ @metrics = metrics
707
753
  @watchers = []
708
754
  end
709
755
 
@@ -736,6 +782,10 @@ module Fluent::Plugin
736
782
  end
737
783
  end
738
784
 
785
+ def eof?
786
+ @io_handler.nil? || @io_handler.eof?
787
+ end
788
+
739
789
  def on_notify
740
790
  begin
741
791
  stat = Fluent::FileWrapper.stat(@path)
@@ -810,7 +860,7 @@ module Fluent::Plugin
810
860
  # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
811
861
  # don't want to swap state because we need latest read offset in pos file even after rotate_wait
812
862
  if stat
813
- target_info = TargetInfo.new(@path, stat)
863
+ target_info = TargetInfo.new(@path, stat.ino)
814
864
  @update_watcher.call(target_info, @pe)
815
865
  end
816
866
  else
@@ -826,6 +876,7 @@ module Fluent::Plugin
826
876
  @log.info "detected rotation of #{@path}"
827
877
  @io_handler = io_handler
828
878
  end
879
+ @metrics.rotated.inc
829
880
  end
830
881
  end
831
882
 
@@ -905,7 +956,7 @@ module Fluent::Plugin
905
956
 
906
957
  attr_accessor :shutdown_timeout
907
958
 
908
- def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
959
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
909
960
  @watcher = watcher
910
961
  @path = path
911
962
  @read_lines_limit = read_lines_limit
@@ -923,6 +974,8 @@ module Fluent::Plugin
923
974
  @shutdown_start_time = nil
924
975
  @shutdown_timeout = SHUTDOWN_TIMEOUT
925
976
  @shutdown_mutex = Mutex.new
977
+ @eof = false
978
+ @metrics = metrics
926
979
 
927
980
  @log.info "following tail of #{@path}"
928
981
  end
@@ -942,6 +995,7 @@ module Fluent::Plugin
942
995
  if @io && !@io.closed?
943
996
  @io.close
944
997
  @io = nil
998
+ @metrics.closed.inc
945
999
  end
946
1000
  end
947
1001
 
@@ -949,6 +1003,10 @@ module Fluent::Plugin
949
1003
  !!@io
950
1004
  end
951
1005
 
1006
+ def eof?
1007
+ @eof
1008
+ end
1009
+
952
1010
  private
953
1011
 
954
1012
  def limit_bytes_per_second_reached?
@@ -989,6 +1047,7 @@ module Fluent::Plugin
989
1047
  while true
990
1048
  @start_reading_time ||= Fluent::Clock.now
991
1049
  data = io.readpartial(BYTES_TO_READ, @iobuf)
1050
+ @eof = false
992
1051
  @number_bytes_read += data.bytesize
993
1052
  @fifo << data
994
1053
  @fifo.read_lines(@lines)
@@ -1005,6 +1064,7 @@ module Fluent::Plugin
1005
1064
  end
1006
1065
  end
1007
1066
  rescue EOFError
1067
+ @eof = true
1008
1068
  end
1009
1069
  end
1010
1070
 
@@ -1023,11 +1083,15 @@ module Fluent::Plugin
1023
1083
  def open
1024
1084
  io = Fluent::FileWrapper.open(@path)
1025
1085
  io.seek(@watcher.pe.read_pos + @fifo.bytesize)
1086
+ @metrics.opened.inc
1026
1087
  io
1027
1088
  rescue RangeError
1028
1089
  io.close if io
1029
1090
  raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
1030
- rescue Errno::ENOENT, Errno::EACCES
1091
+ rescue Errno::EACCES => e
1092
+ @log.warn "#{e}"
1093
+ nil
1094
+ rescue Errno::ENOENT
1031
1095
  nil
1032
1096
  end
1033
1097
 
@@ -1042,14 +1106,17 @@ module Fluent::Plugin
1042
1106
  else
1043
1107
  @io ||= open
1044
1108
  yield @io
1109
+ @eof = true if @io.nil?
1045
1110
  end
1046
1111
  rescue WatcherSetupError => e
1047
1112
  close
1113
+ @eof = true
1048
1114
  raise e
1049
1115
  rescue
1050
1116
  @log.error $!.to_s
1051
1117
  @log.error_backtrace
1052
1118
  close
1119
+ @eof = true
1053
1120
  end
1054
1121
  end
1055
1122
 
@@ -1069,6 +1136,10 @@ module Fluent::Plugin
1069
1136
  def opened?
1070
1137
  false
1071
1138
  end
1139
+
1140
+ def eof?
1141
+ true
1142
+ end
1072
1143
  end
1073
1144
 
1074
1145
  class RotateHandler
@@ -27,7 +27,45 @@ module Fluent
27
27
  include PluginLoggerMixin
28
28
  include PluginHelper::Mixin
29
29
 
30
- helpers_internal :event_emitter
30
+ helpers_internal :event_emitter, :metrics
31
+
32
+ def initialize
33
+ super
34
+ @emit_records_metrics = nil
35
+ @emit_size_metrics = nil
36
+ @counter_mutex = Mutex.new
37
+ @enable_size_metrics = false
38
+ end
39
+
40
+ def emit_records
41
+ @emit_records_metrics.get
42
+ end
43
+
44
+ def emit_size
45
+ @emit_size_metrics.get
46
+ end
47
+
48
+ def configure(conf)
49
+ super
50
+
51
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
52
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
53
+ @enable_size_metrics = !!system_config.enable_size_metrics
54
+ end
55
+
56
+ def statistics
57
+ stats = {
58
+ 'emit_records' => @emit_records_metrics.get,
59
+ 'emit_size' => @emit_size_metrics.get,
60
+ }
61
+
62
+ { 'input' => stats }
63
+ end
64
+
65
+ def metric_callback(es)
66
+ @emit_records_metrics.add(es.size)
67
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
68
+ end
31
69
 
32
70
  def multi_workers_ready?
33
71
  false
@@ -0,0 +1,119 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'socket'
18
+
19
+ require 'fluent/plugin/base'
20
+
21
+ require 'fluent/log'
22
+ require 'fluent/unique_id'
23
+ require 'fluent/plugin_id'
24
+
25
+ module Fluent
26
+ module Plugin
27
+ class Metrics < Base
28
+ include PluginId
29
+ include PluginLoggerMixin
30
+ include UniqueId::Mixin
31
+
32
+ DEFAULT_TYPE = 'local'
33
+
34
+ configured_in :metrics
35
+
36
+ config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
37
+ config_param :labels, :hash, default: {}
38
+
39
+ attr_reader :use_gauge_metric
40
+ attr_reader :has_methods_for_gauge, :has_methods_for_counter
41
+
42
+ def initialize
43
+ super
44
+
45
+ @has_methods_for_counter = false
46
+ @has_methods_for_gauge = false
47
+ @use_gauge_metric = false
48
+ end
49
+
50
+ def configure(conf)
51
+ super
52
+
53
+ if use_gauge_metric
54
+ @has_methods_for_gauge = has_methods_for_gauge?
55
+ else
56
+ @has_methods_for_counter = has_methods_for_counter?
57
+ end
58
+ end
59
+
60
+ # Some metrics should be counted by gauge.
61
+ # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
62
+ def use_gauge_metric=(use_gauge_metric=false)
63
+ @use_gauge_metric = use_gauge_metric
64
+ end
65
+
66
+ def create(namespace:, subsystem:,name:,help_text:,labels: {})
67
+ # This API is for cmetrics type.
68
+ end
69
+
70
+ def get
71
+ raise NotImplementedError, "Implement this method in child class"
72
+ end
73
+
74
+ def inc
75
+ raise NotImplementedError, "Implement this method in child class"
76
+ end
77
+
78
+ def dec
79
+ raise NotImplementedError, "Implement this method in child class"
80
+ end
81
+
82
+ def add(value)
83
+ raise NotImplementedError, "Implement this method in child class"
84
+ end
85
+
86
+ def sub(value)
87
+ raise NotImplementedError, "Implement this method in child class"
88
+ end
89
+
90
+ def set(value)
91
+ raise NotImplementedError, "Implement this method in child class"
92
+ end
93
+
94
+ private
95
+
96
+ def has_methods_for_counter?
97
+ implemented_methods = self.class.instance_methods(false)
98
+
99
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
100
+ [:set].all?{|e| self.class.method_defined?(e)}
101
+ true
102
+ else
103
+ raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
104
+ end
105
+ end
106
+
107
+ def has_methods_for_gauge?
108
+ implemented_methods = self.class.instance_methods(false)
109
+
110
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
111
+ [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
112
+ true
113
+ else
114
+ raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,96 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/metrics'
19
+
20
+ module Fluent
21
+ module Plugin
22
+ class LocalMetrics < Metrics
23
+ Fluent::Plugin.register_metrics('local', self)
24
+
25
+ def initialize
26
+ super
27
+ @store = 0
28
+ @monitor = Monitor.new
29
+ end
30
+
31
+ def configure(conf)
32
+ super
33
+
34
+ if use_gauge_metric
35
+ class << self
36
+ alias_method :dec, :dec_gauge
37
+ alias_method :set, :set_gauge
38
+ alias_method :sub, :sub_gauge
39
+ end
40
+ else
41
+ class << self
42
+ alias_method :set, :set_counter
43
+ end
44
+ end
45
+ end
46
+
47
+ def multi_workers_ready?
48
+ true
49
+ end
50
+
51
+ def get
52
+ @monitor.synchronize do
53
+ @store
54
+ end
55
+ end
56
+
57
+ def inc
58
+ @monitor.synchronize do
59
+ @store += 1
60
+ end
61
+ end
62
+
63
+ def dec_gauge
64
+ @monitor.synchronize do
65
+ @store -= 1
66
+ end
67
+ end
68
+
69
+ def add(value)
70
+ @monitor.synchronize do
71
+ @store += value
72
+ end
73
+ end
74
+
75
+ def sub_gauge(value)
76
+ @monitor.synchronize do
77
+ @store -= value
78
+ end
79
+ end
80
+
81
+ def set_counter(value)
82
+ return if @store > value
83
+
84
+ @monitor.synchronize do
85
+ @store = value
86
+ end
87
+ end
88
+
89
+ def set_gauge(value)
90
+ @monitor.synchronize do
91
+ @store = value
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -27,6 +27,7 @@ module Fluent
27
27
  include PluginHelper::Mixin # for event_emitter
28
28
 
29
29
  helpers :event_emitter # to get router from agent, which will be supplied to child plugins
30
+ helpers_internal :metrics
30
31
 
31
32
  config_section :store, param_name: :stores, multi: true, required: true do
32
33
  config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
46
47
 
47
48
  @counter_mutex = Mutex.new
48
49
  # TODO: well organized counters
49
- @num_errors = 0
50
- @emit_count = 0
51
- @emit_records = 0
50
+ @num_errors_metrics = nil
51
+ @emit_count_metrics = nil
52
+ @emit_records_metrics = nil
53
+ @emit_size_metrics = nil
52
54
  # @write_count = 0
53
55
  # @rollback_count = 0
56
+ @enable_size_metrics = false
57
+ end
58
+
59
+ def num_errors
60
+ @num_errors_metrics.get
61
+ end
62
+
63
+ def emit_count
64
+ @emit_count_metrics.get
65
+ end
66
+
67
+ def emit_size
68
+ @emit_size_metrics.get
69
+ end
70
+
71
+ def emit_records
72
+ @emit_records_metrics.get
73
+ end
74
+
75
+ def statistics
76
+ stats = {
77
+ 'num_errors' => @num_errors_metrics.get,
78
+ 'emit_records' => @emit_records_metrics.get,
79
+ 'emit_count' => @emit_count_metrics.get,
80
+ 'emit_size' => @emit_size_metrics.get,
81
+ }
82
+
83
+ { 'multi_output' => stats }
54
84
  end
55
85
 
56
86
  def multi_output?
@@ -60,6 +90,12 @@ module Fluent
60
90
  def configure(conf)
61
91
  super
62
92
 
93
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
94
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
95
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
96
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
97
+ @enable_size_metrics = !!system_config.enable_size_metrics
98
+
63
99
  @stores.each do |store|
64
100
  store_conf = store.corresponding_config_element
65
101
  type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
143
179
  end
144
180
 
145
181
  def emit_sync(tag, es)
146
- @counter_mutex.synchronize{ @emit_count += 1 }
182
+ @emit_count_metrics.inc
147
183
  begin
148
184
  process(tag, es)
149
- @counter_mutex.synchronize{ @emit_records += es.size }
185
+ @emit_records_metrics.add(es.size)
186
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
150
187
  rescue
151
- @counter_mutex.synchronize{ @num_errors += 1 }
188
+ @num_errors_metrics.inc
152
189
  raise
153
190
  end
154
191
  end
@@ -46,7 +46,7 @@ module Fluent::Plugin
46
46
  @ignore_errors << (store.arg.include?('ignore_error'))
47
47
  @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
48
48
  }
49
- if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
49
+ if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
50
50
  log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
51
51
  end
52
52
  end