fluentd 1.13.2-x64-mingw32 → 1.14.2-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +6 -6
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/workflows/windows-test.yaml +3 -3
- data/CHANGELOG.md +120 -0
- data/example/v0_12_filter.conf +2 -2
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/fluentd.rb +8 -0
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/event_router.rb +28 -1
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +84 -22
- data/lib/fluent/plugin/file_wrapper.rb +22 -0
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/in_http.rb +21 -2
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
- data/lib/fluent/plugin/in_tail.rb +46 -6
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +15 -7
- data/lib/fluent/plugin/output.rb +80 -38
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/server.rb +4 -2
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/supervisor.rb +4 -2
- data/lib/fluent/system_config.rb +9 -1
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_cat.rb +31 -2
- data/test/config/test_system_config.rb +6 -0
- data/test/plugin/in_tail/test_io_handler.rb +12 -4
- data/test/plugin/in_tail/test_position_file.rb +48 -8
- data/test/plugin/test_bare_output.rb +13 -0
- data/test/plugin/test_buffer.rb +8 -2
- data/test/plugin/test_file_wrapper.rb +11 -0
- data/test/plugin/test_filter.rb +11 -0
- data/test/plugin/test_in_http.rb +40 -0
- data/test/plugin/test_in_monitor_agent.rb +214 -8
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +72 -29
- data/test/plugin/test_input.rb +11 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_output.rb +16 -0
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_time_parser.rb +22 -0
- metadata +13 -4
@@ -38,6 +38,7 @@ module Fluent::Plugin
|
|
38
38
|
helpers :timer, :event_loop, :parser, :compat_parameters
|
39
39
|
|
40
40
|
RESERVED_CHARS = ['/', '*', '%'].freeze
|
41
|
+
MetricsInfo = Struct.new(:opened, :closed, :rotated)
|
41
42
|
|
42
43
|
class WatcherSetupError < StandardError
|
43
44
|
def initialize(msg)
|
@@ -57,6 +58,7 @@ module Fluent::Plugin
|
|
57
58
|
@pf = nil
|
58
59
|
@ignore_list = []
|
59
60
|
@shutdown_start_time = nil
|
61
|
+
@metrics = nil
|
60
62
|
end
|
61
63
|
|
62
64
|
desc 'The paths to read. Multiple paths can be specified, separated by comma.'
|
@@ -191,6 +193,10 @@ module Fluent::Plugin
|
|
191
193
|
@read_bytes_limit_per_second = min_bytes
|
192
194
|
end
|
193
195
|
end
|
196
|
+
opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
|
197
|
+
closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
|
198
|
+
rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
|
199
|
+
@metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
|
194
200
|
end
|
195
201
|
|
196
202
|
def configure_tag
|
@@ -375,7 +381,7 @@ module Fluent::Plugin
|
|
375
381
|
|
376
382
|
def setup_watcher(target_info, pe)
|
377
383
|
line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
|
378
|
-
tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
|
384
|
+
tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
|
379
385
|
|
380
386
|
if @enable_watch_timer
|
381
387
|
tt = TimerTrigger.new(1, log) { tw.on_notify }
|
@@ -426,6 +432,7 @@ module Fluent::Plugin
|
|
426
432
|
|
427
433
|
begin
|
428
434
|
target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
|
435
|
+
@tails.delete(target_info)
|
429
436
|
@tails[target_info] = tw
|
430
437
|
tw.on_notify
|
431
438
|
rescue Errno::ENOENT, Errno::EACCES => e
|
@@ -491,10 +498,17 @@ module Fluent::Plugin
|
|
491
498
|
new_position_entry = @pf[target_info]
|
492
499
|
|
493
500
|
if new_position_entry.read_inode == 0
|
501
|
+
# When follow_inodes is true, it's not cleaned up by refresh_watcher.
|
502
|
+
# So it should be unwatched here explicitly.
|
503
|
+
rotated_tw.unwatched = true
|
504
|
+
# Make sure to delete old key, it has a different ino while the hash key is same.
|
505
|
+
@tails.delete(rotated_target_info)
|
494
506
|
@tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
|
495
507
|
@tails[new_target_info].on_notify
|
496
508
|
end
|
497
509
|
else
|
510
|
+
# Make sure to delete old key, it has a different ino while the hash key is same.
|
511
|
+
@tails.delete(rotated_target_info)
|
498
512
|
@tails[new_target_info] = setup_watcher(new_target_info, pe)
|
499
513
|
@tails[new_target_info].on_notify
|
500
514
|
end
|
@@ -662,6 +676,19 @@ module Fluent::Plugin
|
|
662
676
|
es
|
663
677
|
end
|
664
678
|
|
679
|
+
def statistics
|
680
|
+
stats = super
|
681
|
+
|
682
|
+
stats = {
|
683
|
+
'input' => stats["input"].merge({
|
684
|
+
'opened_file_count' => @metrics.opened.get,
|
685
|
+
'closed_file_count' => @metrics.closed.get,
|
686
|
+
'rotated_file_count' => @metrics.rotated.get,
|
687
|
+
})
|
688
|
+
}
|
689
|
+
stats
|
690
|
+
end
|
691
|
+
|
665
692
|
private
|
666
693
|
|
667
694
|
def io_handler(watcher, path)
|
@@ -674,6 +701,7 @@ module Fluent::Plugin
|
|
674
701
|
open_on_every_update: @open_on_every_update,
|
675
702
|
from_encoding: @from_encoding,
|
676
703
|
encoding: @encoding,
|
704
|
+
metrics: @metrics,
|
677
705
|
&method(:receive_lines)
|
678
706
|
)
|
679
707
|
end
|
@@ -709,7 +737,7 @@ module Fluent::Plugin
|
|
709
737
|
end
|
710
738
|
|
711
739
|
class TailWatcher
|
712
|
-
def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
|
740
|
+
def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
|
713
741
|
@path = target_info.path
|
714
742
|
@ino = target_info.ino
|
715
743
|
@pe = pe || MemoryPositionEntry.new
|
@@ -721,6 +749,7 @@ module Fluent::Plugin
|
|
721
749
|
@line_buffer_timer_flusher = line_buffer_timer_flusher
|
722
750
|
@io_handler = nil
|
723
751
|
@io_handler_build = io_handler_build
|
752
|
+
@metrics = metrics
|
724
753
|
@watchers = []
|
725
754
|
end
|
726
755
|
|
@@ -754,7 +783,7 @@ module Fluent::Plugin
|
|
754
783
|
end
|
755
784
|
|
756
785
|
def eof?
|
757
|
-
@io_handler.eof?
|
786
|
+
@io_handler.nil? || @io_handler.eof?
|
758
787
|
end
|
759
788
|
|
760
789
|
def on_notify
|
@@ -831,7 +860,7 @@ module Fluent::Plugin
|
|
831
860
|
# new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
|
832
861
|
# don't want to swap state because we need latest read offset in pos file even after rotate_wait
|
833
862
|
if stat
|
834
|
-
target_info = TargetInfo.new(@path, stat)
|
863
|
+
target_info = TargetInfo.new(@path, stat.ino)
|
835
864
|
@update_watcher.call(target_info, @pe)
|
836
865
|
end
|
837
866
|
else
|
@@ -847,6 +876,7 @@ module Fluent::Plugin
|
|
847
876
|
@log.info "detected rotation of #{@path}"
|
848
877
|
@io_handler = io_handler
|
849
878
|
end
|
879
|
+
@metrics.rotated.inc
|
850
880
|
end
|
851
881
|
end
|
852
882
|
|
@@ -926,7 +956,7 @@ module Fluent::Plugin
|
|
926
956
|
|
927
957
|
attr_accessor :shutdown_timeout
|
928
958
|
|
929
|
-
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
|
959
|
+
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
|
930
960
|
@watcher = watcher
|
931
961
|
@path = path
|
932
962
|
@read_lines_limit = read_lines_limit
|
@@ -945,6 +975,7 @@ module Fluent::Plugin
|
|
945
975
|
@shutdown_timeout = SHUTDOWN_TIMEOUT
|
946
976
|
@shutdown_mutex = Mutex.new
|
947
977
|
@eof = false
|
978
|
+
@metrics = metrics
|
948
979
|
|
949
980
|
@log.info "following tail of #{@path}"
|
950
981
|
end
|
@@ -964,6 +995,7 @@ module Fluent::Plugin
|
|
964
995
|
if @io && !@io.closed?
|
965
996
|
@io.close
|
966
997
|
@io = nil
|
998
|
+
@metrics.closed.inc
|
967
999
|
end
|
968
1000
|
end
|
969
1001
|
|
@@ -1051,11 +1083,15 @@ module Fluent::Plugin
|
|
1051
1083
|
def open
|
1052
1084
|
io = Fluent::FileWrapper.open(@path)
|
1053
1085
|
io.seek(@watcher.pe.read_pos + @fifo.bytesize)
|
1086
|
+
@metrics.opened.inc
|
1054
1087
|
io
|
1055
1088
|
rescue RangeError
|
1056
1089
|
io.close if io
|
1057
1090
|
raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
|
1058
|
-
rescue Errno::
|
1091
|
+
rescue Errno::EACCES => e
|
1092
|
+
@log.warn "#{e}"
|
1093
|
+
nil
|
1094
|
+
rescue Errno::ENOENT
|
1059
1095
|
nil
|
1060
1096
|
end
|
1061
1097
|
|
@@ -1100,6 +1136,10 @@ module Fluent::Plugin
|
|
1100
1136
|
def opened?
|
1101
1137
|
false
|
1102
1138
|
end
|
1139
|
+
|
1140
|
+
def eof?
|
1141
|
+
true
|
1142
|
+
end
|
1103
1143
|
end
|
1104
1144
|
|
1105
1145
|
class RotateHandler
|
data/lib/fluent/plugin/input.rb
CHANGED
@@ -27,7 +27,45 @@ module Fluent
|
|
27
27
|
include PluginLoggerMixin
|
28
28
|
include PluginHelper::Mixin
|
29
29
|
|
30
|
-
helpers_internal :event_emitter
|
30
|
+
helpers_internal :event_emitter, :metrics
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
super
|
34
|
+
@emit_records_metrics = nil
|
35
|
+
@emit_size_metrics = nil
|
36
|
+
@counter_mutex = Mutex.new
|
37
|
+
@enable_size_metrics = false
|
38
|
+
end
|
39
|
+
|
40
|
+
def emit_records
|
41
|
+
@emit_records_metrics.get
|
42
|
+
end
|
43
|
+
|
44
|
+
def emit_size
|
45
|
+
@emit_size_metrics.get
|
46
|
+
end
|
47
|
+
|
48
|
+
def configure(conf)
|
49
|
+
super
|
50
|
+
|
51
|
+
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
|
52
|
+
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
|
53
|
+
@enable_size_metrics = !!system_config.enable_size_metrics
|
54
|
+
end
|
55
|
+
|
56
|
+
def statistics
|
57
|
+
stats = {
|
58
|
+
'emit_records' => @emit_records_metrics.get,
|
59
|
+
'emit_size' => @emit_size_metrics.get,
|
60
|
+
}
|
61
|
+
|
62
|
+
{ 'input' => stats }
|
63
|
+
end
|
64
|
+
|
65
|
+
def metric_callback(es)
|
66
|
+
@emit_records_metrics.add(es.size)
|
67
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
68
|
+
end
|
31
69
|
|
32
70
|
def multi_workers_ready?
|
33
71
|
false
|
@@ -0,0 +1,119 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'socket'
|
18
|
+
|
19
|
+
require 'fluent/plugin/base'
|
20
|
+
|
21
|
+
require 'fluent/log'
|
22
|
+
require 'fluent/unique_id'
|
23
|
+
require 'fluent/plugin_id'
|
24
|
+
|
25
|
+
module Fluent
|
26
|
+
module Plugin
|
27
|
+
class Metrics < Base
|
28
|
+
include PluginId
|
29
|
+
include PluginLoggerMixin
|
30
|
+
include UniqueId::Mixin
|
31
|
+
|
32
|
+
DEFAULT_TYPE = 'local'
|
33
|
+
|
34
|
+
configured_in :metrics
|
35
|
+
|
36
|
+
config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
|
37
|
+
config_param :labels, :hash, default: {}
|
38
|
+
|
39
|
+
attr_reader :use_gauge_metric
|
40
|
+
attr_reader :has_methods_for_gauge, :has_methods_for_counter
|
41
|
+
|
42
|
+
def initialize
|
43
|
+
super
|
44
|
+
|
45
|
+
@has_methods_for_counter = false
|
46
|
+
@has_methods_for_gauge = false
|
47
|
+
@use_gauge_metric = false
|
48
|
+
end
|
49
|
+
|
50
|
+
def configure(conf)
|
51
|
+
super
|
52
|
+
|
53
|
+
if use_gauge_metric
|
54
|
+
@has_methods_for_gauge = has_methods_for_gauge?
|
55
|
+
else
|
56
|
+
@has_methods_for_counter = has_methods_for_counter?
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Some metrics should be counted by gauge.
|
61
|
+
# ref: https://prometheus.io/docs/concepts/metric_types/#gauge
|
62
|
+
def use_gauge_metric=(use_gauge_metric=false)
|
63
|
+
@use_gauge_metric = use_gauge_metric
|
64
|
+
end
|
65
|
+
|
66
|
+
def create(namespace:, subsystem:,name:,help_text:,labels: {})
|
67
|
+
# This API is for cmetrics type.
|
68
|
+
end
|
69
|
+
|
70
|
+
def get
|
71
|
+
raise NotImplementedError, "Implement this method in child class"
|
72
|
+
end
|
73
|
+
|
74
|
+
def inc
|
75
|
+
raise NotImplementedError, "Implement this method in child class"
|
76
|
+
end
|
77
|
+
|
78
|
+
def dec
|
79
|
+
raise NotImplementedError, "Implement this method in child class"
|
80
|
+
end
|
81
|
+
|
82
|
+
def add(value)
|
83
|
+
raise NotImplementedError, "Implement this method in child class"
|
84
|
+
end
|
85
|
+
|
86
|
+
def sub(value)
|
87
|
+
raise NotImplementedError, "Implement this method in child class"
|
88
|
+
end
|
89
|
+
|
90
|
+
def set(value)
|
91
|
+
raise NotImplementedError, "Implement this method in child class"
|
92
|
+
end
|
93
|
+
|
94
|
+
private
|
95
|
+
|
96
|
+
def has_methods_for_counter?
|
97
|
+
implemented_methods = self.class.instance_methods(false)
|
98
|
+
|
99
|
+
if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
|
100
|
+
[:set].all?{|e| self.class.method_defined?(e)}
|
101
|
+
true
|
102
|
+
else
|
103
|
+
raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def has_methods_for_gauge?
|
108
|
+
implemented_methods = self.class.instance_methods(false)
|
109
|
+
|
110
|
+
if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
|
111
|
+
[:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
|
112
|
+
true
|
113
|
+
else
|
114
|
+
raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin'
|
18
|
+
require 'fluent/plugin/metrics'
|
19
|
+
|
20
|
+
module Fluent
|
21
|
+
module Plugin
|
22
|
+
class LocalMetrics < Metrics
|
23
|
+
Fluent::Plugin.register_metrics('local', self)
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
super
|
27
|
+
@store = 0
|
28
|
+
@monitor = Monitor.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def configure(conf)
|
32
|
+
super
|
33
|
+
|
34
|
+
if use_gauge_metric
|
35
|
+
class << self
|
36
|
+
alias_method :dec, :dec_gauge
|
37
|
+
alias_method :set, :set_gauge
|
38
|
+
alias_method :sub, :sub_gauge
|
39
|
+
end
|
40
|
+
else
|
41
|
+
class << self
|
42
|
+
alias_method :set, :set_counter
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def multi_workers_ready?
|
48
|
+
true
|
49
|
+
end
|
50
|
+
|
51
|
+
def get
|
52
|
+
@monitor.synchronize do
|
53
|
+
@store
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def inc
|
58
|
+
@monitor.synchronize do
|
59
|
+
@store += 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def dec_gauge
|
64
|
+
@monitor.synchronize do
|
65
|
+
@store -= 1
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def add(value)
|
70
|
+
@monitor.synchronize do
|
71
|
+
@store += value
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def sub_gauge(value)
|
76
|
+
@monitor.synchronize do
|
77
|
+
@store -= value
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def set_counter(value)
|
82
|
+
return if @store > value
|
83
|
+
|
84
|
+
@monitor.synchronize do
|
85
|
+
@store = value
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_gauge(value)
|
90
|
+
@monitor.synchronize do
|
91
|
+
@store = value
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -27,6 +27,7 @@ module Fluent
|
|
27
27
|
include PluginHelper::Mixin # for event_emitter
|
28
28
|
|
29
29
|
helpers :event_emitter # to get router from agent, which will be supplied to child plugins
|
30
|
+
helpers_internal :metrics
|
30
31
|
|
31
32
|
config_section :store, param_name: :stores, multi: true, required: true do
|
32
33
|
config_argument :arg, :string, default: ''
|
@@ -46,11 +47,40 @@ module Fluent
|
|
46
47
|
|
47
48
|
@counter_mutex = Mutex.new
|
48
49
|
# TODO: well organized counters
|
49
|
-
@
|
50
|
-
@
|
51
|
-
@
|
50
|
+
@num_errors_metrics = nil
|
51
|
+
@emit_count_metrics = nil
|
52
|
+
@emit_records_metrics = nil
|
53
|
+
@emit_size_metrics = nil
|
52
54
|
# @write_count = 0
|
53
55
|
# @rollback_count = 0
|
56
|
+
@enable_size_metrics = false
|
57
|
+
end
|
58
|
+
|
59
|
+
def num_errors
|
60
|
+
@num_errors_metrics.get
|
61
|
+
end
|
62
|
+
|
63
|
+
def emit_count
|
64
|
+
@emit_count_metrics.get
|
65
|
+
end
|
66
|
+
|
67
|
+
def emit_size
|
68
|
+
@emit_size_metrics.get
|
69
|
+
end
|
70
|
+
|
71
|
+
def emit_records
|
72
|
+
@emit_records_metrics.get
|
73
|
+
end
|
74
|
+
|
75
|
+
def statistics
|
76
|
+
stats = {
|
77
|
+
'num_errors' => @num_errors_metrics.get,
|
78
|
+
'emit_records' => @emit_records_metrics.get,
|
79
|
+
'emit_count' => @emit_count_metrics.get,
|
80
|
+
'emit_size' => @emit_size_metrics.get,
|
81
|
+
}
|
82
|
+
|
83
|
+
{ 'multi_output' => stats }
|
54
84
|
end
|
55
85
|
|
56
86
|
def multi_output?
|
@@ -60,6 +90,12 @@ module Fluent
|
|
60
90
|
def configure(conf)
|
61
91
|
super
|
62
92
|
|
93
|
+
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
|
94
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
|
95
|
+
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
|
96
|
+
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
|
97
|
+
@enable_size_metrics = !!system_config.enable_size_metrics
|
98
|
+
|
63
99
|
@stores.each do |store|
|
64
100
|
store_conf = store.corresponding_config_element
|
65
101
|
type = store_conf['@type']
|
@@ -143,12 +179,13 @@ module Fluent
|
|
143
179
|
end
|
144
180
|
|
145
181
|
def emit_sync(tag, es)
|
146
|
-
@
|
182
|
+
@emit_count_metrics.inc
|
147
183
|
begin
|
148
184
|
process(tag, es)
|
149
|
-
@
|
185
|
+
@emit_records_metrics.add(es.size)
|
186
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
150
187
|
rescue
|
151
|
-
@
|
188
|
+
@num_errors_metrics.inc
|
152
189
|
raise
|
153
190
|
end
|
154
191
|
end
|
@@ -46,7 +46,7 @@ module Fluent::Plugin
|
|
46
46
|
@ignore_errors << (store.arg.include?('ignore_error'))
|
47
47
|
@ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
|
48
48
|
}
|
49
|
-
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) &&
|
49
|
+
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
|
50
50
|
log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
|
51
51
|
end
|
52
52
|
end
|
@@ -167,6 +167,8 @@ module Fluent::Plugin
|
|
167
167
|
@usock = nil
|
168
168
|
@keep_alive_watcher_interval = 5 # TODO
|
169
169
|
@suspend_flush = false
|
170
|
+
@healthy_nodes_count_metrics = nil
|
171
|
+
@registered_nodes_count_metrics = nil
|
170
172
|
end
|
171
173
|
|
172
174
|
def configure(conf)
|
@@ -265,6 +267,9 @@ module Fluent::Plugin
|
|
265
267
|
end
|
266
268
|
|
267
269
|
raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
|
270
|
+
@healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
|
271
|
+
@registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
|
272
|
+
|
268
273
|
end
|
269
274
|
|
270
275
|
def multi_workers_ready?
|
@@ -418,18 +423,21 @@ module Fluent::Plugin
|
|
418
423
|
def statistics
|
419
424
|
stats = super
|
420
425
|
services = service_discovery_services
|
421
|
-
|
422
|
-
|
426
|
+
@healthy_nodes_count_metrics.set(0)
|
427
|
+
@registered_nodes_count_metrics.set(services.size)
|
423
428
|
services.each do |s|
|
424
429
|
if s.available?
|
425
|
-
|
430
|
+
@healthy_nodes_count_metrics.inc
|
426
431
|
end
|
427
432
|
end
|
428
433
|
|
429
|
-
stats
|
430
|
-
'
|
431
|
-
|
432
|
-
|
434
|
+
stats = {
|
435
|
+
'output' => stats["output"].merge({
|
436
|
+
'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
|
437
|
+
'registered_nodes_count' => @registered_nodes_count_metrics.get,
|
438
|
+
})
|
439
|
+
}
|
440
|
+
stats
|
433
441
|
end
|
434
442
|
|
435
443
|
# MessagePack FixArray length is 3
|