fluentd 1.13.0-x64-mingw32 → 1.14.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/linux-test.yaml +1 -1
  5. data/.github/workflows/windows-test.yaml +4 -4
  6. data/.gitlab-ci.yml +0 -22
  7. data/CHANGELOG.md +131 -0
  8. data/README.md +2 -2
  9. data/example/v0_12_filter.conf +2 -2
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/command/fluentd.rb +8 -0
  12. data/lib/fluent/command/plugin_generator.rb +15 -5
  13. data/lib/fluent/compat/output.rb +9 -6
  14. data/lib/fluent/config/section.rb +5 -0
  15. data/lib/fluent/config/types.rb +15 -0
  16. data/lib/fluent/config/v1_parser.rb +3 -2
  17. data/lib/fluent/config.rb +1 -1
  18. data/lib/fluent/env.rb +2 -1
  19. data/lib/fluent/event_router.rb +28 -1
  20. data/lib/fluent/oj_options.rb +62 -0
  21. data/lib/fluent/plugin/bare_output.rb +49 -8
  22. data/lib/fluent/plugin/buffer.rb +84 -22
  23. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  24. data/lib/fluent/plugin/filter.rb +35 -1
  25. data/lib/fluent/plugin/formatter.rb +1 -0
  26. data/lib/fluent/plugin/formatter_json.rb +9 -7
  27. data/lib/fluent/plugin/in_http.rb +21 -2
  28. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  29. data/lib/fluent/plugin/in_syslog.rb +13 -1
  30. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  31. data/lib/fluent/plugin/in_tail.rb +45 -4
  32. data/lib/fluent/plugin/input.rb +39 -1
  33. data/lib/fluent/plugin/metrics.rb +119 -0
  34. data/lib/fluent/plugin/metrics_local.rb +96 -0
  35. data/lib/fluent/plugin/multi_output.rb +43 -6
  36. data/lib/fluent/plugin/out_forward.rb +1 -3
  37. data/lib/fluent/plugin/output.rb +74 -33
  38. data/lib/fluent/plugin/parser_json.rb +2 -3
  39. data/lib/fluent/plugin/service_discovery.rb +0 -15
  40. data/lib/fluent/plugin.rb +10 -1
  41. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  42. data/lib/fluent/plugin_helper/http_server/router.rb +1 -1
  43. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  44. data/lib/fluent/plugin_helper/server.rb +4 -2
  45. data/lib/fluent/plugin_helper.rb +1 -0
  46. data/lib/fluent/root_agent.rb +6 -0
  47. data/lib/fluent/supervisor.rb +2 -0
  48. data/lib/fluent/system_config.rb +9 -1
  49. data/lib/fluent/test/driver/storage.rb +30 -0
  50. data/lib/fluent/version.rb +1 -1
  51. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  52. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  53. data/test/command/test_cat.rb +11 -8
  54. data/test/command/test_plugin_generator.rb +2 -1
  55. data/test/config/test_section.rb +9 -0
  56. data/test/config/test_system_config.rb +6 -0
  57. data/test/config/test_types.rb +7 -0
  58. data/test/plugin/in_tail/test_position_file.rb +48 -8
  59. data/test/plugin/test_bare_output.rb +13 -0
  60. data/test/plugin/test_buffer.rb +8 -2
  61. data/test/plugin/test_file_wrapper.rb +11 -0
  62. data/test/plugin/test_filter.rb +11 -0
  63. data/test/plugin/test_in_forward.rb +59 -83
  64. data/test/plugin/test_in_http.rb +86 -43
  65. data/test/plugin/test_in_monitor_agent.rb +214 -8
  66. data/test/plugin/test_in_syslog.rb +101 -56
  67. data/test/plugin/test_in_tail.rb +149 -48
  68. data/test/plugin/test_in_tcp.rb +45 -32
  69. data/test/plugin/test_in_udp.rb +47 -33
  70. data/test/plugin/test_input.rb +11 -0
  71. data/test/plugin/test_metrics.rb +294 -0
  72. data/test/plugin/test_metrics_local.rb +96 -0
  73. data/test/plugin/test_multi_output.rb +25 -1
  74. data/test/plugin/test_out_forward.rb +103 -89
  75. data/test/plugin/test_out_stream.rb +18 -8
  76. data/test/plugin/test_output.rb +16 -0
  77. data/test/plugin_helper/http_server/test_route.rb +1 -1
  78. data/test/plugin_helper/test_child_process.rb +1 -1
  79. data/test/plugin_helper/test_event_emitter.rb +29 -0
  80. data/test/plugin_helper/test_http_server_helper.rb +33 -26
  81. data/test/plugin_helper/test_metrics.rb +137 -0
  82. data/test/plugin_helper/test_server.rb +137 -138
  83. data/test/plugin_helper/test_socket.rb +16 -9
  84. data/test/test_event_time.rb +2 -2
  85. data/test/test_oj_options.rb +55 -0
  86. data/test/test_plugin_classes.rb +102 -0
  87. data/test/test_root_agent.rb +30 -1
  88. metadata +22 -7
  89. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  90. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -27,7 +27,45 @@ module Fluent
27
27
  include PluginLoggerMixin
28
28
  include PluginHelper::Mixin
29
29
 
30
- helpers_internal :event_emitter
30
+ helpers_internal :event_emitter, :metrics
31
+
32
+ def initialize
33
+ super
34
+ @emit_records_metrics = nil
35
+ @emit_size_metrics = nil
36
+ @counter_mutex = Mutex.new
37
+ @enable_size_metrics = false
38
+ end
39
+
40
+ def emit_records
41
+ @emit_records_metrics.get
42
+ end
43
+
44
+ def emit_size
45
+ @emit_size_metrics.get
46
+ end
47
+
48
+ def configure(conf)
49
+ super
50
+
51
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
52
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
53
+ @enable_size_metrics = !!system_config.enable_size_metrics
54
+ end
55
+
56
+ def statistics
57
+ stats = {
58
+ 'emit_records' => @emit_records_metrics.get,
59
+ 'emit_size' => @emit_size_metrics.get,
60
+ }
61
+
62
+ { 'input' => stats }
63
+ end
64
+
65
+ def metric_callback(es)
66
+ @emit_records_metrics.add(es.size)
67
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
68
+ end
31
69
 
32
70
  def multi_workers_ready?
33
71
  false
@@ -0,0 +1,119 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'socket'
18
+
19
+ require 'fluent/plugin/base'
20
+
21
+ require 'fluent/log'
22
+ require 'fluent/unique_id'
23
+ require 'fluent/plugin_id'
24
+
25
+ module Fluent
26
+ module Plugin
27
+ class Metrics < Base
28
+ include PluginId
29
+ include PluginLoggerMixin
30
+ include UniqueId::Mixin
31
+
32
+ DEFAULT_TYPE = 'local'
33
+
34
+ configured_in :metrics
35
+
36
+ config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
37
+ config_param :labels, :hash, default: {}
38
+
39
+ attr_reader :use_gauge_metric
40
+ attr_reader :has_methods_for_gauge, :has_methods_for_counter
41
+
42
+ def initialize
43
+ super
44
+
45
+ @has_methods_for_counter = false
46
+ @has_methods_for_gauge = false
47
+ @use_gauge_metric = false
48
+ end
49
+
50
+ def configure(conf)
51
+ super
52
+
53
+ if use_gauge_metric
54
+ @has_methods_for_gauge = has_methods_for_gauge?
55
+ else
56
+ @has_methods_for_counter = has_methods_for_counter?
57
+ end
58
+ end
59
+
60
+ # Some metrics should be counted by gauge.
61
+ # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
62
+ def use_gauge_metric=(use_gauge_metric=false)
63
+ @use_gauge_metric = use_gauge_metric
64
+ end
65
+
66
+ def create(namespace:, subsystem:,name:,help_text:,labels: {})
67
+ # This API is for cmetrics type.
68
+ end
69
+
70
+ def get
71
+ raise NotImplementedError, "Implement this method in child class"
72
+ end
73
+
74
+ def inc
75
+ raise NotImplementedError, "Implement this method in child class"
76
+ end
77
+
78
+ def dec
79
+ raise NotImplementedError, "Implement this method in child class"
80
+ end
81
+
82
+ def add(value)
83
+ raise NotImplementedError, "Implement this method in child class"
84
+ end
85
+
86
+ def sub(value)
87
+ raise NotImplementedError, "Implement this method in child class"
88
+ end
89
+
90
+ def set(value)
91
+ raise NotImplementedError, "Implement this method in child class"
92
+ end
93
+
94
+ private
95
+
96
+ def has_methods_for_counter?
97
+ implemented_methods = self.class.instance_methods(false)
98
+
99
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
100
+ [:set].all?{|e| self.class.method_defined?(e)}
101
+ true
102
+ else
103
+ raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
104
+ end
105
+ end
106
+
107
+ def has_methods_for_gauge?
108
+ implemented_methods = self.class.instance_methods(false)
109
+
110
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
111
+ [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
112
+ true
113
+ else
114
+ raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,96 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/metrics'
19
+
20
+ module Fluent
21
+ module Plugin
22
+ class LocalMetrics < Metrics
23
+ Fluent::Plugin.register_metrics('local', self)
24
+
25
+ def initialize
26
+ super
27
+ @store = 0
28
+ @monitor = Monitor.new
29
+ end
30
+
31
+ def configure(conf)
32
+ super
33
+
34
+ if use_gauge_metric
35
+ class << self
36
+ alias_method :dec, :dec_gauge
37
+ alias_method :set, :set_gauge
38
+ alias_method :sub, :sub_gauge
39
+ end
40
+ else
41
+ class << self
42
+ alias_method :set, :set_counter
43
+ end
44
+ end
45
+ end
46
+
47
+ def multi_workers_ready?
48
+ true
49
+ end
50
+
51
+ def get
52
+ @monitor.synchronize do
53
+ @store
54
+ end
55
+ end
56
+
57
+ def inc
58
+ @monitor.synchronize do
59
+ @store += 1
60
+ end
61
+ end
62
+
63
+ def dec_gauge
64
+ @monitor.synchronize do
65
+ @store -= 1
66
+ end
67
+ end
68
+
69
+ def add(value)
70
+ @monitor.synchronize do
71
+ @store += value
72
+ end
73
+ end
74
+
75
+ def sub_gauge(value)
76
+ @monitor.synchronize do
77
+ @store -= value
78
+ end
79
+ end
80
+
81
+ def set_counter(value)
82
+ return if @store > value
83
+
84
+ @monitor.synchronize do
85
+ @store = value
86
+ end
87
+ end
88
+
89
+ def set_gauge(value)
90
+ @monitor.synchronize do
91
+ @store = value
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -27,6 +27,7 @@ module Fluent
27
27
  include PluginHelper::Mixin # for event_emitter
28
28
 
29
29
  helpers :event_emitter # to get router from agent, which will be supplied to child plugins
30
+ helpers_internal :metrics
30
31
 
31
32
  config_section :store, param_name: :stores, multi: true, required: true do
32
33
  config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
46
47
 
47
48
  @counter_mutex = Mutex.new
48
49
  # TODO: well organized counters
49
- @num_errors = 0
50
- @emit_count = 0
51
- @emit_records = 0
50
+ @num_errors_metrics = nil
51
+ @emit_count_metrics = nil
52
+ @emit_records_metrics = nil
53
+ @emit_size_metrics = nil
52
54
  # @write_count = 0
53
55
  # @rollback_count = 0
56
+ @enable_size_metrics = false
57
+ end
58
+
59
+ def num_errors
60
+ @num_errors_metrics.get
61
+ end
62
+
63
+ def emit_count
64
+ @emit_count_metrics.get
65
+ end
66
+
67
+ def emit_size
68
+ @emit_size_metrics.get
69
+ end
70
+
71
+ def emit_records
72
+ @emit_records_metrics.get
73
+ end
74
+
75
+ def statistics
76
+ stats = {
77
+ 'num_errors' => @num_errors_metrics.get,
78
+ 'emit_records' => @emit_records_metrics.get,
79
+ 'emit_count' => @emit_count_metrics.get,
80
+ 'emit_size' => @emit_size_metrics.get,
81
+ }
82
+
83
+ { 'multi_output' => stats }
54
84
  end
55
85
 
56
86
  def multi_output?
@@ -60,6 +90,12 @@ module Fluent
60
90
  def configure(conf)
61
91
  super
62
92
 
93
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
94
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
95
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
96
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
97
+ @enable_size_metrics = !!system_config.enable_size_metrics
98
+
63
99
  @stores.each do |store|
64
100
  store_conf = store.corresponding_config_element
65
101
  type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
143
179
  end
144
180
 
145
181
  def emit_sync(tag, es)
146
- @counter_mutex.synchronize{ @emit_count += 1 }
182
+ @emit_count_metrics.inc
147
183
  begin
148
184
  process(tag, es)
149
- @counter_mutex.synchronize{ @emit_records += es.size }
185
+ @emit_records_metrics.add(es.size)
186
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
150
187
  rescue
151
- @counter_mutex.synchronize{ @num_errors += 1 }
188
+ @num_errors_metrics.inc
152
189
  raise
153
190
  end
154
191
  end
@@ -565,8 +565,6 @@ module Fluent::Plugin
565
565
  username: server.username || '',
566
566
  )
567
567
 
568
- @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
569
-
570
568
  @resolved_host = nil
571
569
  @resolved_time = 0
572
570
  @resolved_once = false
@@ -613,7 +611,7 @@ module Fluent::Plugin
613
611
  sleep @sender.read_interval
614
612
  next
615
613
  end
616
- @unpacker.feed_each(buf) do |data|
614
+ Fluent::MessagePackFactory.msgpack_unpacker.feed_each(buf) do |data|
617
615
  if @handshake.invoke(sock, ri, data) == :established
618
616
  @log.debug "connection established", host: @host, port: @port
619
617
  end
@@ -37,7 +37,7 @@ module Fluent
37
37
  include PluginHelper::Mixin
38
38
  include UniqueId::Mixin
39
39
 
40
- helpers_internal :thread, :retry_state
40
+ helpers_internal :thread, :retry_state, :metrics
41
41
 
42
42
  CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
43
43
  CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +164,6 @@ module Fluent
164
164
  end
165
165
 
166
166
  attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
167
- attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
168
167
 
169
168
  # for tests
170
169
  attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +171,30 @@ module Fluent
172
171
  # output_enqueue_thread_waiting: for test of output.rb itself
173
172
  attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
174
173
 
174
+ def num_errors
175
+ @num_errors_metrics.get
176
+ end
177
+
178
+ def emit_count
179
+ @emit_count_metrics.get
180
+ end
181
+
182
+ def emit_size
183
+ @emit_size_metrics.get
184
+ end
185
+
186
+ def emit_records
187
+ @emit_records_metrics.get
188
+ end
189
+
190
+ def write_count
191
+ @write_count_metrics.get
192
+ end
193
+
194
+ def rollback_count
195
+ @rollback_count_metrics.get
196
+ end
197
+
175
198
  def initialize
176
199
  super
177
200
  @counter_mutex = Mutex.new
@@ -181,13 +204,15 @@ module Fluent
181
204
  @primary_instance = nil
182
205
 
183
206
  # TODO: well organized counters
184
- @num_errors = 0
185
- @emit_count = 0
186
- @emit_records = 0
187
- @write_count = 0
188
- @rollback_count = 0
189
- @flush_time_count = 0
190
- @slow_flush_count = 0
207
+ @num_errors_metrics = nil
208
+ @emit_count_metrics = nil
209
+ @emit_records_metrics = nil
210
+ @emit_size_metrics = nil
211
+ @write_count_metrics = nil
212
+ @rollback_count_metrics = nil
213
+ @flush_time_count_metrics = nil
214
+ @slow_flush_count_metrics = nil
215
+ @enable_size_metrics = false
191
216
 
192
217
  # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
193
218
  if implement?(:synchronous)
@@ -246,6 +271,15 @@ module Fluent
246
271
 
247
272
  super
248
273
 
274
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
275
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
276
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
277
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
278
+ @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
279
+ @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
280
+ @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
281
+ @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
282
+
249
283
  if has_buffer_section
250
284
  unless implement?(:buffered) || implement?(:delayed_commit)
251
285
  raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +305,8 @@ module Fluent
271
305
  @buffering = true
272
306
  end
273
307
  end
308
+ # Enable to update record size metrics or not
309
+ @enable_size_metrics = !!system_config.enable_size_metrics
274
310
 
275
311
  if @as_secondary
276
312
  if !@buffering && !@buffering.nil?
@@ -797,18 +833,19 @@ module Fluent
797
833
  end
798
834
 
799
835
  def emit_sync(tag, es)
800
- @counter_mutex.synchronize{ @emit_count += 1 }
836
+ @emit_count_metrics.inc
801
837
  begin
802
838
  process(tag, es)
803
- @counter_mutex.synchronize{ @emit_records += es.size }
839
+ @emit_records_metrics.add(es.size)
840
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
804
841
  rescue
805
- @counter_mutex.synchronize{ @num_errors += 1 }
842
+ @num_errors_metrics.inc
806
843
  raise
807
844
  end
808
845
  end
809
846
 
810
847
  def emit_buffered(tag, es)
811
- @counter_mutex.synchronize{ @emit_count += 1 }
848
+ @emit_count_metrics.inc
812
849
  begin
813
850
  execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
814
851
  if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +853,7 @@ module Fluent
816
853
  end
817
854
  rescue
818
855
  # TODO: separate number of errors into emit errors and write/flush errors
819
- @counter_mutex.synchronize{ @num_errors += 1 }
856
+ @num_errors_metrics.inc
820
857
  raise
821
858
  end
822
859
  end
@@ -966,7 +1003,8 @@ module Fluent
966
1003
  write_guard do
967
1004
  @buffer.write(meta_and_data, enqueue: enqueue)
968
1005
  end
969
- @counter_mutex.synchronize{ @emit_records += records }
1006
+ @emit_records_metrics.add(es.size)
1007
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
970
1008
  true
971
1009
  end
972
1010
 
@@ -983,7 +1021,8 @@ module Fluent
983
1021
  write_guard do
984
1022
  @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
985
1023
  end
986
- @counter_mutex.synchronize{ @emit_records += records }
1024
+ @emit_records_metrics.add(es.size)
1025
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
987
1026
  true
988
1027
  end
989
1028
 
@@ -1008,7 +1047,8 @@ module Fluent
1008
1047
  write_guard do
1009
1048
  @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
1010
1049
  end
1011
- @counter_mutex.synchronize{ @emit_records += records }
1050
+ @emit_records_metrics.add(es.size)
1051
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1012
1052
  true
1013
1053
  end
1014
1054
 
@@ -1046,7 +1086,7 @@ module Fluent
1046
1086
  # false if chunk was already flushed and couldn't be rollbacked unexpectedly
1047
1087
  # in many cases, false can be just ignored
1048
1088
  if @buffer.takeback_chunk(chunk_id)
1049
- @counter_mutex.synchronize{ @rollback_count += 1 }
1089
+ @rollback_count_metrics.inc
1050
1090
  if update_retry
1051
1091
  primary = @as_secondary ? @primary_instance : self
1052
1092
  primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1102,7 @@ module Fluent
1062
1102
  while @dequeued_chunks.first && @dequeued_chunks.first.expired?
1063
1103
  info = @dequeued_chunks.shift
1064
1104
  if @buffer.takeback_chunk(info.chunk_id)
1065
- @counter_mutex.synchronize{ @rollback_count += 1 }
1105
+ @rollback_count_metrics.inc
1066
1106
  log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
1067
1107
  primary = @as_secondary ? @primary_instance : self
1068
1108
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1117,7 @@ module Fluent
1077
1117
  until @dequeued_chunks.empty?
1078
1118
  info = @dequeued_chunks.shift
1079
1119
  if @buffer.takeback_chunk(info.chunk_id)
1080
- @counter_mutex.synchronize{ @rollback_count += 1 }
1120
+ @rollback_count_metrics.inc
1081
1121
  log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
1082
1122
  primary = @as_secondary ? @primary_instance : self
1083
1123
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1160,7 @@ module Fluent
1120
1160
 
1121
1161
  if output.delayed_commit
1122
1162
  log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
1123
- @counter_mutex.synchronize{ @write_count += 1 }
1163
+ @write_count_metrics.inc
1124
1164
  @dequeued_chunks_mutex.synchronize do
1125
1165
  # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
1126
1166
  @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1172,7 @@ module Fluent
1132
1172
  chunk_id = chunk.unique_id
1133
1173
  dump_chunk_id = dump_unique_id_hex(chunk_id)
1134
1174
  log.trace "adding write count", instance: self.object_id
1135
- @counter_mutex.synchronize{ @write_count += 1 }
1175
+ @write_count_metrics.inc
1136
1176
  log.trace "executing sync write", chunk: dump_chunk_id
1137
1177
 
1138
1178
  output.write(chunk)
@@ -1188,7 +1228,7 @@ module Fluent
1188
1228
  end
1189
1229
 
1190
1230
  if @buffer.takeback_chunk(chunk.unique_id)
1191
- @counter_mutex.synchronize { @rollback_count += 1 }
1231
+ @rollback_count_metrics.inc
1192
1232
  end
1193
1233
 
1194
1234
  update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1219,9 +1259,9 @@ module Fluent
1219
1259
  def check_slow_flush(start)
1220
1260
  elapsed_time = Fluent::Clock.now - start
1221
1261
  elapsed_millsec = (elapsed_time * 1000).to_i
1222
- @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
1262
+ @flush_time_count_metrics.add(elapsed_millsec)
1223
1263
  if elapsed_time > @slow_flush_log_threshold
1224
- @counter_mutex.synchronize { @slow_flush_count += 1 }
1264
+ @slow_flush_count_metrics.inc
1225
1265
  log.warn "buffer flush took longer time than slow_flush_log_threshold:",
1226
1266
  elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
1227
1267
  end
@@ -1229,7 +1269,7 @@ module Fluent
1229
1269
 
1230
1270
  def update_retry_state(chunk_id, using_secondary, error = nil)
1231
1271
  @retry_mutex.synchronize do
1232
- @counter_mutex.synchronize{ @num_errors += 1 }
1272
+ @num_errors_metrics.inc
1233
1273
  chunk_id_hex = dump_unique_id_hex(chunk_id)
1234
1274
 
1235
1275
  unless @retry
@@ -1490,15 +1530,16 @@ module Fluent
1490
1530
 
1491
1531
  def statistics
1492
1532
  stats = {
1493
- 'emit_records' => @emit_records,
1533
+ 'emit_records' => @emit_records_metrics.get,
1534
+ 'emit_size' => @emit_size_metrics.get,
1494
1535
  # Respect original name
1495
1536
  # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
1496
- 'retry_count' => @num_errors,
1497
- 'emit_count' => @emit_count,
1498
- 'write_count' => @write_count,
1499
- 'rollback_count' => @rollback_count,
1500
- 'slow_flush_count' => @slow_flush_count,
1501
- 'flush_time_count' => @flush_time_count,
1537
+ 'retry_count' => @num_errors_metrics.get,
1538
+ 'emit_count' => @emit_count_metrics.get,
1539
+ 'write_count' => @write_count_metrics.get,
1540
+ 'rollback_count' => @rollback_count_metrics.get,
1541
+ 'slow_flush_count' => @slow_flush_count_metrics.get,
1542
+ 'flush_time_count' => @flush_time_count_metrics.get,
1502
1543
  }
1503
1544
 
1504
1545
  if @buffer && @buffer.respond_to?(:statistics)