fluentd 1.13.1-x86-mingw32 → 1.14.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +136 -0
  6. data/README.md +2 -2
  7. data/example/v0_12_filter.conf +2 -2
  8. data/fluentd.gemspec +1 -1
  9. data/lib/fluent/command/fluentd.rb +8 -0
  10. data/lib/fluent/command/plugin_generator.rb +15 -5
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/types.rb +15 -0
  14. data/lib/fluent/config/v1_parser.rb +4 -3
  15. data/lib/fluent/config.rb +1 -1
  16. data/lib/fluent/env.rb +2 -1
  17. data/lib/fluent/event_router.rb +28 -1
  18. data/lib/fluent/oj_options.rb +62 -0
  19. data/lib/fluent/plugin/bare_output.rb +49 -8
  20. data/lib/fluent/plugin/buffer.rb +84 -22
  21. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  22. data/lib/fluent/plugin/filter.rb +35 -1
  23. data/lib/fluent/plugin/formatter.rb +1 -0
  24. data/lib/fluent/plugin/formatter_json.rb +9 -7
  25. data/lib/fluent/plugin/in_http.rb +21 -2
  26. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  27. data/lib/fluent/plugin/in_syslog.rb +13 -1
  28. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  29. data/lib/fluent/plugin/in_tail.rb +77 -6
  30. data/lib/fluent/plugin/input.rb +39 -1
  31. data/lib/fluent/plugin/metrics.rb +119 -0
  32. data/lib/fluent/plugin/metrics_local.rb +96 -0
  33. data/lib/fluent/plugin/multi_output.rb +43 -6
  34. data/lib/fluent/plugin/out_copy.rb +1 -1
  35. data/lib/fluent/plugin/out_forward.rb +15 -7
  36. data/lib/fluent/plugin/output.rb +77 -36
  37. data/lib/fluent/plugin/parser_json.rb +2 -3
  38. data/lib/fluent/plugin.rb +10 -1
  39. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  40. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  41. data/lib/fluent/plugin_helper/server.rb +4 -2
  42. data/lib/fluent/plugin_helper.rb +1 -0
  43. data/lib/fluent/root_agent.rb +6 -0
  44. data/lib/fluent/supervisor.rb +2 -0
  45. data/lib/fluent/system_config.rb +9 -1
  46. data/lib/fluent/test/driver/storage.rb +30 -0
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  49. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  50. data/test/command/test_plugin_generator.rb +2 -1
  51. data/test/config/test_system_config.rb +6 -0
  52. data/test/config/test_types.rb +7 -0
  53. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  54. data/test/plugin/in_tail/test_position_file.rb +48 -8
  55. data/test/plugin/test_bare_output.rb +13 -0
  56. data/test/plugin/test_buffer.rb +8 -2
  57. data/test/plugin/test_file_wrapper.rb +11 -0
  58. data/test/plugin/test_filter.rb +11 -0
  59. data/test/plugin/test_in_http.rb +40 -0
  60. data/test/plugin/test_in_monitor_agent.rb +214 -8
  61. data/test/plugin/test_in_syslog.rb +35 -0
  62. data/test/plugin/test_in_tail.rb +157 -29
  63. data/test/plugin/test_input.rb +11 -0
  64. data/test/plugin/test_metrics.rb +294 -0
  65. data/test/plugin/test_metrics_local.rb +96 -0
  66. data/test/plugin/test_multi_output.rb +25 -1
  67. data/test/plugin/test_output.rb +16 -0
  68. data/test/plugin_helper/test_event_emitter.rb +29 -0
  69. data/test/plugin_helper/test_metrics.rb +137 -0
  70. data/test/test_event_time.rb +2 -2
  71. data/test/test_oj_options.rb +55 -0
  72. data/test/test_plugin_classes.rb +102 -0
  73. data/test/test_root_agent.rb +30 -1
  74. metadata +21 -6
  75. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  76. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -167,6 +167,8 @@ module Fluent::Plugin
167
167
  @usock = nil
168
168
  @keep_alive_watcher_interval = 5 # TODO
169
169
  @suspend_flush = false
170
+ @healthy_nodes_count_metrics = nil
171
+ @registered_nodes_count_metrics = nil
170
172
  end
171
173
 
172
174
  def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
265
267
  end
266
268
 
267
269
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
270
+ @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
271
+ @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
272
+
268
273
  end
269
274
 
270
275
  def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
418
423
  def statistics
419
424
  stats = super
420
425
  services = service_discovery_services
421
- healthy_nodes_count = 0
422
- registed_nodes_count = services.size
426
+ @healthy_nodes_count_metrics.set(0)
427
+ @registered_nodes_count_metrics.set(services.size)
423
428
  services.each do |s|
424
429
  if s.available?
425
- healthy_nodes_count += 1
430
+ @healthy_nodes_count_metrics.inc
426
431
  end
427
432
  end
428
433
 
429
- stats.merge(
430
- 'healthy_nodes_count' => healthy_nodes_count,
431
- 'registered_nodes_count' => registed_nodes_count,
432
- )
434
+ stats = {
435
+ 'output' => stats["output"].merge({
436
+ 'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
437
+ 'registered_nodes_count' => @registered_nodes_count_metrics.get,
438
+ })
439
+ }
440
+ stats
433
441
  end
434
442
 
435
443
  # MessagePack FixArray length is 3
@@ -37,7 +37,7 @@ module Fluent
37
37
  include PluginHelper::Mixin
38
38
  include UniqueId::Mixin
39
39
 
40
- helpers_internal :thread, :retry_state
40
+ helpers_internal :thread, :retry_state, :metrics
41
41
 
42
42
  CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
43
43
  CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +164,6 @@ module Fluent
164
164
  end
165
165
 
166
166
  attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
167
- attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
168
167
 
169
168
  # for tests
170
169
  attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +171,30 @@ module Fluent
172
171
  # output_enqueue_thread_waiting: for test of output.rb itself
173
172
  attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
174
173
 
174
+ def num_errors
175
+ @num_errors_metrics.get
176
+ end
177
+
178
+ def emit_count
179
+ @emit_count_metrics.get
180
+ end
181
+
182
+ def emit_size
183
+ @emit_size_metrics.get
184
+ end
185
+
186
+ def emit_records
187
+ @emit_records_metrics.get
188
+ end
189
+
190
+ def write_count
191
+ @write_count_metrics.get
192
+ end
193
+
194
+ def rollback_count
195
+ @rollback_count_metrics.get
196
+ end
197
+
175
198
  def initialize
176
199
  super
177
200
  @counter_mutex = Mutex.new
@@ -181,13 +204,15 @@ module Fluent
181
204
  @primary_instance = nil
182
205
 
183
206
  # TODO: well organized counters
184
- @num_errors = 0
185
- @emit_count = 0
186
- @emit_records = 0
187
- @write_count = 0
188
- @rollback_count = 0
189
- @flush_time_count = 0
190
- @slow_flush_count = 0
207
+ @num_errors_metrics = nil
208
+ @emit_count_metrics = nil
209
+ @emit_records_metrics = nil
210
+ @emit_size_metrics = nil
211
+ @write_count_metrics = nil
212
+ @rollback_count_metrics = nil
213
+ @flush_time_count_metrics = nil
214
+ @slow_flush_count_metrics = nil
215
+ @enable_size_metrics = false
191
216
 
192
217
  # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
193
218
  if implement?(:synchronous)
@@ -246,6 +271,15 @@ module Fluent
246
271
 
247
272
  super
248
273
 
274
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
275
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
276
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
277
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
278
+ @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
279
+ @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
280
+ @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
281
+ @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
282
+
249
283
  if has_buffer_section
250
284
  unless implement?(:buffered) || implement?(:delayed_commit)
251
285
  raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +305,8 @@ module Fluent
271
305
  @buffering = true
272
306
  end
273
307
  end
308
+ # Enable to update record size metrics or not
309
+ @enable_size_metrics = !!system_config.enable_size_metrics
274
310
 
275
311
  if @as_secondary
276
312
  if !@buffering && !@buffering.nil?
@@ -797,18 +833,19 @@ module Fluent
797
833
  end
798
834
 
799
835
  def emit_sync(tag, es)
800
- @counter_mutex.synchronize{ @emit_count += 1 }
836
+ @emit_count_metrics.inc
801
837
  begin
802
838
  process(tag, es)
803
- @counter_mutex.synchronize{ @emit_records += es.size }
839
+ @emit_records_metrics.add(es.size)
840
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
804
841
  rescue
805
- @counter_mutex.synchronize{ @num_errors += 1 }
842
+ @num_errors_metrics.inc
806
843
  raise
807
844
  end
808
845
  end
809
846
 
810
847
  def emit_buffered(tag, es)
811
- @counter_mutex.synchronize{ @emit_count += 1 }
848
+ @emit_count_metrics.inc
812
849
  begin
813
850
  execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
814
851
  if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +853,7 @@ module Fluent
816
853
  end
817
854
  rescue
818
855
  # TODO: separate number of errors into emit errors and write/flush errors
819
- @counter_mutex.synchronize{ @num_errors += 1 }
856
+ @num_errors_metrics.inc
820
857
  raise
821
858
  end
822
859
  end
@@ -966,7 +1003,8 @@ module Fluent
966
1003
  write_guard do
967
1004
  @buffer.write(meta_and_data, enqueue: enqueue)
968
1005
  end
969
- @counter_mutex.synchronize{ @emit_records += records }
1006
+ @emit_records_metrics.add(es.size)
1007
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
970
1008
  true
971
1009
  end
972
1010
 
@@ -983,7 +1021,8 @@ module Fluent
983
1021
  write_guard do
984
1022
  @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
985
1023
  end
986
- @counter_mutex.synchronize{ @emit_records += records }
1024
+ @emit_records_metrics.add(es.size)
1025
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
987
1026
  true
988
1027
  end
989
1028
 
@@ -1008,7 +1047,8 @@ module Fluent
1008
1047
  write_guard do
1009
1048
  @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
1010
1049
  end
1011
- @counter_mutex.synchronize{ @emit_records += records }
1050
+ @emit_records_metrics.add(es.size)
1051
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1012
1052
  true
1013
1053
  end
1014
1054
 
@@ -1046,7 +1086,7 @@ module Fluent
1046
1086
  # false if chunk was already flushed and couldn't be rollbacked unexpectedly
1047
1087
  # in many cases, false can be just ignored
1048
1088
  if @buffer.takeback_chunk(chunk_id)
1049
- @counter_mutex.synchronize{ @rollback_count += 1 }
1089
+ @rollback_count_metrics.inc
1050
1090
  if update_retry
1051
1091
  primary = @as_secondary ? @primary_instance : self
1052
1092
  primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1102,7 @@ module Fluent
1062
1102
  while @dequeued_chunks.first && @dequeued_chunks.first.expired?
1063
1103
  info = @dequeued_chunks.shift
1064
1104
  if @buffer.takeback_chunk(info.chunk_id)
1065
- @counter_mutex.synchronize{ @rollback_count += 1 }
1105
+ @rollback_count_metrics.inc
1066
1106
  log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
1067
1107
  primary = @as_secondary ? @primary_instance : self
1068
1108
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1117,7 @@ module Fluent
1077
1117
  until @dequeued_chunks.empty?
1078
1118
  info = @dequeued_chunks.shift
1079
1119
  if @buffer.takeback_chunk(info.chunk_id)
1080
- @counter_mutex.synchronize{ @rollback_count += 1 }
1120
+ @rollback_count_metrics.inc
1081
1121
  log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
1082
1122
  primary = @as_secondary ? @primary_instance : self
1083
1123
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1160,7 @@ module Fluent
1120
1160
 
1121
1161
  if output.delayed_commit
1122
1162
  log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
1123
- @counter_mutex.synchronize{ @write_count += 1 }
1163
+ @write_count_metrics.inc
1124
1164
  @dequeued_chunks_mutex.synchronize do
1125
1165
  # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
1126
1166
  @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1172,7 @@ module Fluent
1132
1172
  chunk_id = chunk.unique_id
1133
1173
  dump_chunk_id = dump_unique_id_hex(chunk_id)
1134
1174
  log.trace "adding write count", instance: self.object_id
1135
- @counter_mutex.synchronize{ @write_count += 1 }
1175
+ @write_count_metrics.inc
1136
1176
  log.trace "executing sync write", chunk: dump_chunk_id
1137
1177
 
1138
1178
  output.write(chunk)
@@ -1188,7 +1228,7 @@ module Fluent
1188
1228
  end
1189
1229
 
1190
1230
  if @buffer.takeback_chunk(chunk.unique_id)
1191
- @counter_mutex.synchronize { @rollback_count += 1 }
1231
+ @rollback_count_metrics.inc
1192
1232
  end
1193
1233
 
1194
1234
  update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1219,9 +1259,9 @@ module Fluent
1219
1259
  def check_slow_flush(start)
1220
1260
  elapsed_time = Fluent::Clock.now - start
1221
1261
  elapsed_millsec = (elapsed_time * 1000).to_i
1222
- @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
1262
+ @flush_time_count_metrics.add(elapsed_millsec)
1223
1263
  if elapsed_time > @slow_flush_log_threshold
1224
- @counter_mutex.synchronize { @slow_flush_count += 1 }
1264
+ @slow_flush_count_metrics.inc
1225
1265
  log.warn "buffer flush took longer time than slow_flush_log_threshold:",
1226
1266
  elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
1227
1267
  end
@@ -1229,13 +1269,13 @@ module Fluent
1229
1269
 
1230
1270
  def update_retry_state(chunk_id, using_secondary, error = nil)
1231
1271
  @retry_mutex.synchronize do
1232
- @counter_mutex.synchronize{ @num_errors += 1 }
1272
+ @num_errors_metrics.inc
1233
1273
  chunk_id_hex = dump_unique_id_hex(chunk_id)
1234
1274
 
1235
1275
  unless @retry
1236
1276
  @retry = retry_state(@buffer_config.retry_randomize)
1237
1277
  if error
1238
- log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1278
+ log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1239
1279
  log.warn_backtrace error.backtrace
1240
1280
  end
1241
1281
  return
@@ -1264,11 +1304,11 @@ module Fluent
1264
1304
  if error
1265
1305
  if using_secondary
1266
1306
  msg = "failed to flush the buffer with secondary output."
1267
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1307
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1268
1308
  log.warn_backtrace error.backtrace
1269
1309
  else
1270
1310
  msg = "failed to flush the buffer."
1271
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1311
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1272
1312
  log.warn_backtrace error.backtrace
1273
1313
  end
1274
1314
  end
@@ -1490,15 +1530,16 @@ module Fluent
1490
1530
 
1491
1531
  def statistics
1492
1532
  stats = {
1493
- 'emit_records' => @emit_records,
1533
+ 'emit_records' => @emit_records_metrics.get,
1534
+ 'emit_size' => @emit_size_metrics.get,
1494
1535
  # Respect original name
1495
1536
  # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
1496
- 'retry_count' => @num_errors,
1497
- 'emit_count' => @emit_count,
1498
- 'write_count' => @write_count,
1499
- 'rollback_count' => @rollback_count,
1500
- 'slow_flush_count' => @slow_flush_count,
1501
- 'flush_time_count' => @flush_time_count,
1537
+ 'retry_count' => @num_errors_metrics.get,
1538
+ 'emit_count' => @emit_count_metrics.get,
1539
+ 'write_count' => @write_count_metrics.get,
1540
+ 'rollback_count' => @rollback_count_metrics.get,
1541
+ 'slow_flush_count' => @slow_flush_count_metrics.get,
1542
+ 'flush_time_count' => @flush_time_count_metrics.get,
1502
1543
  }
1503
1544
 
1504
1545
  if @buffer && @buffer.respond_to?(:statistics)
@@ -15,8 +15,8 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin/parser'
18
- require 'fluent/env'
19
18
  require 'fluent/time'
19
+ require 'fluent/oj_options'
20
20
 
21
21
  require 'yajl'
22
22
  require 'json'
@@ -50,8 +50,7 @@ module Fluent
50
50
  def configure_json_parser(name)
51
51
  case name
52
52
  when :oj
53
- require 'oj'
54
- Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
53
+ raise LoadError unless Fluent::OjOptions.available?
55
54
  [Oj.method(:load), Oj::ParseError]
56
55
  when :json then [JSON.method(:load), JSON::ParserError]
57
56
  when :yajl then [Yajl.method(:load), Yajl::ParseError]
data/lib/fluent/plugin.rb CHANGED
@@ -36,8 +36,9 @@ module Fluent
36
36
  FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_', dir_search_prefix: 'formatter_')
37
37
  STORAGE_REGISTRY = Registry.new(:storage, 'fluent/plugin/storage_', dir_search_prefix: 'storage_')
38
38
  SD_REGISTRY = Registry.new(:sd, 'fluent/plugin/sd_', dir_search_prefix: 'sd_')
39
+ METRICS_REGISTRY = Registry.new(:metrics, 'fluent/plugin/metrics_', dir_search_prefix: 'metrics_')
39
40
 
40
- REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
41
+ REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
41
42
 
42
43
  def self.register_input(type, klass)
43
44
  register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
59
60
  register_impl('sd', SD_REGISTRY, type, klass)
60
61
  end
61
62
 
63
+ def self.register_metrics(type, klass)
64
+ register_impl('metrics', METRICS_REGISTRY, type, klass)
65
+ end
66
+
62
67
  def self.register_parser(type, klass_or_proc)
63
68
  if klass_or_proc.is_a?(Regexp)
64
69
  # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
121
126
  new_impl('sd', SD_REGISTRY, type, parent)
122
127
  end
123
128
 
129
+ def self.new_metrics(type, parent: nil)
130
+ new_impl('metrics', METRICS_REGISTRY, type, parent)
131
+ end
132
+
124
133
  class << self
125
134
  # This should be defined for fluent-plugin-config-formatter type arguments.
126
135
  alias_method :new_service_discovery, :new_sd
@@ -29,6 +29,9 @@ module Fluent
29
29
  if @_event_emitter_lazy_init
30
30
  @router = @primary_instance.router
31
31
  end
32
+ if @router.respond_to?(:caller_plugin_id=)
33
+ @router.caller_plugin_id = self.plugin_id
34
+ end
32
35
  @router
33
36
  end
34
37
 
@@ -47,7 +50,11 @@ module Fluent
47
50
 
48
51
  def event_emitter_router(label_name)
49
52
  if label_name
50
- Engine.root_agent.find_label(label_name).event_router
53
+ if label_name == "@ROOT"
54
+ Engine.root_agent.event_router
55
+ else
56
+ Engine.root_agent.find_label(label_name).event_router
57
+ end
51
58
  elsif self.respond_to?(:as_secondary) && self.as_secondary
52
59
  if @primary_instance.has_router?
53
60
  @_event_emitter_lazy_init = true
@@ -0,0 +1,129 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'forwardable'
18
+
19
+ require 'fluent/plugin'
20
+ require 'fluent/plugin/metrics'
21
+ require 'fluent/plugin_helper/timer'
22
+ require 'fluent/config/element'
23
+ require 'fluent/configurable'
24
+ require 'fluent/system_config'
25
+
26
+ module Fluent
27
+ module PluginHelper
28
+ module Metrics
29
+ include Fluent::SystemConfig::Mixin
30
+
31
+ attr_reader :_metrics # For tests.
32
+
33
+ def initialize
34
+ super
35
+ @_metrics_started = false
36
+ @_metrics = {} # usage => metrics_state
37
+ end
38
+
39
+ def configure(conf)
40
+ super
41
+
42
+ @plugin_type_or_id = if self.plugin_id_configured?
43
+ self.plugin_id
44
+ else
45
+ if type = (conf["@type"] || conf["type"])
46
+ "#{type}.#{self.plugin_id}"
47
+ else
48
+ "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
49
+ end
50
+ end
51
+ end
52
+
53
+ def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
54
+ metrics = if system_config.metrics
55
+ Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
56
+ else
57
+ Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
58
+ end
59
+ config = if system_config.metrics
60
+ system_config.metrics.corresponding_config_element
61
+ else
62
+ Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
63
+ end
64
+ metrics.use_gauge_metric = prefer_gauge
65
+ metrics.configure(config)
66
+ # For multi workers environment, cmetrics should be distinguish with static labels.
67
+ if Fluent::Engine.system_config.workers > 1
68
+ labels.merge!(worker_id: fluentd_worker_id.to_s)
69
+ end
70
+ labels.merge!(plugin: @plugin_type_or_id)
71
+ metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
72
+
73
+ @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
74
+
75
+ metrics
76
+ end
77
+
78
+ def metrics_operate(method_name, &block)
79
+ @_metrics.each_pair do |key, m|
80
+ begin
81
+ block.call(s) if block_given?
82
+ m.__send__(method_name)
83
+ rescue => e
84
+ log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
85
+ end
86
+ end
87
+ end
88
+
89
+ def start
90
+ super
91
+
92
+ metrics_operate(:start)
93
+ @_metrics_started = true
94
+ end
95
+
96
+ def stop
97
+ super
98
+ # timer stops automatically in super
99
+ metrics_operate(:stop)
100
+ end
101
+
102
+ def before_shutdown
103
+ metrics_operate(:before_shutdown)
104
+ super
105
+ end
106
+
107
+ def shutdown
108
+ metrics_operate(:shutdown)
109
+ super
110
+ end
111
+
112
+ def after_shutdown
113
+ metrics_operate(:after_shutdown)
114
+ super
115
+ end
116
+
117
+ def close
118
+ metrics_operate(:close)
119
+ super
120
+ end
121
+
122
+ def terminate
123
+ metrics_operate(:terminate)
124
+ @_metrics = {}
125
+ super
126
+ end
127
+ end
128
+ end
129
+ end
@@ -709,13 +709,15 @@ module Fluent
709
709
  return true
710
710
  end
711
711
  rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
712
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
712
713
  @log.trace "unexpected error before accepting TLS connection",
713
- host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
714
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
714
715
  close rescue nil
715
716
  rescue OpenSSL::SSL::SSLError => e
717
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
716
718
  # Use same log level as on_readable
717
719
  @log.warn "unexpected error before accepting TLS connection by OpenSSL",
718
- host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
720
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
719
721
  close rescue nil
720
722
  end
721
723
 
@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
32
32
  require 'fluent/plugin_helper/record_accessor'
33
33
  require 'fluent/plugin_helper/compat_parameters'
34
34
  require 'fluent/plugin_helper/service_discovery'
35
+ require 'fluent/plugin_helper/metrics'
35
36
 
36
37
  module Fluent
37
38
  module PluginHelper
@@ -55,9 +55,11 @@ module Fluent
55
55
  @suppress_emit_error_log_interval = 0
56
56
  @next_emit_error_log_time = nil
57
57
  @without_source = false
58
+ @enable_input_metrics = false
58
59
 
59
60
  suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
60
61
  @without_source = system_config.without_source unless system_config.without_source.nil?
62
+ @enable_input_metrics = !!system_config.enable_input_metrics
61
63
  end
62
64
 
63
65
  attr_reader :inputs
@@ -131,6 +133,7 @@ module Fluent
131
133
  end
132
134
  name = e.arg
133
135
  raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
136
+ raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
134
137
 
135
138
  if name == ERROR_LABEL
136
139
  error_label_config = e
@@ -315,6 +318,9 @@ module Fluent
315
318
  # See also 'fluentd/plugin/input.rb'
316
319
  input.context_router = @event_router
317
320
  input.configure(conf)
321
+ if @enable_input_metrics
322
+ @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
323
+ end
318
324
  @inputs << input
319
325
 
320
326
  input
@@ -570,6 +570,8 @@ module Fluent
570
570
  suppress_repeated_stacktrace: true,
571
571
  ignore_repeated_log_interval: nil,
572
572
  without_source: nil,
573
+ enable_input_metrics: nil,
574
+ enable_size_metrics: nil,
573
575
  use_v1_config: true,
574
576
  strict_config_value: nil,
575
577
  supervise: true,
@@ -27,7 +27,8 @@ module Fluent
27
27
  :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
28
28
  :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
29
29
  :file_permission, :dir_permission, :counter_server, :counter_client,
30
- :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
30
+ :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
31
+ :metrics, :enable_input_metrics, :enable_size_metrics
31
32
  ]
32
33
 
33
34
  config_param :workers, :integer, default: 1
@@ -46,6 +47,8 @@ module Fluent
46
47
  config_param :strict_config_value, :bool, default: nil
47
48
  config_param :enable_msgpack_time_support, :bool, default: nil
48
49
  config_param :disable_shared_socket, :bool, default: nil
50
+ config_param :enable_input_metrics, :bool, default: nil
51
+ config_param :enable_size_metrics, :bool, default: nil
49
52
  config_param :file_permission, default: nil do |v|
50
53
  v.to_i(8)
51
54
  end
@@ -93,6 +96,11 @@ module Fluent
93
96
  config_param :timeout, :time, default: nil
94
97
  end
95
98
 
99
+ config_section :metrics, multi: false do
100
+ config_param :@type, :string, default: "local"
101
+ config_param :labels, :hash, default: {}
102
+ end
103
+
96
104
  def self.create(conf, strict_config_value=false)
97
105
  systems = conf.elements(name: 'system')
98
106
  return SystemConfig.new if systems.empty?
@@ -0,0 +1,30 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/test/driver/base_owned'
18
+
19
+ module Fluent
20
+ module Test
21
+ module Driver
22
+ class Storage < BaseOwned
23
+ def initialize(klass, **kwargs, &block)
24
+ super
25
+ @section_name = "storage"
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.13.1'
19
+ VERSION = '1.14.1'
20
20
 
21
21
  end