fluentd 1.13.3-x86-mingw32 → 1.14.3-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +6 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +126 -0
  6. data/README.md +2 -0
  7. data/SECURITY.md +18 -0
  8. data/fluentd.gemspec +3 -3
  9. data/lib/fluent/command/cat.rb +13 -3
  10. data/lib/fluent/command/fluentd.rb +8 -0
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/event_router.rb +28 -1
  15. data/lib/fluent/plugin/bare_output.rb +49 -8
  16. data/lib/fluent/plugin/buf_file.rb +2 -2
  17. data/lib/fluent/plugin/buffer.rb +123 -27
  18. data/lib/fluent/plugin/filter.rb +35 -1
  19. data/lib/fluent/plugin/in_http.rb +21 -2
  20. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  21. data/lib/fluent/plugin/in_syslog.rb +13 -1
  22. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  23. data/lib/fluent/plugin/in_tail.rb +47 -8
  24. data/lib/fluent/plugin/input.rb +39 -1
  25. data/lib/fluent/plugin/metrics.rb +119 -0
  26. data/lib/fluent/plugin/metrics_local.rb +96 -0
  27. data/lib/fluent/plugin/multi_output.rb +43 -6
  28. data/lib/fluent/plugin/out_copy.rb +1 -1
  29. data/lib/fluent/plugin/out_forward.rb +15 -7
  30. data/lib/fluent/plugin/output.rb +80 -38
  31. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  32. data/lib/fluent/plugin/storage_local.rb +3 -5
  33. data/lib/fluent/plugin.rb +10 -1
  34. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  35. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  36. data/lib/fluent/plugin_helper/server.rb +4 -2
  37. data/lib/fluent/plugin_helper.rb +1 -0
  38. data/lib/fluent/plugin_id.rb +2 -1
  39. data/lib/fluent/root_agent.rb +6 -0
  40. data/lib/fluent/supervisor.rb +4 -2
  41. data/lib/fluent/system_config.rb +9 -1
  42. data/lib/fluent/time.rb +21 -20
  43. data/lib/fluent/version.rb +1 -1
  44. data/test/command/test_cat.rb +31 -2
  45. data/test/config/test_system_config.rb +6 -0
  46. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  47. data/test/plugin/in_tail/test_position_file.rb +26 -4
  48. data/test/plugin/test_bare_output.rb +13 -0
  49. data/test/plugin/test_buffer.rb +80 -3
  50. data/test/plugin/test_filter.rb +11 -0
  51. data/test/plugin/test_in_http.rb +40 -0
  52. data/test/plugin/test_in_monitor_agent.rb +214 -8
  53. data/test/plugin/test_in_syslog.rb +35 -0
  54. data/test/plugin/test_in_tail.rb +83 -35
  55. data/test/plugin/test_input.rb +11 -0
  56. data/test/plugin/test_metrics.rb +294 -0
  57. data/test/plugin/test_metrics_local.rb +96 -0
  58. data/test/plugin/test_multi_output.rb +25 -1
  59. data/test/plugin/test_out_exec_filter.rb +4 -0
  60. data/test/plugin/test_output.rb +16 -0
  61. data/test/plugin_helper/test_child_process.rb +9 -9
  62. data/test/plugin_helper/test_event_emitter.rb +29 -0
  63. data/test/plugin_helper/test_metrics.rb +137 -0
  64. data/test/test_plugin_classes.rb +102 -0
  65. data/test/test_root_agent.rb +30 -1
  66. data/test/test_time_parser.rb +22 -0
  67. metadata +18 -8
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/env'
17
18
  require 'fluent/error'
18
19
  require 'fluent/plugin/base'
19
20
  require 'fluent/plugin/buffer'
@@ -37,7 +38,7 @@ module Fluent
37
38
  include PluginHelper::Mixin
38
39
  include UniqueId::Mixin
39
40
 
40
- helpers_internal :thread, :retry_state
41
+ helpers_internal :thread, :retry_state, :metrics
41
42
 
42
43
  CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
43
44
  CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +165,6 @@ module Fluent
164
165
  end
165
166
 
166
167
  attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
167
- attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
168
168
 
169
169
  # for tests
170
170
  attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +172,30 @@ module Fluent
172
172
  # output_enqueue_thread_waiting: for test of output.rb itself
173
173
  attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
174
174
 
175
+ def num_errors
176
+ @num_errors_metrics.get
177
+ end
178
+
179
+ def emit_count
180
+ @emit_count_metrics.get
181
+ end
182
+
183
+ def emit_size
184
+ @emit_size_metrics.get
185
+ end
186
+
187
+ def emit_records
188
+ @emit_records_metrics.get
189
+ end
190
+
191
+ def write_count
192
+ @write_count_metrics.get
193
+ end
194
+
195
+ def rollback_count
196
+ @rollback_count_metrics.get
197
+ end
198
+
175
199
  def initialize
176
200
  super
177
201
  @counter_mutex = Mutex.new
@@ -181,13 +205,15 @@ module Fluent
181
205
  @primary_instance = nil
182
206
 
183
207
  # TODO: well organized counters
184
- @num_errors = 0
185
- @emit_count = 0
186
- @emit_records = 0
187
- @write_count = 0
188
- @rollback_count = 0
189
- @flush_time_count = 0
190
- @slow_flush_count = 0
208
+ @num_errors_metrics = nil
209
+ @emit_count_metrics = nil
210
+ @emit_records_metrics = nil
211
+ @emit_size_metrics = nil
212
+ @write_count_metrics = nil
213
+ @rollback_count_metrics = nil
214
+ @flush_time_count_metrics = nil
215
+ @slow_flush_count_metrics = nil
216
+ @enable_size_metrics = false
191
217
 
192
218
  # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
193
219
  if implement?(:synchronous)
@@ -246,6 +272,15 @@ module Fluent
246
272
 
247
273
  super
248
274
 
275
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
276
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
277
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
278
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
279
+ @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
280
+ @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
281
+ @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
282
+ @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
283
+
249
284
  if has_buffer_section
250
285
  unless implement?(:buffered) || implement?(:delayed_commit)
251
286
  raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +306,8 @@ module Fluent
271
306
  @buffering = true
272
307
  end
273
308
  end
309
+ # Enable to update record size metrics or not
310
+ @enable_size_metrics = !!system_config.enable_size_metrics
274
311
 
275
312
  if @as_secondary
276
313
  if !@buffering && !@buffering.nil?
@@ -797,18 +834,19 @@ module Fluent
797
834
  end
798
835
 
799
836
  def emit_sync(tag, es)
800
- @counter_mutex.synchronize{ @emit_count += 1 }
837
+ @emit_count_metrics.inc
801
838
  begin
802
839
  process(tag, es)
803
- @counter_mutex.synchronize{ @emit_records += es.size }
840
+ @emit_records_metrics.add(es.size)
841
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
804
842
  rescue
805
- @counter_mutex.synchronize{ @num_errors += 1 }
843
+ @num_errors_metrics.inc
806
844
  raise
807
845
  end
808
846
  end
809
847
 
810
848
  def emit_buffered(tag, es)
811
- @counter_mutex.synchronize{ @emit_count += 1 }
849
+ @emit_count_metrics.inc
812
850
  begin
813
851
  execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
814
852
  if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +854,7 @@ module Fluent
816
854
  end
817
855
  rescue
818
856
  # TODO: separate number of errors into emit errors and write/flush errors
819
- @counter_mutex.synchronize{ @num_errors += 1 }
857
+ @num_errors_metrics.inc
820
858
  raise
821
859
  end
822
860
  end
@@ -966,7 +1004,8 @@ module Fluent
966
1004
  write_guard do
967
1005
  @buffer.write(meta_and_data, enqueue: enqueue)
968
1006
  end
969
- @counter_mutex.synchronize{ @emit_records += records }
1007
+ @emit_records_metrics.add(es.size)
1008
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
970
1009
  true
971
1010
  end
972
1011
 
@@ -983,7 +1022,8 @@ module Fluent
983
1022
  write_guard do
984
1023
  @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
985
1024
  end
986
- @counter_mutex.synchronize{ @emit_records += records }
1025
+ @emit_records_metrics.add(es.size)
1026
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
987
1027
  true
988
1028
  end
989
1029
 
@@ -1008,7 +1048,8 @@ module Fluent
1008
1048
  write_guard do
1009
1049
  @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
1010
1050
  end
1011
- @counter_mutex.synchronize{ @emit_records += records }
1051
+ @emit_records_metrics.add(es.size)
1052
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
1012
1053
  true
1013
1054
  end
1014
1055
 
@@ -1046,7 +1087,7 @@ module Fluent
1046
1087
  # false if chunk was already flushed and couldn't be rollbacked unexpectedly
1047
1088
  # in many cases, false can be just ignored
1048
1089
  if @buffer.takeback_chunk(chunk_id)
1049
- @counter_mutex.synchronize{ @rollback_count += 1 }
1090
+ @rollback_count_metrics.inc
1050
1091
  if update_retry
1051
1092
  primary = @as_secondary ? @primary_instance : self
1052
1093
  primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1103,7 @@ module Fluent
1062
1103
  while @dequeued_chunks.first && @dequeued_chunks.first.expired?
1063
1104
  info = @dequeued_chunks.shift
1064
1105
  if @buffer.takeback_chunk(info.chunk_id)
1065
- @counter_mutex.synchronize{ @rollback_count += 1 }
1106
+ @rollback_count_metrics.inc
1066
1107
  log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
1067
1108
  primary = @as_secondary ? @primary_instance : self
1068
1109
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1118,7 @@ module Fluent
1077
1118
  until @dequeued_chunks.empty?
1078
1119
  info = @dequeued_chunks.shift
1079
1120
  if @buffer.takeback_chunk(info.chunk_id)
1080
- @counter_mutex.synchronize{ @rollback_count += 1 }
1121
+ @rollback_count_metrics.inc
1081
1122
  log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
1082
1123
  primary = @as_secondary ? @primary_instance : self
1083
1124
  primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1161,7 @@ module Fluent
1120
1161
 
1121
1162
  if output.delayed_commit
1122
1163
  log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
1123
- @counter_mutex.synchronize{ @write_count += 1 }
1164
+ @write_count_metrics.inc
1124
1165
  @dequeued_chunks_mutex.synchronize do
1125
1166
  # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
1126
1167
  @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1173,7 @@ module Fluent
1132
1173
  chunk_id = chunk.unique_id
1133
1174
  dump_chunk_id = dump_unique_id_hex(chunk_id)
1134
1175
  log.trace "adding write count", instance: self.object_id
1135
- @counter_mutex.synchronize{ @write_count += 1 }
1176
+ @write_count_metrics.inc
1136
1177
  log.trace "executing sync write", chunk: dump_chunk_id
1137
1178
 
1138
1179
  output.write(chunk)
@@ -1188,7 +1229,7 @@ module Fluent
1188
1229
  end
1189
1230
 
1190
1231
  if @buffer.takeback_chunk(chunk.unique_id)
1191
- @counter_mutex.synchronize { @rollback_count += 1 }
1232
+ @rollback_count_metrics.inc
1192
1233
  end
1193
1234
 
1194
1235
  update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1208,8 +1249,8 @@ module Fluent
1208
1249
  backup_dir = File.dirname(backup_file)
1209
1250
 
1210
1251
  log.warn "bad chunk is moved to #{backup_file}"
1211
- FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
1212
- File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
1252
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
1253
+ File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
1213
1254
  chunk.write_to(f)
1214
1255
  }
1215
1256
  end
@@ -1219,9 +1260,9 @@ module Fluent
1219
1260
  def check_slow_flush(start)
1220
1261
  elapsed_time = Fluent::Clock.now - start
1221
1262
  elapsed_millsec = (elapsed_time * 1000).to_i
1222
- @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
1263
+ @flush_time_count_metrics.add(elapsed_millsec)
1223
1264
  if elapsed_time > @slow_flush_log_threshold
1224
- @counter_mutex.synchronize { @slow_flush_count += 1 }
1265
+ @slow_flush_count_metrics.inc
1225
1266
  log.warn "buffer flush took longer time than slow_flush_log_threshold:",
1226
1267
  elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
1227
1268
  end
@@ -1229,13 +1270,13 @@ module Fluent
1229
1270
 
1230
1271
  def update_retry_state(chunk_id, using_secondary, error = nil)
1231
1272
  @retry_mutex.synchronize do
1232
- @counter_mutex.synchronize{ @num_errors += 1 }
1273
+ @num_errors_metrics.inc
1233
1274
  chunk_id_hex = dump_unique_id_hex(chunk_id)
1234
1275
 
1235
1276
  unless @retry
1236
1277
  @retry = retry_state(@buffer_config.retry_randomize)
1237
1278
  if error
1238
- log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1279
+ log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1239
1280
  log.warn_backtrace error.backtrace
1240
1281
  end
1241
1282
  return
@@ -1264,11 +1305,11 @@ module Fluent
1264
1305
  if error
1265
1306
  if using_secondary
1266
1307
  msg = "failed to flush the buffer with secondary output."
1267
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1308
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1268
1309
  log.warn_backtrace error.backtrace
1269
1310
  else
1270
1311
  msg = "failed to flush the buffer."
1271
- log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
1312
+ log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
1272
1313
  log.warn_backtrace error.backtrace
1273
1314
  end
1274
1315
  end
@@ -1490,15 +1531,16 @@ module Fluent
1490
1531
 
1491
1532
  def statistics
1492
1533
  stats = {
1493
- 'emit_records' => @emit_records,
1534
+ 'emit_records' => @emit_records_metrics.get,
1535
+ 'emit_size' => @emit_size_metrics.get,
1494
1536
  # Respect original name
1495
1537
  # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
1496
- 'retry_count' => @num_errors,
1497
- 'emit_count' => @emit_count,
1498
- 'write_count' => @write_count,
1499
- 'rollback_count' => @rollback_count,
1500
- 'slow_flush_count' => @slow_flush_count,
1501
- 'flush_time_count' => @flush_time_count,
1538
+ 'retry_count' => @num_errors_metrics.get,
1539
+ 'emit_count' => @emit_count_metrics.get,
1540
+ 'write_count' => @write_count_metrics.get,
1541
+ 'rollback_count' => @rollback_count_metrics.get,
1542
+ 'slow_flush_count' => @slow_flush_count_metrics.get,
1543
+ 'flush_time_count' => @flush_time_count_metrics.get,
1502
1544
  }
1503
1545
 
1504
1546
  if @buffer && @buffer.respond_to?(:statistics)
@@ -21,7 +21,7 @@ module Fluent
21
21
  class Apache2Parser < Parser
22
22
  Plugin.register_parser('apache2', self)
23
23
 
24
- REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
24
+ REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
25
25
  TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
26
26
 
27
27
  def initialize
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/env'
17
18
  require 'fluent/plugin'
18
19
  require 'fluent/plugin/storage'
19
20
 
@@ -25,14 +26,11 @@ module Fluent
25
26
  class LocalStorage < Storage
26
27
  Fluent::Plugin.register_storage('local', self)
27
28
 
28
- DEFAULT_DIR_MODE = 0755
29
- DEFAULT_FILE_MODE = 0644
30
-
31
29
  config_param :path, :string, default: nil
32
- config_param :mode, default: DEFAULT_FILE_MODE do |v|
30
+ config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
33
31
  v.to_i(8)
34
32
  end
35
- config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
33
+ config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
36
34
  v.to_i(8)
37
35
  end
38
36
  config_param :pretty_print, :bool, default: false
data/lib/fluent/plugin.rb CHANGED
@@ -36,8 +36,9 @@ module Fluent
36
36
  FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_', dir_search_prefix: 'formatter_')
37
37
  STORAGE_REGISTRY = Registry.new(:storage, 'fluent/plugin/storage_', dir_search_prefix: 'storage_')
38
38
  SD_REGISTRY = Registry.new(:sd, 'fluent/plugin/sd_', dir_search_prefix: 'sd_')
39
+ METRICS_REGISTRY = Registry.new(:metrics, 'fluent/plugin/metrics_', dir_search_prefix: 'metrics_')
39
40
 
40
- REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
41
+ REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
41
42
 
42
43
  def self.register_input(type, klass)
43
44
  register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
59
60
  register_impl('sd', SD_REGISTRY, type, klass)
60
61
  end
61
62
 
63
+ def self.register_metrics(type, klass)
64
+ register_impl('metrics', METRICS_REGISTRY, type, klass)
65
+ end
66
+
62
67
  def self.register_parser(type, klass_or_proc)
63
68
  if klass_or_proc.is_a?(Regexp)
64
69
  # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
121
126
  new_impl('sd', SD_REGISTRY, type, parent)
122
127
  end
123
128
 
129
+ def self.new_metrics(type, parent: nil)
130
+ new_impl('metrics', METRICS_REGISTRY, type, parent)
131
+ end
132
+
124
133
  class << self
125
134
  # This should be defined for fluent-plugin-config-formatter type arguments.
126
135
  alias_method :new_service_discovery, :new_sd
@@ -29,6 +29,9 @@ module Fluent
29
29
  if @_event_emitter_lazy_init
30
30
  @router = @primary_instance.router
31
31
  end
32
+ if @router.respond_to?(:caller_plugin_id=)
33
+ @router.caller_plugin_id = self.plugin_id
34
+ end
32
35
  @router
33
36
  end
34
37
 
@@ -47,7 +50,11 @@ module Fluent
47
50
 
48
51
  def event_emitter_router(label_name)
49
52
  if label_name
50
- Engine.root_agent.find_label(label_name).event_router
53
+ if label_name == "@ROOT"
54
+ Engine.root_agent.event_router
55
+ else
56
+ Engine.root_agent.find_label(label_name).event_router
57
+ end
51
58
  elsif self.respond_to?(:as_secondary) && self.as_secondary
52
59
  if @primary_instance.has_router?
53
60
  @_event_emitter_lazy_init = true
@@ -0,0 +1,129 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'forwardable'
18
+
19
+ require 'fluent/plugin'
20
+ require 'fluent/plugin/metrics'
21
+ require 'fluent/plugin_helper/timer'
22
+ require 'fluent/config/element'
23
+ require 'fluent/configurable'
24
+ require 'fluent/system_config'
25
+
26
+ module Fluent
27
+ module PluginHelper
28
+ module Metrics
29
+ include Fluent::SystemConfig::Mixin
30
+
31
+ attr_reader :_metrics # For tests.
32
+
33
+ def initialize
34
+ super
35
+ @_metrics_started = false
36
+ @_metrics = {} # usage => metrics_state
37
+ end
38
+
39
+ def configure(conf)
40
+ super
41
+
42
+ @plugin_type_or_id = if self.plugin_id_configured?
43
+ self.plugin_id
44
+ else
45
+ if type = (conf["@type"] || conf["type"])
46
+ "#{type}.#{self.plugin_id}"
47
+ else
48
+ "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
49
+ end
50
+ end
51
+ end
52
+
53
+ def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
54
+ metrics = if system_config.metrics
55
+ Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
56
+ else
57
+ Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
58
+ end
59
+ config = if system_config.metrics
60
+ system_config.metrics.corresponding_config_element
61
+ else
62
+ Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
63
+ end
64
+ metrics.use_gauge_metric = prefer_gauge
65
+ metrics.configure(config)
66
+ # For multi workers environment, cmetrics should be distinguish with static labels.
67
+ if Fluent::Engine.system_config.workers > 1
68
+ labels.merge!(worker_id: fluentd_worker_id.to_s)
69
+ end
70
+ labels.merge!(plugin: @plugin_type_or_id)
71
+ metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
72
+
73
+ @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
74
+
75
+ metrics
76
+ end
77
+
78
+ def metrics_operate(method_name, &block)
79
+ @_metrics.each_pair do |key, m|
80
+ begin
81
+ block.call(s) if block_given?
82
+ m.__send__(method_name)
83
+ rescue => e
84
+ log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
85
+ end
86
+ end
87
+ end
88
+
89
+ def start
90
+ super
91
+
92
+ metrics_operate(:start)
93
+ @_metrics_started = true
94
+ end
95
+
96
+ def stop
97
+ super
98
+ # timer stops automatically in super
99
+ metrics_operate(:stop)
100
+ end
101
+
102
+ def before_shutdown
103
+ metrics_operate(:before_shutdown)
104
+ super
105
+ end
106
+
107
+ def shutdown
108
+ metrics_operate(:shutdown)
109
+ super
110
+ end
111
+
112
+ def after_shutdown
113
+ metrics_operate(:after_shutdown)
114
+ super
115
+ end
116
+
117
+ def close
118
+ metrics_operate(:close)
119
+ super
120
+ end
121
+
122
+ def terminate
123
+ metrics_operate(:terminate)
124
+ @_metrics = {}
125
+ super
126
+ end
127
+ end
128
+ end
129
+ end
@@ -709,13 +709,15 @@ module Fluent
709
709
  return true
710
710
  end
711
711
  rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
712
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
712
713
  @log.trace "unexpected error before accepting TLS connection",
713
- host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
714
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
714
715
  close rescue nil
715
716
  rescue OpenSSL::SSL::SSLError => e
717
+ peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
716
718
  # Use same log level as on_readable
717
719
  @log.warn "unexpected error before accepting TLS connection by OpenSSL",
718
- host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
720
+ addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
719
721
  close rescue nil
720
722
  end
721
723
 
@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
32
32
  require 'fluent/plugin_helper/record_accessor'
33
33
  require 'fluent/plugin_helper/compat_parameters'
34
34
  require 'fluent/plugin_helper/service_discovery'
35
+ require 'fluent/plugin_helper/metrics'
35
36
 
36
37
  module Fluent
37
38
  module PluginHelper
@@ -15,6 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'set'
18
+ require 'fluent/env'
18
19
  require 'fluent/variable_store'
19
20
 
20
21
  module Fluent
@@ -76,7 +77,7 @@ module Fluent
76
77
 
77
78
  # Fluent::Plugin::Base#fluentd_worker_id
78
79
  dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
79
- FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
80
+ FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
80
81
  @_plugin_root_dir = dir.freeze
81
82
  dir
82
83
  end
@@ -55,9 +55,11 @@ module Fluent
55
55
  @suppress_emit_error_log_interval = 0
56
56
  @next_emit_error_log_time = nil
57
57
  @without_source = false
58
+ @enable_input_metrics = false
58
59
 
59
60
  suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
60
61
  @without_source = system_config.without_source unless system_config.without_source.nil?
62
+ @enable_input_metrics = !!system_config.enable_input_metrics
61
63
  end
62
64
 
63
65
  attr_reader :inputs
@@ -131,6 +133,7 @@ module Fluent
131
133
  end
132
134
  name = e.arg
133
135
  raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
136
+ raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
134
137
 
135
138
  if name == ERROR_LABEL
136
139
  error_label_config = e
@@ -315,6 +318,9 @@ module Fluent
315
318
  # See also 'fluentd/plugin/input.rb'
316
319
  input.context_router = @event_router
317
320
  input.configure(conf)
321
+ if @enable_input_metrics
322
+ @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
323
+ end
318
324
  @inputs << input
319
325
 
320
326
  input
@@ -544,7 +544,7 @@ module Fluent
544
544
  $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
545
545
 
546
546
  if @path && log_dir_perm
547
- File.chmod(log_dir_perm || 0755, File.dirname(@path))
547
+ File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
548
548
  end
549
549
  end
550
550
 
@@ -570,6 +570,8 @@ module Fluent
570
570
  suppress_repeated_stacktrace: true,
571
571
  ignore_repeated_log_interval: nil,
572
572
  without_source: nil,
573
+ enable_input_metrics: nil,
574
+ enable_size_metrics: nil,
573
575
  use_v1_config: true,
574
576
  strict_config_value: nil,
575
577
  supervise: true,
@@ -649,7 +651,7 @@ module Fluent
649
651
  end
650
652
  else
651
653
  begin
652
- FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
654
+ FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
653
655
  rescue => e
654
656
  raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
655
657
  end
@@ -27,7 +27,8 @@ module Fluent
27
27
  :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
28
28
  :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
29
29
  :file_permission, :dir_permission, :counter_server, :counter_client,
30
- :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
30
+ :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
31
+ :metrics, :enable_input_metrics, :enable_size_metrics
31
32
  ]
32
33
 
33
34
  config_param :workers, :integer, default: 1
@@ -46,6 +47,8 @@ module Fluent
46
47
  config_param :strict_config_value, :bool, default: nil
47
48
  config_param :enable_msgpack_time_support, :bool, default: nil
48
49
  config_param :disable_shared_socket, :bool, default: nil
50
+ config_param :enable_input_metrics, :bool, default: nil
51
+ config_param :enable_size_metrics, :bool, default: nil
49
52
  config_param :file_permission, default: nil do |v|
50
53
  v.to_i(8)
51
54
  end
@@ -93,6 +96,11 @@ module Fluent
93
96
  config_param :timeout, :time, default: nil
94
97
  end
95
98
 
99
+ config_section :metrics, multi: false do
100
+ config_param :@type, :string, default: "local"
101
+ config_param :labels, :hash, default: {}
102
+ end
103
+
96
104
  def self.create(conf, strict_config_value=false)
97
105
  systems = conf.elements(name: 'system')
98
106
  return SystemConfig.new if systems.empty?