fluentd 1.13.2-x64-mingw32 → 1.14.2-x64-mingw32

data/lib/fluent/plugin/output.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/error'
 require 'fluent/plugin/base'
 require 'fluent/plugin/buffer'
@@ -37,7 +38,7 @@ module Fluent
       include PluginHelper::Mixin
       include UniqueId::Mixin
 
-      helpers_internal :thread, :retry_state
+      helpers_internal :thread, :retry_state, :metrics
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +165,6 @@ module Fluent
       end
 
       attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
-      attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +172,30 @@ module Fluent
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def write_count
+        @write_count_metrics.get
+      end
+
+      def rollback_count
+        @rollback_count_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
@@ -181,13 +205,15 @@ module Fluent
         @primary_instance = nil
 
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
-        @write_count = 0
-        @rollback_count = 0
-        @flush_time_count = 0
-        @slow_flush_count = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @write_count_metrics = nil
+        @rollback_count_metrics = nil
+        @flush_time_count_metrics = nil
+        @slow_flush_count_metrics = nil
+        @enable_size_metrics = false
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
         if implement?(:synchronous)
@@ -246,6 +272,15 @@ module Fluent
 
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
+        @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
+        @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
+        @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
+        @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
+
         if has_buffer_section
           unless implement?(:buffered) || implement?(:delayed_commit)
             raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +306,8 @@ module Fluent
             @buffering = true
           end
         end
+        # Enable to update record size metrics or not
+        @enable_size_metrics = !!system_config.enable_size_metrics
 
         if @as_secondary
           if !@buffering && !@buffering.nil?
@@ -797,18 +834,19 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
 
       def emit_buffered(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
           if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +854,7 @@ module Fluent
           end
         rescue
           # TODO: separate number of errors into emit errors and write/flush errors
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
@@ -966,7 +1004,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -983,7 +1022,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1008,7 +1048,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1046,7 +1087,7 @@ module Fluent
         #         false if chunk was already flushed and couldn't be rollbacked unexpectedly
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
-          @counter_mutex.synchronize{ @rollback_count += 1 }
+          @rollback_count_metrics.inc
           if update_retry
             primary = @as_secondary ? @primary_instance : self
             primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1103,7 @@ module Fluent
           while @dequeued_chunks.first && @dequeued_chunks.first.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1118,7 @@ module Fluent
           until @dequeued_chunks.empty?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1161,7 @@ module Fluent
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1173,7 @@ module Fluent
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1188,7 +1229,7 @@ module Fluent
           end
 
           if @buffer.takeback_chunk(chunk.unique_id)
-            @counter_mutex.synchronize { @rollback_count += 1 }
+            @rollback_count_metrics.inc
           end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1208,8 +1249,8 @@ module Fluent
           backup_dir = File.dirname(backup_file)
 
           log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
-          File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+          File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
             chunk.write_to(f)
           }
         end
@@ -1219,9 +1260,9 @@ module Fluent
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         elapsed_millsec = (elapsed_time * 1000).to_i
-        @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
+        @flush_time_count_metrics.add(elapsed_millsec)
         if elapsed_time > @slow_flush_log_threshold
-          @counter_mutex.synchronize { @slow_flush_count += 1 }
+          @slow_flush_count_metrics.inc
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1229,13 +1270,13 @@ module Fluent
 
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           chunk_id_hex = dump_unique_id_hex(chunk_id)
 
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
             if error
-              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
               log.warn_backtrace error.backtrace
             end
             return
@@ -1264,11 +1305,11 @@ module Fluent
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               else
                 msg = "failed to flush the buffer."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               end
             end
@@ -1490,15 +1531,16 @@ module Fluent
 
       def statistics
         stats = {
-          'emit_records' => @emit_records,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
           # Respect original name
           # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
-          'retry_count' => @num_errors,
-          'emit_count' => @emit_count,
-          'write_count' => @write_count,
-          'rollback_count' => @rollback_count,
-          'slow_flush_count' => @slow_flush_count,
-          'flush_time_count' => @flush_time_count,
+          'retry_count' => @num_errors_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'write_count' => @write_count_metrics.get,
+          'rollback_count' => @rollback_count_metrics.get,
+          'slow_flush_count' => @slow_flush_count_metrics.get,
+          'flush_time_count' => @flush_time_count_metrics.get,
         }
 
         if @buffer && @buffer.respond_to?(:statistics)

data/lib/fluent/plugin/parser_apache2.rb

@@ -21,7 +21,7 @@ module Fluent
     class Apache2Parser < Parser
       Plugin.register_parser('apache2', self)
 
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize

data/lib/fluent/plugin/storage_local.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
 
@@ -25,14 +26,11 @@ module Fluent
     class LocalStorage < Storage
       Fluent::Plugin.register_storage('local', self)
 
-      DEFAULT_DIR_MODE = 0755
-      DEFAULT_FILE_MODE = 0644
-
       config_param :path, :string, default: nil
-      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+      config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
         v.to_i(8)
       end
-      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+      config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
         v.to_i(8)
       end
       config_param :pretty_print, :bool, default: false

data/lib/fluent/plugin.rb

@@ -36,8 +36,9 @@ module Fluent
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
     SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
+    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
       register_impl('sd', SD_REGISTRY, type, klass)
     end
 
+    def self.register_metrics(type, klass)
+      register_impl('metrics', METRICS_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    def self.new_metrics(type, parent: nil)
+      new_impl('metrics', METRICS_REGISTRY, type, parent)
+    end
+
     class << self
       # This should be defined for fluent-plugin-config-formatter type arguments.
       alias_method :new_service_discovery, :new_sd

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -29,6 +29,9 @@ module Fluent
         if @_event_emitter_lazy_init
           @router = @primary_instance.router
         end
+        if @router.respond_to?(:caller_plugin_id=)
+          @router.caller_plugin_id = self.plugin_id
+        end
         @router
       end
 
@@ -47,7 +50,11 @@ module Fluent
 
       def event_emitter_router(label_name)
         if label_name
-          Engine.root_agent.find_label(label_name).event_router
+          if label_name == "@ROOT"
+            Engine.root_agent.event_router
+          else
+            Engine.root_agent.find_label(label_name).event_router
+          end
         elsif self.respond_to?(:as_secondary) && self.as_secondary
           if @primary_instance.has_router?
             @_event_emitter_lazy_init = true

data/lib/fluent/plugin_helper/metrics.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module PluginHelper
+    module Metrics
+      include Fluent::SystemConfig::Mixin
+
+      attr_reader :_metrics # For tests.
+
+      def initialize
+        super
+        @_metrics_started = false
+        @_metrics = {} # usage => metrics_state
+      end
+
+      def configure(conf)
+        super
+
+        @plugin_type_or_id = if self.plugin_id_configured?
+                               self.plugin_id
+                             else
+                               if type = (conf["@type"] || conf["type"])
+                                 "#{type}.#{self.plugin_id}"
+                               else
+                                 "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
+                               end
+                             end
+      end
+
+      def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
+        metrics = if system_config.metrics
+                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
+                  else
+                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
+                  end
+        config = if system_config.metrics
+                   system_config.metrics.corresponding_config_element
+                 else
+                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
+                 end
+        metrics.use_gauge_metric = prefer_gauge
+        metrics.configure(config)
+        # For multi workers environment, cmetrics should be distinguish with static labels.
+        if Fluent::Engine.system_config.workers > 1
+          labels.merge!(worker_id: fluentd_worker_id.to_s)
+        end
+        labels.merge!(plugin: @plugin_type_or_id)
+        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
+
+        @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
+
+        metrics
+      end
+
+      def metrics_operate(method_name, &block)
+        @_metrics.each_pair do |key, m|
+          begin
+            block.call(s) if block_given?
+            m.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
+          end
+        end
+      end
+
+      def start
+        super
+
+        metrics_operate(:start)
+        @_metrics_started = true
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        metrics_operate(:stop)
+      end
+
+      def before_shutdown
+        metrics_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        metrics_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        metrics_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        metrics_operate(:close)
+        super
+      end
+
+      def terminate
+        metrics_operate(:terminate)
+        @_metrics = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/server.rb

@@ -709,13 +709,15 @@ module Fluent
                 return true
               end
             rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               @log.trace "unexpected error before accepting TLS connection",
-                         host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                         addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             rescue OpenSSL::SSL::SSLError => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               # Use same log level as on_readable
               @log.warn "unexpected error before accepting TLS connection by OpenSSL",
-                        host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                        addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             end
 

data/lib/fluent/plugin_helper.rb

@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
 require 'fluent/plugin_helper/service_discovery'
+require 'fluent/plugin_helper/metrics'
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_id.rb

@@ -15,6 +15,7 @@
 #
 
 require 'set'
+require 'fluent/env'
 require 'fluent/variable_store'
 
 module Fluent
@@ -76,7 +77,7 @@ module Fluent
 
       # Fluent::Plugin::Base#fluentd_worker_id
       dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
-      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
+      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
       @_plugin_root_dir = dir.freeze
       dir
     end

data/lib/fluent/root_agent.rb

@@ -55,9 +55,11 @@ module Fluent
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
       @without_source = false
+      @enable_input_metrics = false
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
+      @enable_input_metrics = !!system_config.enable_input_metrics
     end
 
     attr_reader :inputs
@@ -131,6 +133,7 @@ module Fluent
         end
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
+        raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
 
         if name == ERROR_LABEL
           error_label_config = e
@@ -315,6 +318,9 @@ module Fluent
       # See also 'fluentd/plugin/input.rb'
       input.context_router = @event_router
       input.configure(conf)
+      if @enable_input_metrics
+        @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
+      end
       @inputs << input
 
       input

data/lib/fluent/supervisor.rb

@@ -544,7 +544,7 @@ module Fluent
         $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
-          File.chmod(log_dir_perm || 0755, File.dirname(@path))
+          File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
         end
       end
 
@@ -570,6 +570,8 @@ module Fluent
         suppress_repeated_stacktrace: true,
         ignore_repeated_log_interval: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
         supervise: true,
@@ -649,7 +651,7 @@ module Fluent
           end
         else
           begin
-            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
+            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
           rescue => e
             raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end

data/lib/fluent/system_config.rb

@@ -27,7 +27,8 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
+      :metrics, :enable_input_metrics, :enable_size_metrics
     ]
 
     config_param :workers,   :integer, default: 1
@@ -46,6 +47,8 @@ module Fluent
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_size_metrics, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -93,6 +96,11 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
+    config_section :metrics, multi: false do
+      config_param :@type, :string, default: "local"
+      config_param :labels, :hash, default: {}
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?