fluentd 1.13.3-x64-mingw32 → 1.14.0-x64-mingw32

data/lib/fluent/plugin/multi_output.rb

@@ -27,6 +27,7 @@ module Fluent
       include PluginHelper::Mixin # for event_emitter
 
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
 
       config_section :store, param_name: :stores, multi: true, required: true do
         config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
 
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
         # @write_count = 0
         # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
       end
 
       def multi_output?
@@ -60,6 +90,12 @@ module Fluent
       def configure(conf)
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
         @stores.each do |store|
           store_conf = store.corresponding_config_element
           type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/output.rb

@@ -37,7 +37,7 @@ module Fluent
       include PluginHelper::Mixin
       include UniqueId::Mixin
 
-      helpers_internal :thread, :retry_state
+      helpers_internal :thread, :retry_state, :metrics
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +164,6 @@ module Fluent
       end
 
       attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
-      attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +171,30 @@ module Fluent
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def write_count
+        @write_count_metrics.get
+      end
+
+      def rollback_count
+        @rollback_count_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
@@ -181,13 +204,15 @@ module Fluent
         @primary_instance = nil
 
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
-        @write_count = 0
-        @rollback_count = 0
-        @flush_time_count = 0
-        @slow_flush_count = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @write_count_metrics = nil
+        @rollback_count_metrics = nil
+        @flush_time_count_metrics = nil
+        @slow_flush_count_metrics = nil
+        @enable_size_metrics = false
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
         if implement?(:synchronous)
@@ -246,6 +271,15 @@ module Fluent
 
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
+        @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
+        @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
+        @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
+        @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
+
         if has_buffer_section
           unless implement?(:buffered) || implement?(:delayed_commit)
             raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +305,8 @@ module Fluent
             @buffering = true
           end
         end
+        # Enable to update record size metrics or not
+        @enable_size_metrics = !!system_config.enable_size_metrics
 
         if @as_secondary
           if !@buffering && !@buffering.nil?
@@ -797,18 +833,19 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
 
       def emit_buffered(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
           if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +853,7 @@ module Fluent
           end
         rescue
           # TODO: separate number of errors into emit errors and write/flush errors
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
@@ -966,7 +1003,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -983,7 +1021,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1008,7 +1047,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1046,7 +1086,7 @@ module Fluent
         #         false if chunk was already flushed and couldn't be rollbacked unexpectedly
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
-          @counter_mutex.synchronize{ @rollback_count += 1 }
+          @rollback_count_metrics.inc
           if update_retry
             primary = @as_secondary ? @primary_instance : self
             primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1102,7 @@ module Fluent
           while @dequeued_chunks.first && @dequeued_chunks.first.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1117,7 @@ module Fluent
           until @dequeued_chunks.empty?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1160,7 @@ module Fluent
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1172,7 @@ module Fluent
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1188,7 +1228,7 @@ module Fluent
           end
 
           if @buffer.takeback_chunk(chunk.unique_id)
-            @counter_mutex.synchronize { @rollback_count += 1 }
+            @rollback_count_metrics.inc
           end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1219,9 +1259,9 @@ module Fluent
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         elapsed_millsec = (elapsed_time * 1000).to_i
-        @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
+        @flush_time_count_metrics.add(elapsed_millsec)
         if elapsed_time > @slow_flush_log_threshold
-          @counter_mutex.synchronize { @slow_flush_count += 1 }
+          @slow_flush_count_metrics.inc
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1229,7 +1269,7 @@ module Fluent
 
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           chunk_id_hex = dump_unique_id_hex(chunk_id)
 
           unless @retry
@@ -1490,15 +1530,16 @@ module Fluent
 
       def statistics
         stats = {
-          'emit_records' => @emit_records,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
           # Respect original name
           # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
-          'retry_count' => @num_errors,
-          'emit_count' => @emit_count,
-          'write_count' => @write_count,
-          'rollback_count' => @rollback_count,
-          'slow_flush_count' => @slow_flush_count,
-          'flush_time_count' => @flush_time_count,
+          'retry_count' => @num_errors_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'write_count' => @write_count_metrics.get,
+          'rollback_count' => @rollback_count_metrics.get,
+          'slow_flush_count' => @slow_flush_count_metrics.get,
+          'flush_time_count' => @flush_time_count_metrics.get,
         }
 
         if @buffer && @buffer.respond_to?(:statistics)

data/lib/fluent/plugin.rb

@@ -36,8 +36,9 @@ module Fluent
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
     SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
+    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
       register_impl('sd', SD_REGISTRY, type, klass)
     end
 
+    def self.register_metrics(type, klass)
+      register_impl('metrics', METRICS_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    def self.new_metrics(type, parent: nil)
+      new_impl('metrics', METRICS_REGISTRY, type, parent)
+    end
+
     class << self
       # This should be defined for fluent-plugin-config-formatter type arguments.
       alias_method :new_service_discovery, :new_sd

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -29,6 +29,9 @@ module Fluent
         if @_event_emitter_lazy_init
           @router = @primary_instance.router
         end
+        if @router.respond_to?(:caller_plugin_id=)
+          @router.caller_plugin_id = self.plugin_id
+        end
         @router
       end
 
@@ -47,7 +50,11 @@ module Fluent
 
       def event_emitter_router(label_name)
         if label_name
-          Engine.root_agent.find_label(label_name).event_router
+          if label_name == "@ROOT"
+            Engine.root_agent.event_router
+          else
+            Engine.root_agent.find_label(label_name).event_router
+          end
         elsif self.respond_to?(:as_secondary) && self.as_secondary
           if @primary_instance.has_router?
             @_event_emitter_lazy_init = true

data/lib/fluent/plugin_helper/metrics.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module PluginHelper
+    module Metrics
+      include Fluent::SystemConfig::Mixin
+
+      attr_reader :_metrics # For tests.
+
+      def initialize
+        super
+        @_metrics_started = false
+        @_metrics = {} # usage => metrics_state
+      end
+
+      def configure(conf)
+        super
+
+        @plugin_type_or_id = if self.plugin_id_configured?
+                               self.plugin_id
+                             else
+                               if type = (conf["@type"] || conf["type"])
+                                 "#{type}.#{self.plugin_id}"
+                               else
+                                 "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
+                               end
+                             end
+      end
+
+      def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
+        metrics = if system_config.metrics
+                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
+                  else
+                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
+                  end
+        config = if system_config.metrics
+                   system_config.metrics.corresponding_config_element
+                 else
+                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
+                 end
+        metrics.use_gauge_metric = prefer_gauge
+        metrics.configure(config)
+        # For multi workers environment, cmetrics should be distinguish with static labels.
+        if Fluent::Engine.system_config.workers > 1
+          labels.merge!(worker_id: fluentd_worker_id.to_s)
+        end
+        labels.merge!(plugin: @plugin_type_or_id)
+        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
+
+        @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
+
+        metrics
+      end
+
+      def metrics_operate(method_name, &block)
+        @_metrics.each_pair do |key, m|
+          begin
+            block.call(s) if block_given?
+            m.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
+          end
+        end
+      end
+
+      def start
+        super
+
+        metrics_operate(:start)
+        @_metrics_started = true
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        metrics_operate(:stop)
+      end
+
+      def before_shutdown
+        metrics_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        metrics_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        metrics_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        metrics_operate(:close)
+        super
+      end
+
+      def terminate
+        metrics_operate(:terminate)
+        @_metrics = {}
+        super
+      end
+    end
+  end
+end