fluentd 1.13.0-x86-mingw32 → 1.14.0-x86-mingw32

data/lib/fluent/plugin/input.rb

@@ -27,7 +27,45 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers_internal :event_emitter
+      helpers_internal :event_emitter, :metrics
+
+      def initialize
+        super
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @counter_mutex = Mutex.new
+        @enable_size_metrics = false
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def configure(conf)
+        super
+
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
+        @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'input' => stats }
+      end
+
+      def metric_callback(es)
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+      end
 
       def multi_workers_ready?
         false

data/lib/fluent/plugin/metrics.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class Metrics < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :metrics
+
+      config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
+      config_param :labels, :hash, default: {}
+
+      attr_reader :use_gauge_metric
+      attr_reader :has_methods_for_gauge, :has_methods_for_counter
+
+      def initialize
+        super
+
+        @has_methods_for_counter = false
+        @has_methods_for_gauge = false
+        @use_gauge_metric = false
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          @has_methods_for_gauge = has_methods_for_gauge?
+        else
+          @has_methods_for_counter = has_methods_for_counter?
+        end
+      end
+
+      # Some metrics should be counted by gauge.
+      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
+      def use_gauge_metric=(use_gauge_metric=false)
+        @use_gauge_metric = use_gauge_metric
+      end
+
+      def create(namespace:, subsystem:,name:,help_text:,labels: {})
+        # This API is for cmetrics type.
+      end
+
+      def get
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def inc
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def dec
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def add(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def sub(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def set(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      private
+
+      def has_methods_for_counter?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
+        end
+      end
+
+      def has_methods_for_gauge?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/metrics_local.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+
+module Fluent
+  module Plugin
+    class LocalMetrics < Metrics
+      Fluent::Plugin.register_metrics('local', self)
+
+      def initialize
+        super
+        @store = 0
+        @monitor = Monitor.new
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          class << self
+            alias_method :dec, :dec_gauge
+            alias_method :set, :set_gauge
+            alias_method :sub, :sub_gauge
+          end
+        else
+          class << self
+            alias_method :set, :set_counter
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get
+        @monitor.synchronize do
+          @store
+        end
+      end
+
+      def inc
+        @monitor.synchronize do
+          @store += 1
+        end
+      end
+
+      def dec_gauge
+        @monitor.synchronize do
+          @store -= 1
+        end
+      end
+
+      def add(value)
+        @monitor.synchronize do
+          @store += value
+        end
+      end
+
+      def sub_gauge(value)
+        @monitor.synchronize do
+          @store -= value
+        end
+      end
+
+      def set_counter(value)
+        return if @store > value
+
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+
+      def set_gauge(value)
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -27,6 +27,7 @@ module Fluent
       include PluginHelper::Mixin # for event_emitter
 
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
 
       config_section :store, param_name: :stores, multi: true, required: true do
         config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
 
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
         # @write_count = 0
         # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
       end
 
       def multi_output?
@@ -60,6 +90,12 @@ module Fluent
       def configure(conf)
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
         @stores.each do |store|
           store_conf = store.corresponding_config_element
           type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/out_forward.rb

@@ -565,8 +565,6 @@ module Fluent::Plugin
           username: server.username || '',
         )
 
-        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
-
         @resolved_host = nil
         @resolved_time = 0
         @resolved_once = false
@@ -613,7 +611,7 @@ module Fluent::Plugin
               sleep @sender.read_interval
               next
             end
-            @unpacker.feed_each(buf) do |data|
+            Fluent::MessagePackFactory.msgpack_unpacker.feed_each(buf) do |data|
               if @handshake.invoke(sock, ri, data) == :established
                 @log.debug "connection established", host: @host, port: @port
               end

data/lib/fluent/plugin/output.rb

@@ -37,7 +37,7 @@ module Fluent
       include PluginHelper::Mixin
       include UniqueId::Mixin
 
-      helpers_internal :thread, :retry_state
+      helpers_internal :thread, :retry_state, :metrics
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
@@ -164,7 +164,6 @@ module Fluent
       end
 
       attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
-      attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
@@ -172,6 +171,30 @@ module Fluent
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def write_count
+        @write_count_metrics.get
+      end
+
+      def rollback_count
+        @rollback_count_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
@@ -181,13 +204,15 @@ module Fluent
         @primary_instance = nil
 
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
-        @write_count = 0
-        @rollback_count = 0
-        @flush_time_count = 0
-        @slow_flush_count = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @write_count_metrics = nil
+        @rollback_count_metrics = nil
+        @flush_time_count_metrics = nil
+        @slow_flush_count_metrics = nil
+        @enable_size_metrics = false
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
         if implement?(:synchronous)
@@ -246,6 +271,15 @@ module Fluent
 
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
+        @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
+        @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
+        @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
+        @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
+
         if has_buffer_section
           unless implement?(:buffered) || implement?(:delayed_commit)
             raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
@@ -271,6 +305,8 @@ module Fluent
             @buffering = true
           end
         end
+        # Enable to update record size metrics or not
+        @enable_size_metrics = !!system_config.enable_size_metrics
 
         if @as_secondary
           if !@buffering && !@buffering.nil?
@@ -797,18 +833,19 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
 
       def emit_buffered(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
           if !@retry && @buffer.queued?(nil, optimistic: true)
@@ -816,7 +853,7 @@ module Fluent
           end
         rescue
           # TODO: separate number of errors into emit errors and write/flush errors
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end
@@ -966,7 +1003,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -983,7 +1021,8 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1008,7 +1047,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += records }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         true
       end
 
@@ -1046,7 +1086,7 @@ module Fluent
         #         false if chunk was already flushed and couldn't be rollbacked unexpectedly
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
-          @counter_mutex.synchronize{ @rollback_count += 1 }
+          @rollback_count_metrics.inc
           if update_retry
             primary = @as_secondary ? @primary_instance : self
             primary.update_retry_state(chunk_id, @as_secondary)
@@ -1062,7 +1102,7 @@ module Fluent
           while @dequeued_chunks.first && @dequeued_chunks.first.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1077,7 +1117,7 @@ module Fluent
           until @dequeued_chunks.empty?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counter_mutex.synchronize{ @rollback_count += 1 }
+              @rollback_count_metrics.inc
               log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1120,7 +1160,7 @@ module Fluent
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1132,7 +1172,7 @@ module Fluent
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
-            @counter_mutex.synchronize{ @write_count += 1 }
+            @write_count_metrics.inc
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1188,7 +1228,7 @@ module Fluent
           end
 
           if @buffer.takeback_chunk(chunk.unique_id)
-            @counter_mutex.synchronize { @rollback_count += 1 }
+            @rollback_count_metrics.inc
           end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1219,9 +1259,9 @@ module Fluent
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         elapsed_millsec = (elapsed_time * 1000).to_i
-        @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
+        @flush_time_count_metrics.add(elapsed_millsec)
         if elapsed_time > @slow_flush_log_threshold
-          @counter_mutex.synchronize { @slow_flush_count += 1 }
+          @slow_flush_count_metrics.inc
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1229,7 +1269,7 @@ module Fluent
 
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           chunk_id_hex = dump_unique_id_hex(chunk_id)
 
           unless @retry
@@ -1490,15 +1530,16 @@ module Fluent
 
       def statistics
         stats = {
-          'emit_records' => @emit_records,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
           # Respect original name
           # https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
-          'retry_count' => @num_errors,
-          'emit_count' => @emit_count,
-          'write_count' => @write_count,
-          'rollback_count' => @rollback_count,
-          'slow_flush_count' => @slow_flush_count,
-          'flush_time_count' => @flush_time_count,
+          'retry_count' => @num_errors_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'write_count' => @write_count_metrics.get,
+          'rollback_count' => @rollback_count_metrics.get,
+          'slow_flush_count' => @slow_flush_count_metrics.get,
+          'flush_time_count' => @flush_time_count_metrics.get,
         }
 
         if @buffer && @buffer.respond_to?(:statistics)