dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin/metrics_local.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+
+module Fluent
+  module Plugin
+    class LocalMetrics < Metrics
+      Fluent::Plugin.register_metrics('local', self)
+
+      def initialize
+        super
+        @store = 0
+        @monitor = Monitor.new
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          class << self
+            alias_method :dec, :dec_gauge
+            alias_method :set, :set_gauge
+            alias_method :sub, :sub_gauge
+          end
+        else
+          class << self
+            alias_method :set, :set_counter
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get
+        @monitor.synchronize do
+          @store
+        end
+      end
+
+      def inc
+        @monitor.synchronize do
+          @store += 1
+        end
+      end
+
+      def dec_gauge
+        @monitor.synchronize do
+          @store -= 1
+        end
+      end
+
+      def add(value)
+        @monitor.synchronize do
+          @store += value
+        end
+      end
+
+      def sub_gauge(value)
+        @monitor.synchronize do
+          @store -= value
+        end
+      end
+
+      def set_counter(value)
+        return if @store > value
+
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+
+      def set_gauge(value)
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -0,0 +1,195 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class MultiOutput < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin # for event_emitter
+
+      helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
+
+      config_section :store, param_name: :stores, multi: true, required: true do
+        config_argument :arg, :string, default: ''
+        config_param :@type, :string, default: nil
+      end
+
+      attr_reader :outputs, :outputs_statically_created
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @outputs = []
+        @outputs_statically_created = false
+
+        @counter_mutex = Mutex.new
+        # TODO: well organized counters
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        # @write_count = 0
+        # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
+      end
+
+      def multi_output?
+        true
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
+        @stores.each do |store|
+          store_conf = store.corresponding_config_element
+          type = store_conf['@type']
+          unless type
+            raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
+          end
+
+          log.debug "adding store", type: type
+
+          output = Fluent::Plugin.new_output(type)
+          output.context_router = self.context_router
+          output.configure(store_conf)
+          @outputs << output
+        end
+      end
+
+      def static_outputs
+        @outputs_statically_created = true
+        @outputs
+      end
+
+      # Child plugin's lifecycles are controlled by agent automatically.
+      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
+      # * `start` of this plugin will be called after child plugins
+      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
+
+      # But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find
+      # sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.
+      # TODO: this hack will be removed at v2.
+      def call_lifecycle_method(method_name, checker_name)
+        return if @outputs_statically_created
+        @outputs.each do |o|
+          begin
+            log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
+            o.__send__(method_name) unless o.__send__(checker_name)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
+            log.warn_backtrace
+          end
+        end
+      end
+
+      def start
+        super
+        call_lifecycle_method(:start, :started?)
+      end
+
+      def after_start
+        super
+        call_lifecycle_method(:after_start, :after_started?)
+      end
+
+      def stop
+        super
+        call_lifecycle_method(:stop, :stopped?)
+      end
+
+      def before_shutdown
+        super
+        call_lifecycle_method(:before_shutdown, :before_shutdown?)
+      end
+
+      def shutdown
+        super
+        call_lifecycle_method(:shutdown, :shutdown?)
+      end
+
+      def after_shutdown
+        super
+        call_lifecycle_method(:after_shutdown, :after_shutdown?)
+      end
+
+      def close
+        super
+        call_lifecycle_method(:close, :closed?)
+      end
+
+      def terminate
+        super
+        call_lifecycle_method(:terminate, :terminated?)
+      end
+
+      def emit_sync(tag, es)
+        @emit_count_metrics.inc
+        begin
+          process(tag, es)
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+        rescue
+          @num_errors_metrics.inc
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/out_copy.rb

@@ -0,0 +1,120 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/multi_output'
+require 'fluent/config/error'
+require 'fluent/event'
+
+module Fluent::Plugin
+  class CopyOutput < MultiOutput
+    Fluent::Plugin.register_output('copy', self)
+
+    desc 'If true, pass different record to each `store` plugin.'
+    config_param :deep_copy, :bool, default: false, deprecated: "use 'copy_mode' parameter instead"
+    desc 'Pass different record to each `store` plugin by specified method'
+    config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
+
+    attr_reader :ignore_errors, :ignore_if_prev_successes
+
+    def initialize
+      super
+      @ignore_errors = []
+      @ignore_if_prev_successes = []
+    end
+
+    def configure(conf)
+      super
+
+      @copy_proc = gen_copy_proc
+      @stores.each_with_index { |store, i|
+        if i == 0 && store.arg.include?('ignore_if_prev_success')
+          raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
+        end
+        @ignore_errors << (store.arg.include?('ignore_error'))
+        @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
+      }
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
+        log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
+      end
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
+      unless es.repeatable?
+        m = Fluent::MultiEventStream.new
+        es.each {|time,record|
+          m.add(time, record)
+        }
+        es = m
+      end
+      success = Array.new(outputs.size)
+      outputs.each_with_index do |output, i|
+        begin
+          if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
+            log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
+          else
+            output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
+            success[i] = true
+          end
+        rescue => e
+          if @ignore_errors[i]
+            log.error "ignore emit error in #{output.plugin_id}", error: e
+          else
+            raise e
+          end
+        end
+      end
+    end
+
+    private
+
+    def gen_copy_proc
+      @copy_mode = :shallow if @deep_copy
+
+      case @copy_mode
+      when :no_copy
+         nil
+      when :shallow
+        Proc.new { |es| es.dup }
+      when :deep
+        Proc.new { |es|
+          packer = Fluent::MessagePackFactory.msgpack_packer
+          times = []
+          records = []
+          es.each { |time, record|
+            times << time
+            packer.pack(record)
+          }
+          Fluent::MessagePackFactory.msgpack_unpacker.feed_each(packer.full_pack) { |record|
+            records << record
+          }
+          Fluent::MultiEventStream.new(times, records)
+        }
+      when :marshal
+        Proc.new { |es|
+          new_es = Fluent::MultiEventStream.new
+          es.each { |time, record|
+            new_es.add(time, Marshal.load(Marshal.dump(record)))
+          }
+          new_es
+        }
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_exec.rb

@@ -0,0 +1,105 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'tempfile'
+
+require 'fluent/plugin/output'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  class ExecOutput < Output
+    Fluent::Plugin.register_output('exec', self)
+
+    helpers :inject, :formatter, :compat_parameters, :child_process
+
+    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argument'
+    config_param :command, :string
+
+    config_param :command_timeout, :time, default: 270 # 4min 30sec
+
+    config_section :format do
+      config_set_default :@type, 'tsv'
+    end
+
+    config_section :inject do
+      config_set_default :time_type, :string
+      config_set_default :localtime, false
+    end
+
+    config_section :buffer do
+      config_set_default :delayed_commit_timeout, 300 # 5 min
+    end
+
+    attr_reader :formatter # for tests
+
+    def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter, :buffer, default_chunk_key: 'time')
+      super
+      @formatter = formatter_create
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    NEWLINE = "\n"
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      if @formatter.formatter_type == :text_per_line
+        @formatter.format(tag, time, record).chomp + NEWLINE
+      else
+        @formatter.format(tag, time, record)
+      end
+    end
+
+    def try_write(chunk)
+      tmpfile = nil
+      prog = if chunk.respond_to?(:path)
+               "#{@command} #{chunk.path}"
+             else
+               tmpfile = Tempfile.new("fluent-plugin-out-exec-")
+               tmpfile.binmode
+               chunk.write_to(tmpfile)
+               tmpfile.close
+               "#{@command} #{tmpfile.path}"
+             end
+      chunk_id = chunk.unique_id
+      callback = ->(status){
+        begin
+          if tmpfile
+            tmpfile.delete rescue nil
+          end
+          if status && status.success?
+            commit_write(chunk_id)
+          elsif status
+            # #rollback_write will be done automatically if it isn't called at here.
+            # But it's after command_timeout, and this timeout should be longer than users expectation.
+            # So here, this plugin calls it explicitly.
+            rollback_write(chunk_id)
+            log.warn "command exits with error code", prog: prog, status: status.exitstatus, signal: status.termsig
+          else
+            rollback_write(chunk_id)
+            log.warn "command unexpectedly exits without exit status", prog: prog
+          end
+        rescue => e
+          log.error "unexpected error in child process callback", error: e
+        end
+      }
+      child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: @command_timeout, on_exit_callback: callback)
+    end
+  end
+end