fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -0,0 +1,99 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/element'
+
+module Fluent
+  module PluginHelper
+    module CompatParameters
+      # This plugin helper is to bring old-fashioned buffer/other
+      # configuration parameters to v0.14 plugin API configurations.
+      # This helper is mainly to convert plugins from v0.12 API
+      # to v0.14 API safely, without breaking user deployment.
+
+      PARAMS_MAP = {
+        "buffer_type" => "@type",
+        "buffer_path" => "path",
+        "num_threads"                 => "flush_thread_count",
+        "flush_interval"              => "flush_interval",
+        "try_flush_interval"          => "flush_thread_interval",
+        "queued_chunk_flush_interval" => "flush_thread_burst_interval",
+        "disable_retry_limit" => "retry_forever",
+        "retry_limit"         => "retry_max_times",
+        "max_retry_wait"      => "retry_max_interval",
+        "buffer_chunk_limit"  => "chunk_limit_size",
+        "buffer_queue_limit"  => "queue_length_limit",
+        "buffer_queue_full_action" => "overflow_action",
+        "flush_at_shutdown" => "flush_at_shutdown",
+      }
+
+      TIME_SLICED_PARAMS = {
+        "time_slice_format" => nil,
+        "time_slice_wait" => "timekey_wait",
+      }
+
+      def compat_parameters_default_chunk_key
+        # '', 'time' or 'tag'
+        raise NotImplementedError, "return one of '', 'time' or 'tag'"
+      end
+
+      def configure(conf)
+        if conf.elements('buffer').empty? && PARAMS_MAP.keys.any?{|k| conf.has_key?(k) } || TIME_SLICED_PARAMS.keys.any?{|k| conf.has_key?(k) }
+          # TODO: warn obsolete parameters if these are deprecated
+          attr = {}
+          PARAMS_MAP.each do |compat, current|
+            next unless current
+            if conf.has_key?(compat)
+              if compat == 'buffer_queue_full_action' && conf[compat] == 'exception'
+                attr[current] = 'throw_exception'
+              else
+                attr[current] = conf[compat]
+              end
+            end
+          end
+          TIME_SLICED_PARAMS.each do |compat, current|
+            next unless current
+            attr[current] = conf[compat] if conf.has_key?(compat)
+          end
+
+          chunk_key = nil
+
+          if conf.has_key?('time_slice_format')
+            chunk_key = 'time'
+            attr['timekey'] = case conf['time_slice_format']
+                              when /\%S/ then 1
+                              when /\%M/ then 60
+                              when /\%H/ then 3600
+                              when /\%d/ then 86400
+                              else
+                                raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
+                              end
+          else
+            chunk_key = compat_parameters_default_chunk_key
+            if chunk_key == 'time'
+              attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
+            end
+          end
+
+          e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
+          conf.elements << e
+        end
+
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/time'
+
+module Fluent
+  module PluginHelper
+    module EventEmitter
+      # stop     : [-]
+      # shutdown : disable @router
+      # close    : [-]
+      # terminate: [-]
+
+      def router
+        @_event_emitter_used_actually = true
+        @router
+      end
+
+      def router=(r)
+        @router = r
+      end
+
+      def has_router?
+        true
+      end
+
+      def event_emitter_used_actually?
+        @_event_emitter_used_actually
+      end
+
+      def event_emitter_router(label_name)
+        if label_name
+          Engine.root_agent.find_label(label_name).event_router
+        else
+          Engine.root_agent.event_router
+        end
+      end
+
+      def initialize
+        super
+        @_event_emitter_used_actually = false
+        @router = nil
+      end
+
+      def configure(conf)
+        require 'fluent/engine'
+        super
+        @router = event_emitter_router(conf['@label'])
+      end
+
+      def after_shutdown
+        @router = nil
+        super
+      end
+
+      def close # unset router many times to reduce test cost
+        @router = nil
+        super
+      end
+
+      def terminate
+        @router = nil
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/event_loop.rb

@@ -0,0 +1,118 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/plugin_helper/thread'
+
+module Fluent
+  module PluginHelper
+    module EventLoop
+      # Currently this plugin helper is only for other helpers, not plugins.
+      # there's no way to create customized watchers to attach event loops.
+      include Fluent::PluginHelper::Thread
+
+      # stop     : [-]
+      # shutdown : detach all event watchers on event loop
+      # close    : stop event loop
+      # terminate: initialize internal state
+
+      EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
+
+      attr_reader :_event_loop # for tests
+
+      def event_loop_attach(watcher)
+        @_event_loop_mutex.synchronize do
+          @_event_loop.attach(watcher)
+        end
+      end
+
+      def event_loop_wait_until_start
+        sleep(0.1) until event_loop_running?
+      end
+
+      def event_loop_wait_until_stop
+        sleep(0.1) while event_loop_running?
+      end
+
+      def event_loop_running?
+        @_event_loop_running
+      end
+
+      def initialize
+        super
+        @_event_loop = Coolio::Loop.new
+        @_event_loop_running = false
+        @_event_loop_mutex = Mutex.new
+        # plugin MAY configure loop run timeout in #configure
+        @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
+      end
+
+      def start
+        super
+
+        # event loop does not run here, so mutex lock is not required
+        thread_create :event_loop do
+          default_watcher = DefaultWatcher.new
+          @_event_loop.attach(default_watcher)
+          @_event_loop_running = true
+          @_event_loop.run(@_event_loop_run_timeout) # this method blocks
+          @_event_loop_running = false
+        end
+      end
+
+      def shutdown
+        @_event_loop_mutex.synchronize do
+          @_event_loop.watchers.each {|w| w.detach if w.attached? }
+        end
+        while @_event_loop_running
+          sleep 0.1
+        end
+
+        super
+      end
+
+      def close
+        if @_event_loop_running
+          begin
+            @_event_loop.stop # we cannot check loop is running or not
+          rescue RuntimeError => e
+            raise unless e.message == 'loop not running'
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        @_event_loop = nil
+        @_event_loop_running = false
+        @_event_loop_mutex = nil
+        @_event_loop_run_timeout = nil
+
+        super
+      end
+
+      # watcher to block to run event loop until shutdown
+      class DefaultWatcher < Coolio::TimerWatcher
+        def initialize
+          super(1, true) # interval: 1, repeat: true
+        end
+        # do nothing
+        def on_timer; end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/retry_state.rb

@@ -0,0 +1,177 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module RetryState
+      def retry_state_create(
+          title, retry_type, wait, timeout,
+          forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,
+          secondary: false, secondary_threshold: 0.8
+      )
+        case retry_type
+        when :exponential_backoff
+          ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
+        when :periodic
+          PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+        else
+          raise "BUG: unknown retry_type specified: '#{retry_type}'"
+        end
+      end
+
+      class RetryStateMachine
+        attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps
+
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+          @title = title
+
+          @start = current_time
+          @steps = 0
+          @next_time = nil # should be initialized for first retry by child class
+
+          @timeout = timeout
+          @timeout_at = @start + timeout
+          @current = :primary
+
+          if randomize_width < 0 || randomize_width > 0.5
+            raise "BUG: randomize_width MUST be between 0 and 0.5"
+          end
+
+          @randomize = randomize
+          @randomize_width = randomize_width
+
+          if forever && secondary
+            raise "BUG: forever and secondary are exclusive to each other"
+          end
+
+          @forever = forever
+          @max_steps = max_steps
+
+          @secondary = secondary
+          @secondary_threshold = secondary_threshold
+          if @secondary
+            raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
+            @secondary_transition_at = @start + timeout * @secondary_threshold
+            @secondary_transition_steps = nil
+          end
+        end
+
+        def current_time
+          Time.now
+        end
+
+        def randomize(interval)
+          return interval unless @randomize
+
+          interval + (interval * @randomize_width * (2 * rand - 1.0))
+        end
+
+        def calc_next_time
+          if @forever || !@secondary # primary
+            naive = naive_next_time(@steps)
+            if @forever
+              naive
+            elsif naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          elsif @current == :primary && @secondary
+            naive = naive_next_time(@steps)
+            if naive >= @secondary_transition_at
+              @secondary_transition_at
+            else
+              naive
+            end
+          elsif @current == :secondary
+            naive = naive_next_time(@steps - @secondary_transition_steps + 1)
+            if naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          else
+            raise "BUG: it's out of design"
+          end
+        end
+
+        def naive_next_time(retry_times)
+          raise NotImplementedError
+        end
+
+        def secondary?
+          @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
+        end
+
+        def step
+          @steps += 1
+          if @secondary && @current != :secondary && current_time >= @secondary_transition_at
+            @current = :secondary
+            @secondary_transition_steps = @steps
+          end
+          @next_time = calc_next_time
+          nil
+        end
+
+        def limit?
+          if @forever
+            false
+          else
+            @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
+          end
+        end
+      end
+
+      class ExponentialBackOffRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold)
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
+          @constant_factor = wait
+          @backoff_base = backoff_base
+          @max_interval = max_interval
+
+          @next_time = @start + @constant_factor
+        end
+
+        def naive_next_time(retry_next_times)
+          # make it infinite if calculated "interval" is too big
+          interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
+          intr = if interval.finite?
+                   if @max_interval && interval > @max_interval
+                     @max_interval
+                   else
+                     interval
+                   end
+                 else
+                   interval
+                 end
+          current_time + randomize(intr)
+        end
+      end
+
+      class PeriodicRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
+          @retry_wait = wait
+          @next_time = @start + @retry_wait
+        end
+
+        def naive_next_time(retry_next_times)
+          current_time + randomize(@retry_wait)
+        end
+      end
+    end
+  end
+end