fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin_helper/parser.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Parser
+      def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
+        parser = @_parsers[usage]
+        return parser if parser && !type && !conf
+
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
+        parser = Fluent::Plugin.new_parser(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('parse', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('parse', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        parser.configure(config)
+        if @_parsers_started
+          parser.start
+        end
+
+        @_parsers[usage] = parser
+        parser
+      end
+
+      module ParserParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate parser plugin instances
+        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string # config_set_default required for :@type
+        end
+      end
+
+      def self.included(mod)
+        mod.include ParserParams
+      end
+
+      attr_reader :_parsers # for tests
+
+      def initialize
+        super
+        @_parsers_started = false
+        @_parsers = {} # usage => parser
+      end
+
+      def configure(conf)
+        super
+
+        @parser_configs.each do |section|
+          if @_parsers[section.usage]
+            raise Fluent::ConfigError, "duplicated parsers configured: #{section.usage}"
+          end
+          parser = Plugin.new_parser(section[:@type], parent: self)
+          parser.configure(section.corresponding_config_element)
+          @_parsers[section.usage] = parser
+        end
+      end
+
+      def start
+        super
+        @_parsers_started = true
+        @_parsers.each_pair do |usage, parser|
+          parser.start
+        end
+      end
+
+      def parser_operate(method_name, &block)
+        @_parsers.each_pair do |usage, parser|
+          begin
+            parser.send(method_name)
+            block.call(parser) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        parser_operate(:stop)
+      end
+
+      def before_shutdown
+        parser_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        parser_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        parser_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        parser_operate(:close)
+        super
+      end
+
+      def terminate
+        parser_operate(:terminate)
+        @_parsers_started = false
+        @_parsers = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -0,0 +1,210 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+unless {}.respond_to?(:dig)
+  begin
+    # backport_dig is faster than dig_rb so prefer backport_dig.
+    require 'backport_dig'
+  rescue LoadError
+    require 'dig_rb'
+  end
+end
+
+module Fluent
+  module PluginHelper
+    module RecordAccessor
+      def record_accessor_create(param)
+        Accessor.new(param)
+      end
+
+      class Accessor
+        attr_reader :keys
+
+        def initialize(param)
+          @keys = Accessor.parse_parameter(param)
+
+          if @keys.is_a?(Array)
+            @last_key = @keys.last
+            @dig_keys = @keys[0..-2]
+            if @dig_keys.empty?
+              @keys = @keys.first
+              mcall = method(:call_index)
+              mdelete = method(:delete_top)
+            else
+              mcall = method(:call_dig)
+              mdelete = method(:delete_nest)
+            end
+          else
+            # Call [] for single key to reduce dig overhead
+            mcall = method(:call_index)
+            mdelete = method(:delete_top)
+          end
+
+          singleton_class.module_eval do
+            define_method(:call, mcall)
+            define_method(:delete, mdelete)
+          end
+        end
+
+        def call(r)
+        end
+
+        # To optimize the performance, use class_eval with pre-expanding @keys
+        # See https://gist.github.com/repeatedly/ab553ed260cd080bd01ec71da9427312
+        def call_dig(r)
+          r.dig(*@keys)
+        end
+
+        def call_index(r)
+          r[@keys]
+        end
+
+        def delete(r)
+        end
+
+        def delete_nest(r)
+          if target = r.dig(*@dig_keys)
+            if target.is_a?(Array)
+              target.delete_at(@last_key)
+            else
+              target.delete(@last_key)
+            end
+          end
+        rescue
+          nil
+        end
+
+        def delete_top(r)
+          r.delete(@keys)
+        end
+
+        def self.parse_parameter(param)
+          if param.start_with?('$.')
+            parse_dot_notation(param)
+          elsif param.start_with?('$[')
+            parse_bracket_notation(param)
+          else
+            param
+          end
+        end
+
+        def self.parse_dot_notation(param)
+          result = []
+          keys = param[2..-1].split('.')
+          keys.each { |key|
+            if key.include?('[')
+              result.concat(parse_dot_array_op(key, param))
+            else
+              result << key
+            end
+          }
+
+          raise Fluent::ConfigError, "empty keys in dot notation" if result.empty?
+          validate_dot_keys(result)
+
+          result
+        end
+
+        def self.validate_dot_keys(keys)
+          keys.each { |key|
+            next unless key.is_a?(String)
+            if /\s+/.match(key)
+              raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
+            end
+          }
+        end
+
+        def self.parse_dot_array_op(key, param)
+          start = key.index('[')
+          result = if start.zero?
+                     []
+                   else
+                     [key[0..start - 1]]
+                   end
+          key = key[start + 1..-1]
+          in_bracket = true
+
+          until key.empty?
+            if in_bracket
+              if i = key.index(']')
+                index_value = key[0..i - 1]
+                raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                result << Integer(index_value)
+                key = key[i + 1..-1]
+                in_bracket = false
+              else
+                raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{param}"
+              end
+            else
+              if i = key.index('[')
+                key = key[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{param}"
+              end
+            end
+          end
+
+          result
+        end
+
+        def self.parse_bracket_notation(param)
+          orig_param = param
+          result = []
+          param = param[1..-1]
+          in_bracket = false
+
+          until param.empty?
+            if in_bracket
+              if param[0] == "'" || param[0] == '"'
+                if i = param.index("']") || param.index('"]')
+                  raise Fluent::ConfigError, "Mismatched quotes. Invalid syntax: #{orig_param}" unless param[0] == param[i]
+                  result << param[1..i - 1]
+                  param = param[i + 2..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "Incomplete bracket. Invalid syntax: #{orig_param}"
+                end
+              else
+                if i = param.index(']')
+                  index_value = param[0..i - 1]
+                  raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                  result << Integer(index_value)
+                  param = param[i + 1..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{orig_param}"
+                end
+              end
+            else
+              if i = param.index('[')
+                param = param[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{orig_param}"
+              end
+            end
+          end
+
+          raise Fluent::ConfigError, "empty keys in bracket notation" if result.empty?
+
+          result
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/retry_state.rb

@@ -0,0 +1,205 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module RetryState
+      def retry_state_create(
+          title, retry_type, wait, timeout,
+          forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,
+          secondary: false, secondary_threshold: 0.8
+      )
+        case retry_type
+        when :exponential_backoff
+          ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
+        when :periodic
+          PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+        else
+          raise "BUG: unknown retry_type specified: '#{retry_type}'"
+        end
+      end
+
+      class RetryStateMachine
+        attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps
+
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+          @title = title
+
+          @start = current_time
+          @steps = 0
+          @next_time = nil # should be initialized for first retry by child class
+
+          @timeout = timeout
+          @timeout_at = @start + timeout
+          @current = :primary
+
+          if randomize_width < 0 || randomize_width > 0.5
+            raise "BUG: randomize_width MUST be between 0 and 0.5"
+          end
+
+          @randomize = randomize
+          @randomize_width = randomize_width
+
+          @forever = forever
+          @max_steps = max_steps
+
+          @secondary = secondary
+          @secondary_threshold = secondary_threshold
+          if @secondary
+            raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
+            max_retry_timeout = timeout
+            if max_steps
+              timeout_by_max_steps = calc_max_retry_timeout(max_steps)
+              max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout
+            end
+            @secondary_transition_at = @start + max_retry_timeout * @secondary_threshold
+            @secondary_transition_steps = nil
+          end
+        end
+
+        def current_time
+          Time.now
+        end
+
+        def randomize(interval)
+          return interval unless @randomize
+
+          interval + (interval * @randomize_width * (2 * rand - 1.0))
+        end
+
+        def calc_next_time
+          if @forever || !@secondary # primary
+            naive = naive_next_time(@steps)
+            if @forever
+              naive
+            elsif naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          elsif @current == :primary && @secondary
+            naive = naive_next_time(@steps)
+            if naive >= @secondary_transition_at
+              @secondary_transition_at
+            else
+              naive
+            end
+          elsif @current == :secondary
+            naive = naive_next_time(@steps - @secondary_transition_steps + 1)
+            if naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          else
+            raise "BUG: it's out of design"
+          end
+        end
+
+        def naive_next_time(retry_times)
+          raise NotImplementedError
+        end
+
+        def secondary?
+          !@forever && @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
+        end
+
+        def step
+          @steps += 1
+          if !@forever && @secondary && @current != :secondary && current_time >= @secondary_transition_at
+            @current = :secondary
+            @secondary_transition_steps = @steps
+          end
+          @next_time = calc_next_time
+          nil
+        end
+
+        def limit?
+          if @forever
+            false
+          else
+            @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
+          end
+        end
+      end
+
+      class ExponentialBackOffRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
+          @constant_factor = wait
+          @backoff_base = backoff_base
+          @max_interval = max_interval
+
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
+          @next_time = @start + @constant_factor
+        end
+
+        def naive_next_time(retry_next_times)
+          intr = calc_interval(retry_next_times)
+          current_time + randomize(intr)
+        end
+
+        def calc_max_retry_timeout(max_steps)
+          result = 0
+          max_steps.times { |i|
+            result += calc_interval(i)
+          }
+          result
+        end
+
+        def calc_interval(num)
+          interval = raw_interval(num - 1)
+          if @max_interval && interval > @max_interval
+            @max_interval
+          else
+            if interval.finite?
+              interval
+            else
+              # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
+              until interval.finite?
+                num -= 1
+                interval = raw_interval(num - 1)
+              end
+              interval
+            end
+          end
+        end
+
+        def raw_interval(num)
+          @constant_factor.to_f * (@backoff_base ** (num))
+        end
+      end
+
+      class PeriodicRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+          @retry_wait = wait
+
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
+          @next_time = @start + @retry_wait
+        end
+
+        def naive_next_time(retry_next_times)
+          current_time + randomize(@retry_wait)
+        end
+
+        def calc_max_retry_timeout(max_steps)
+          @retry_wait * max_steps
+        end
+      end
+    end
+  end
+end