fluentd-hubspot 0.14.14.1

data/lib/fluent/plugin_helper/inject.rb

@@ -0,0 +1,151 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+require 'socket'
+
+module Fluent
+  module PluginHelper
+    module Inject
+      def inject_values_to_record(tag, time, record)
+        return record unless @_inject_enabled
+
+        r = record.dup
+        if @_inject_hostname_key
+          r[@_inject_hostname_key] = @_inject_hostname
+        end
+        if @_inject_worker_id_key
+          r[@_inject_worker_id_key] = @_inject_worker_id
+        end
+        if @_inject_tag_key
+          r[@_inject_tag_key] = tag
+        end
+        if @_inject_time_key
+          r[@_inject_time_key] = @_inject_time_formatter.call(time)
+        end
+
+        r
+      end
+
+      def inject_values_to_event_stream(tag, es)
+        return es unless @_inject_enabled
+
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          r = record.dup
+          if @_inject_hostname_key
+            r[@_inject_hostname_key] = @_inject_hostname
+          end
+          if @_inject_worker_id_key
+            r[@_inject_worker_id_key] = @_inject_worker_id
+          end
+          if @_inject_tag_key
+            r[@_inject_tag_key] = tag
+          end
+          if @_inject_time_key
+            r[@_inject_time_key] = @_inject_time_formatter.call(time)
+          end
+          new_es.add(time, r)
+        end
+
+        new_es
+      end
+
+      module InjectParams
+        include Fluent::Configurable
+        config_section :inject, required: false, multi: false, param_name: :inject_config do
+          config_param :hostname_key, :string, default: nil
+          config_param :hostname, :string, default: nil
+          config_param :worker_id_key, :string, default: nil
+          config_param :tag_key, :string, default: nil
+          config_param :time_key, :string, default: nil
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param name, type, opts
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include InjectParams
+      end
+
+      def initialize
+        super
+        @_inject_enabled = false
+        @_inject_hostname_key = nil
+        @_inject_hostname = nil
+        @_inject_worker_id_key = nil
+        @_inject_worker_id = nil
+        @_inject_tag_key = nil
+        @_inject_time_key = nil
+        @_inject_time_formatter = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @inject_config
+          @_inject_hostname_key = @inject_config.hostname_key
+          if @_inject_hostname_key
+            if self.respond_to?(:buffer_config)
+              # Output plugin cannot use "hostname"(specified by @hostname_key),
+              # injected by this plugin helper, in chunk keys.
+              # This plugin helper works in `#format` (in many cases), but modified record
+              # don't have any side effect in chunking of output plugin.
+              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
+                log.error "Use filters to inject hostname to use it in buffer chunking."
+                raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
+              end
+            end
+
+            @_inject_hostname =  @inject_config.hostname
+            unless @_inject_hostname
+              @_inject_hostname = ::Socket.gethostname
+              log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
+            end
+          end
+          @_inject_worker_id_key = @inject_config.worker_id_key
+          if @_inject_worker_id_key
+            @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests
+          end
+          @_inject_tag_key = @inject_config.tag_key
+          @_inject_time_key = @inject_config.time_key
+          if @_inject_time_key
+            @_inject_time_formatter = case @inject_config.time_type
+                                      when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime then ->(time){ time.to_i }
+                                      else
+                                        localtime = @inject_config.localtime && !@inject_config.utc
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
+                                      end
+          else
+            if @inject_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
+          end
+
+          @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/parser.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Parser
+      def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
+        parser = @_parsers[usage]
+        return parser if parser && !type && !conf
+
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
+        parser = Fluent::Plugin.new_parser(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('parse', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('parse', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        parser.configure(config)
+        if @_parsers_started
+          parser.start
+        end
+
+        @_parsers[usage] = parser
+        parser
+      end
+
+      module ParserParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate parser plugin instances
+        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string # config_set_default required for :@type
+        end
+      end
+
+      def self.included(mod)
+        mod.include ParserParams
+      end
+
+      attr_reader :_parsers # for tests
+
+      def initialize
+        super
+        @_parsers_started = false
+        @_parsers = {} # usage => parser
+      end
+
+      def configure(conf)
+        super
+
+        @parser_configs.each do |section|
+          if @_parsers[section.usage]
+            raise Fluent::ConfigError, "duplicated parsers configured: #{section.usage}"
+          end
+          parser = Plugin.new_parser(section[:@type], parent: self)
+          parser.configure(section.corresponding_config_element)
+          @_parsers[section.usage] = parser
+        end
+      end
+
+      def start
+        super
+        @_parsers_started = true
+        @_parsers.each_pair do |usage, parser|
+          parser.start
+        end
+      end
+
+      def parser_operate(method_name, &block)
+        @_parsers.each_pair do |usage, parser|
+          begin
+            parser.send(method_name)
+            block.call(parser) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        parser_operate(:stop)
+      end
+
+      def before_shutdown
+        parser_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        parser_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        parser_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        parser_operate(:close)
+        super
+      end
+
+      def terminate
+        parser_operate(:terminate)
+        @_parsers_started = false
+        @_parsers = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/retry_state.rb

@@ -0,0 +1,201 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module RetryState
+      def retry_state_create(
+          title, retry_type, wait, timeout,
+          forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125,
+          secondary: false, secondary_threshold: 0.8
+      )
+        case retry_type
+        when :exponential_backoff
+          ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
+        when :periodic
+          PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+        else
+          raise "BUG: unknown retry_type specified: '#{retry_type}'"
+        end
+      end
+
+      class RetryStateMachine
+        attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps
+
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+          @title = title
+
+          @start = current_time
+          @steps = 0
+          @next_time = nil # should be initialized for first retry by child class
+
+          @timeout = timeout
+          @timeout_at = @start + timeout
+          @current = :primary
+
+          if randomize_width < 0 || randomize_width > 0.5
+            raise "BUG: randomize_width MUST be between 0 and 0.5"
+          end
+
+          @randomize = randomize
+          @randomize_width = randomize_width
+
+          if forever && secondary
+            raise "BUG: forever and secondary are exclusive to each other"
+          end
+
+          @forever = forever
+          @max_steps = max_steps
+
+          @secondary = secondary
+          @secondary_threshold = secondary_threshold
+          if @secondary
+            raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
+            max_retry_timeout = timeout
+            if max_steps
+              timeout_by_max_steps = calc_max_retry_timeout(max_steps)
+              max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout
+            end
+            @secondary_transition_at = @start + max_retry_timeout * @secondary_threshold
+            @secondary_transition_steps = nil
+          end
+        end
+
+        def current_time
+          Time.now
+        end
+
+        def randomize(interval)
+          return interval unless @randomize
+
+          interval + (interval * @randomize_width * (2 * rand - 1.0))
+        end
+
+        def calc_next_time
+          if @forever || !@secondary # primary
+            naive = naive_next_time(@steps)
+            if @forever
+              naive
+            elsif naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          elsif @current == :primary && @secondary
+            naive = naive_next_time(@steps)
+            if naive >= @secondary_transition_at
+              @secondary_transition_at
+            else
+              naive
+            end
+          elsif @current == :secondary
+            naive = naive_next_time(@steps - @secondary_transition_steps + 1)
+            if naive >= @timeout_at
+              @timeout_at
+            else
+              naive
+            end
+          else
+            raise "BUG: it's out of design"
+          end
+        end
+
+        def naive_next_time(retry_times)
+          raise NotImplementedError
+        end
+
+        def secondary?
+          @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
+        end
+
+        def step
+          @steps += 1
+          if @secondary && @current != :secondary && current_time >= @secondary_transition_at
+            @current = :secondary
+            @secondary_transition_steps = @steps
+          end
+          @next_time = calc_next_time
+          nil
+        end
+
+        def limit?
+          if @forever
+            false
+          else
+            @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
+          end
+        end
+      end
+
+      class ExponentialBackOffRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
+          @constant_factor = wait
+          @backoff_base = backoff_base
+          @max_interval = max_interval
+
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
+          @next_time = @start + @constant_factor
+        end
+
+        def naive_next_time(retry_next_times)
+          intr = calc_interval(retry_next_times)
+          current_time + randomize(intr)
+        end
+
+        def calc_max_retry_timeout(max_steps)
+          result = 0
+          max_steps.times { |i|
+            result += calc_interval(i)
+          }
+          result
+        end
+
+        def calc_interval(num)
+          # make it infinite if calculated "interval" is too big
+          interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
+          if interval.finite?
+            if @max_interval && interval > @max_interval
+              @max_interval
+            else
+              interval
+            end
+          else
+            interval
+          end
+        end
+      end
+
+      class PeriodicRetry < RetryStateMachine
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+          @retry_wait = wait
+
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
+          @next_time = @start + @retry_wait
+        end
+
+        def naive_next_time(retry_next_times)
+          current_time + randomize(@retry_wait)
+        end
+
+        def calc_max_retry_timeout(max_steps)
+          @retry_wait * max_steps
+        end
+      end
+    end
+  end
+end