fluentd-hubspot 0.14.14.1

data/lib/fluent/system_config.rb

@@ -0,0 +1,159 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+require 'fluent/config/element'
+
+module Fluent
+  class SystemConfig
+    include Configurable
+
+    SYSTEM_CONFIG_PARAMETERS = [
+      :workers, :root_dir, :log_level,
+      :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
+      :log_event_verbose,
+      :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
+      :file_permission, :dir_permission,
+    ]
+
+    config_param :workers,   :integer, default: 1
+    config_param :root_dir,  :string, default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: nil
+    config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :emit_error_log_interval,      :time, default: nil
+    config_param :suppress_config_dump, :bool, default: nil
+    config_param :log_event_verbose,    :bool, default: nil
+    config_param :without_source,  :bool, default: nil
+    config_param :rpc_endpoint,    :string, default: nil
+    config_param :enable_get_dump, :bool, default: nil
+    config_param :process_name,    :string, default: nil
+    config_param :file_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_param :dir_permission, default: nil do |v|
+      v.to_i(8)
+    end
+
+    def self.create(conf)
+      systems = conf.elements(name: 'system')
+      return SystemConfig.new if systems.empty?
+      raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
+
+      SystemConfig.new(systems.first)
+    end
+
+    def self.blank_system_config
+      Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
+    end
+
+    def self.overwrite_system_config(hash)
+      older = defined?($_system_config) ? $_system_config : nil
+      begin
+        $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, []))
+        yield
+      ensure
+        $_system_config = older
+      end
+    end
+
+    def initialize(conf=nil)
+      super()
+      conf ||= SystemConfig.blank_system_config
+      configure(conf)
+    end
+
+    def configure(conf)
+      super
+
+      @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+    end
+
+    def dup
+      s = SystemConfig.new
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        s.__send__("#{param}=", instance_variable_get("@#{param}"))
+      end
+      s
+    end
+
+    def attach(supervisor)
+      system = self
+      supervisor.instance_eval {
+        SYSTEM_CONFIG_PARAMETERS.each do |param|
+          case param
+          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission
+            next # doesn't exist in command line options
+          when :emit_error_log_interval
+            system.emit_error_log_interval = @suppress_interval if @suppress_interval
+          when :log_level
+            ll_value = instance_variable_get("@log_level")
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            if ll_value != Fluent::Log::LEVEL_INFO
+              system.log_level = ll_value
+            end
+          else
+            next unless instance_variable_defined?("@#{param}")
+            supervisor_value = instance_variable_get("@#{param}")
+            next if supervisor_value.nil? # it's not configured by command line options
+
+            system.send("#{param}=", supervisor_value)
+          end
+        end
+      }
+    end
+
+    def apply(supervisor)
+      system = self
+      supervisor.instance_eval {
+        SYSTEM_CONFIG_PARAMETERS.each do |param|
+          param_value = system.__send__(param)
+          next if param_value.nil?
+
+          case param
+          when :log_level
+            @log.level = @log_level = param_value
+          when :emit_error_log_interval
+            @suppress_interval = param_value
+          else
+            instance_variable_set("@#{param}", param_value)
+          end
+        end
+      }
+    end
+
+    module Mixin
+      def system_config
+        require 'fluent/engine'
+        unless defined?($_system_config)
+          $_system_config = nil
+        end
+        (instance_variable_defined?("@_system_config") && @_system_config) ||
+          $_system_config || Fluent::Engine.system_config
+      end
+
+      def system_config_override(opts={})
+        require 'fluent/engine'
+        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+          @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
+        end
+        opts.each_pair do |key, value|
+          @_system_config.send(:"#{key.to_s}=", value)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'test/unit'
+require 'fluent/env' # for Fluent.windows?
+require 'fluent/test/log'
+require 'fluent/test/base'
+require 'fluent/test/input_test'
+require 'fluent/test/output_test'
+require 'fluent/test/filter_test'
+require 'fluent/test/parser_test'
+require 'fluent/test/formatter_test'
+require 'serverengine'
+
+
+module Fluent
+  module Test
+    def self.dummy_logger
+      dl_opts = {log_level: ServerEngine::DaemonLogger::INFO}
+      logdev = Fluent::Test::DummyLogDevice.new
+      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+      Fluent::Log.new(logger)
+    end
+
+    def self.setup
+      ENV['SERVERENGINE_WORKER_ID'] = '0'
+
+      $log = dummy_logger
+
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
+
+      engine.define_singleton_method(:now=) {|n|
+        @now = n
+      }
+      engine.define_singleton_method(:now) {
+        @now ||= super()
+      }
+
+      nil
+    end
+  end
+end
+
+$log ||= Fluent::Test.dummy_logger

data/lib/fluent/test/base.rb

@@ -0,0 +1,78 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/system_config'
+require 'fluent/test/log'
+require 'serverengine'
+
+module Fluent
+  module Test
+    class TestDriver
+      include ::Test::Unit::Assertions
+
+      def initialize(klass, &block)
+        if klass.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass.name
+            klass = Class.new(klass)
+            klass.define_singleton_method("name") { klass_name }
+            klass.module_eval(&block)
+          end
+          @instance = klass.new
+        else
+          @instance = klass
+        end
+        @instance.router = Engine.root_agent.event_router
+        @instance.log = TestLogger.new
+        Engine.root_agent.instance_variable_set(:@log, @instance.log)
+
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(str, use_v1 = false)
+        if str.is_a?(Fluent::Config::Element)
+          @config = str
+        else
+          @config = Config.parse(str, "(test)", "(test_dir)", use_v1)
+        end
+        if label_name = @config['@label']
+          Engine.root_agent.add_label(label_name)
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      # num_waits is for checking thread status. This will be removed after improved plugin API
+      def run(num_waits = 10, &block)
+        @instance.start
+        @instance.after_start
+        begin
+          # wait until thread starts
+          num_waits.times { sleep 0.05 }
+          return yield
+        ensure
+          @instance.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/base.rb

@@ -0,0 +1,224 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/clock'
+
+require 'serverengine/socket_manager'
+require 'fileutils'
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class TestTimedOut < RuntimeError; end
+
+      class Base
+        attr_reader :instance, :logs
+
+        DEFAULT_TIMEOUT = 300
+
+        def initialize(klass, opts: {}, &block)
+          if klass.is_a?(Class)
+            @instance = klass.new
+            if block
+              @instance.singleton_class.module_eval(&block)
+              @instance.send(:initialize)
+            end
+          else
+            @instance = klass
+          end
+          @instance.under_plugin_development = true
+
+          @socket_manager_server = nil
+
+          @logs = []
+
+          @run_post_conditions = []
+          @run_breaking_conditions = []
+          @broken = false
+        end
+
+        def configure(conf, syntax: :v1)
+          raise NotImplementedError
+        end
+
+        def end_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_post_conditions << block
+        end
+
+        def break_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_breaking_conditions << block
+        end
+
+        def broken?
+          @broken
+        end
+
+        def run(timeout: nil, start: true, shutdown: true, &block)
+          instance_start if start
+
+          timeout ||= DEFAULT_TIMEOUT
+          stop_at = Fluent::Clock.now + timeout
+          @run_breaking_conditions << ->(){ Fluent::Clock.now >= stop_at }
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
+          end
+
+          sleep_with_watching_threads = ->(){
+            if @instance.respond_to?(:_threads)
+              @instance._threads.values.each{|t| t.join(0) }
+            end
+            sleep 0.1
+          }
+
+          begin
+            retval = run_actual(timeout: timeout, &block)
+            sleep_with_watching_threads.call until stop?
+            retval
+          ensure
+            instance_shutdown if shutdown
+          end
+        end
+
+        def instance_start
+          if @instance.respond_to?(:server_wait_until_start)
+            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+          end
+
+          unless @instance.started?
+            @instance.start
+          end
+          unless @instance.after_started?
+            @instance.after_start
+          end
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          instance_hook_after_started
+        end
+
+        def instance_hook_after_started
+          # insert hooks for tests available after instance.start
+        end
+
+        def instance_hook_before_stopped
+          # same with above
+        end
+
+        def instance_shutdown
+          instance_hook_before_stopped
+
+          show_errors_if_exists = ->(label, block){
+            begin
+              block.call
+            rescue => e
+              puts "unexpected error while #{label}, #{e.class}:#{e.message}"
+              e.backtrace.each do |bt|
+                puts "\t#{bt}"
+              end
+            end
+          }
+
+          show_errors_if_exists.call(:stop,            ->(){ @instance.stop unless @instance.stopped? })
+          show_errors_if_exists.call(:before_shutdown, ->(){ @instance.before_shutdown unless @instance.before_shutdown? })
+          show_errors_if_exists.call(:shutdown,        ->(){ @instance.shutdown unless @instance.shutdown? })
+          show_errors_if_exists.call(:after_shutdown,  ->(){ @instance.after_shutdown unless @instance.after_shutdown? })
+
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_stop)
+            @instance.event_loop_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:close, ->(){ @instance.close unless @instance.closed? })
+
+          if @instance.respond_to?(:thread_wait_until_stop)
+            @instance.thread_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:terminate, ->(){ @instance.terminate unless @instance.terminated? })
+
+          if @socket_manager_server
+            @socket_manager_server.close
+            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+          end
+        end
+
+        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+          if @instance.respond_to?(:_threads)
+            sleep 0.1 until @instance._threads.values.all?(&:alive?)
+          end
+
+          if @instance.respond_to?(:event_loop_running?)
+            sleep 0.1 until @instance.event_loop_running?
+          end
+
+          if @instance.respond_to?(:_child_process_processes)
+            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }
+          end
+
+          return_value = nil
+          begin
+            Timeout.timeout(timeout * 2) do |sec|
+              return_value = block.call if block_given?
+            end
+          rescue Timeout::Error
+            raise TestTimedOut, "Test case timed out with hard limit."
+          end
+          return_value
+        end
+
+        def stop?
+          # Should stop running if post conditions are not registered.
+          return true unless @run_post_conditions || @run_post_conditions.empty?
+
+          # Should stop running if all of the post conditions are true.
+          return true if @run_post_conditions.all? {|proc| proc.call }
+
+          # Should stop running if some of the breaking conditions is true.
+          # In this case, some post conditions may be not true.
+          if @run_breaking_conditions.any? {|proc| proc.call }
+            @broken = true
+            return true
+          end
+
+          false
+        end
+      end
+    end
+  end
+end