dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/system_config.rb

@@ -0,0 +1,188 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+require 'fluent/config/element'
+
+module Fluent
+  class SystemConfig
+    include Configurable
+
+    SYSTEM_CONFIG_PARAMETERS = [
+      :workers, :restart_worker_interval, :root_dir, :log_level,
+      :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
+      :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
+      :file_permission, :dir_permission, :counter_server, :counter_client,
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
+      :metrics, :enable_input_metrics, :enable_size_metrics
+    ]
+
+    config_param :workers,   :integer, default: 1
+    config_param :restart_worker_interval, :time, default: 0
+    config_param :root_dir,  :string, default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
+    config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
+    config_param :emit_error_log_interval,      :time, default: nil
+    config_param :suppress_config_dump, :bool, default: nil
+    config_param :log_event_verbose,    :bool, default: nil
+    config_param :without_source,  :bool, default: nil
+    config_param :rpc_endpoint,    :string, default: nil
+    config_param :enable_get_dump, :bool, default: nil
+    config_param :process_name,    :string, default: nil
+    config_param :strict_config_value, :bool, default: nil
+    config_param :enable_msgpack_time_support, :bool, default: nil
+    config_param :disable_shared_socket, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_size_metrics, :bool, default: nil
+    config_param :file_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_param :dir_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_section :log, required: false, init: true, multi: false do
+      config_param :format, :enum, list: [:text, :json], default: :text
+      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+      config_param :rotate_age, default: nil do |v|
+        if Fluent::Log::LOG_ROTATE_AGE.include?(v)
+          v.to_sym
+        else
+          begin
+            Integer(v)
+          rescue ArgumentError => e
+            raise Fluent::ConfigError, e.message
+          else
+            v.to_i
+          end
+        end
+      end
+      config_param :rotate_size, :size, default: nil
+    end
+
+    config_section :counter_server, multi: false do
+      desc 'scope name of counter server'
+      config_param :scope, :string
+
+      desc 'the port of counter server to listen to'
+      config_param :port, :integer, default: nil
+      desc 'the bind address of counter server to listen to'
+      config_param :bind, :string, default: nil
+
+      desc 'backup file path of counter values'
+      config_param :backup_path, :string
+    end
+
+    config_section :counter_client, multi: false do
+      desc 'the port of counter server'
+      config_param :port, :integer, default: nil
+      desc 'the IP address or hostname of counter server'
+      config_param :host, :string
+      desc 'the timeout of each operation'
+      config_param :timeout, :time, default: nil
+    end
+
+    config_section :metrics, multi: false do
+      config_param :@type, :string, default: "local"
+      config_param :labels, :hash, default: {}
+    end
+
+    def self.create(conf, strict_config_value=false)
+      systems = conf.elements(name: 'system')
+      return SystemConfig.new if systems.empty?
+      raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
+
+      SystemConfig.new(systems.first, strict_config_value)
+    end
+
+    def self.blank_system_config
+      Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
+    end
+
+    def self.overwrite_system_config(hash)
+      older = defined?($_system_config) ? $_system_config : nil
+      begin
+        $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, []))
+        yield
+      ensure
+        $_system_config = older
+      end
+    end
+
+    def initialize(conf=nil, strict_config_value=false)
+      super()
+      conf ||= SystemConfig.blank_system_config
+      configure(conf, strict_config_value)
+    end
+
+    def configure(conf, strict_config_value=false)
+      strict = strict_config_value
+      if !strict && conf && conf.has_key?("strict_config_value")
+        strict = Fluent::Config.bool_value(conf["strict_config_value"])
+      end
+
+      begin
+        super(conf, strict)
+      rescue ConfigError => e
+        $log.error "config error in:\n#{conf}"
+        $log.error 'config error', error: e
+        $log.debug_backtrace
+        exit!(1)
+      end
+
+      @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+    end
+
+    def dup
+      s = SystemConfig.new
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        s.__send__("#{param}=", instance_variable_get("@#{param}"))
+      end
+      s
+    end
+
+    def overwrite_variables(**opt)
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if opt.key?(param) && !opt[param].nil? && instance_variable_defined?("@#{param}")
+          instance_variable_set("@#{param}", opt[param])
+        end
+      end
+    end
+
+    module Mixin
+      def system_config
+        require 'fluent/engine'
+        unless defined?($_system_config)
+          $_system_config = nil
+        end
+        (instance_variable_defined?("@_system_config") && @_system_config) ||
+          $_system_config || Fluent::Engine.system_config
+      end
+
+      def system_config_override(opts={})
+        require 'fluent/engine'
+        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+          @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
+        end
+        opts.each_pair do |key, value|
+          @_system_config.__send__(:"#{key.to_s}=", value)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/base.rb

@@ -0,0 +1,78 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/system_config'
+require 'fluent/test/log'
+require 'serverengine'
+
+module Fluent
+  module Test
+    class TestDriver
+      include ::Test::Unit::Assertions
+
+      def initialize(klass, &block)
+        if klass.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass.name
+            klass = Class.new(klass)
+            klass.define_singleton_method("name") { klass_name }
+            klass.module_eval(&block)
+          end
+          @instance = klass.new
+        else
+          @instance = klass
+        end
+        @instance.router = Engine.root_agent.event_router
+        @instance.log = TestLogger.new
+        Engine.root_agent.instance_variable_set(:@log, @instance.log)
+
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(str, use_v1 = false)
+        if str.is_a?(Fluent::Config::Element)
+          @config = str
+        else
+          @config = Config.parse(str, "(test)", "(test_dir)", use_v1)
+        end
+        if label_name = @config['@label']
+          Engine.root_agent.add_label(label_name)
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      # num_waits is for checking thread status. This will be removed after improved plugin API
+      def run(num_waits = 10, &block)
+        @instance.start
+        @instance.after_start
+        begin
+          # wait until thread starts
+          num_waits.times { sleep 0.05 }
+          return yield
+        ensure
+          @instance.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/base.rb

@@ -0,0 +1,225 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/clock'
+
+require 'serverengine/socket_manager'
+require 'fileutils'
+require 'timeout'
+require 'logger'
+
+module Fluent
+  module Test
+    module Driver
+      class TestTimedOut < RuntimeError; end
+
+      class Base
+        attr_reader :instance, :logs
+
+        DEFAULT_TIMEOUT = 300
+
+        def initialize(klass, opts: {}, &block)
+          if klass.is_a?(Class)
+            @instance = klass.new
+            if block
+              @instance.singleton_class.module_eval(&block)
+              @instance.send(:initialize)
+            end
+          else
+            @instance = klass
+          end
+          @instance.under_plugin_development = true
+
+          @socket_manager_server = nil
+
+          @logs = []
+
+          @run_post_conditions = []
+          @run_breaking_conditions = []
+          @broken = false
+        end
+
+        def configure(conf, syntax: :v1)
+          raise NotImplementedError
+        end
+
+        def end_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_post_conditions << block
+        end
+
+        def break_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_breaking_conditions << block
+        end
+
+        def broken?
+          @broken
+        end
+
+        def run(timeout: nil, start: true, shutdown: true, &block)
+          instance_start if start
+
+          timeout ||= DEFAULT_TIMEOUT
+          stop_at = Fluent::Clock.now + timeout
+          @run_breaking_conditions << ->(){ Fluent::Clock.now >= stop_at }
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
+          end
+
+          sleep_with_watching_threads = ->(){
+            if @instance.respond_to?(:_threads)
+              @instance._threads.values.each{|t| t.join(0) }
+            end
+            sleep 0.1
+          }
+
+          begin
+            retval = run_actual(timeout: timeout, &block)
+            sleep_with_watching_threads.call until stop?
+            retval
+          ensure
+            instance_shutdown if shutdown
+          end
+        end
+
+        def instance_start
+          if @instance.respond_to?(:server_wait_until_start)
+            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+          end
+
+          unless @instance.started?
+            @instance.start
+          end
+          unless @instance.after_started?
+            @instance.after_start
+          end
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          instance_hook_after_started
+        end
+
+        def instance_hook_after_started
+          # insert hooks for tests available after instance.start
+        end
+
+        def instance_hook_before_stopped
+          # same with above
+        end
+
+        def instance_shutdown(log: Logger.new($stdout))
+          instance_hook_before_stopped
+
+          show_errors_if_exists = ->(label, block){
+            begin
+              block.call
+            rescue => e
+              log.error "unexpected error while #{label}, #{e.class}:#{e.message}"
+              e.backtrace.each do |bt|
+                log.error "\t#{bt}"
+              end
+            end
+          }
+
+          show_errors_if_exists.call(:stop,            ->(){ @instance.stop unless @instance.stopped? })
+          show_errors_if_exists.call(:before_shutdown, ->(){ @instance.before_shutdown unless @instance.before_shutdown? })
+          show_errors_if_exists.call(:shutdown,        ->(){ @instance.shutdown unless @instance.shutdown? })
+          show_errors_if_exists.call(:after_shutdown,  ->(){ @instance.after_shutdown unless @instance.after_shutdown? })
+
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_stop)
+            @instance.event_loop_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:close, ->(){ @instance.close unless @instance.closed? })
+
+          if @instance.respond_to?(:thread_wait_until_stop)
+            @instance.thread_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:terminate, ->(){ @instance.terminate unless @instance.terminated? })
+
+          if @socket_manager_server
+            @socket_manager_server.close
+            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+          end
+        end
+
+        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+          if @instance.respond_to?(:_threads)
+            sleep 0.1 until @instance._threads.values.all?(&:alive?)
+          end
+
+          if @instance.respond_to?(:event_loop_running?)
+            sleep 0.1 until @instance.event_loop_running?
+          end
+
+          if @instance.respond_to?(:_child_process_processes)
+            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }
+          end
+
+          return_value = nil
+          begin
+            Timeout.timeout(timeout * 2) do |sec|
+              return_value = block.call if block_given?
+            end
+          rescue Timeout::Error
+            raise TestTimedOut, "Test case timed out with hard limit."
+          end
+          return_value
+        end
+
+        def stop?
+          # Should stop running if post conditions are not registered.
+          return true unless @run_post_conditions || @run_post_conditions.empty?
+
+          # Should stop running if all of the post conditions are true.
+          return true if @run_post_conditions.all? {|proc| proc.call }
+
+          # Should stop running if some of the breaking conditions is true.
+          # In this case, some post conditions may be not true.
+          if @run_breaking_conditions.any? {|proc| proc.call }
+            @broken = true
+            return true
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/base_owned.rb

@@ -0,0 +1,83 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/log'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Test
+    module Driver
+      class OwnerDummy < Fluent::Plugin::Base
+        include PluginId
+        include PluginLoggerMixin
+        include PluginHelper::Mixin
+      end
+
+      class BaseOwned < Base
+        attr_accessor :section_name
+
+        def initialize(klass, opts: {}, &block)
+          super
+
+          owner = OwnerDummy.new
+          if opts
+            owner.system_config_override(opts)
+          end
+          owner.log = TestLogger.new
+
+          if @instance.respond_to?(:owner=)
+            @instance.owner = owner
+            if opts
+              @instance.system_config_override(opts)
+            end
+          end
+
+          @logs = owner.log.out.logs
+          @section_name = ''
+        end
+
+        def configure(conf)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          elsif conf.is_a?(Hash)
+            @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
+          else
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: :v1)
+          end
+          @instance.configure(@config)
+          self
+        end
+
+        # this is special method for v0 and should be deleted
+        def configure_v0(conf)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          elsif conf.is_a?(Hash)
+            @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
+          else
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: :v0)
+          end
+          @instance.configure(@config)
+          self
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/base_owner.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/test_event_router'
+
+module Fluent
+  module Test
+    module Driver
+      class BaseOwner < Base
+        def initialize(klass, opts: {}, &block)
+          super
+
+          if opts
+            @instance.system_config_override(opts)
+          end
+          @instance.log = TestLogger.new
+          @logs = @instance.log.out.logs
+
+          @event_streams = nil
+          @error_events = nil
+        end
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          else
+            @config = Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+          end
+
+          if @instance.respond_to?(:router=)
+            @event_streams = []
+            @error_events = []
+
+            driver = self
+            mojule = Module.new do
+              define_method(:event_emitter_router) do |label_name|
+                TestEventRouter.new(driver)
+              end
+            end
+            @instance.singleton_class.prepend mojule
+          end
+
+          @instance.configure(@config)
+          self
+        end
+
+        Emit = Struct.new(:tag, :es)
+        ErrorEvent = Struct.new(:tag, :time, :record, :error)
+
+        # via TestEventRouter
+        def emit_event_stream(tag, es)
+          @event_streams << Emit.new(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @error_events << ErrorEvent.new(tag, time, record, error)
+        end
+
+        def events(tag: nil)
+          if block_given?
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                yield t, time, record
+              end
+            end
+          else
+            list = []
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                list << [t, time, record]
+              end
+            end
+            list
+          end
+        end
+
+        def event_streams(tag: nil)
+          return [] if @event_streams.nil?
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.es
+            end
+          else
+            selected.map{|e| [e.tag, e.es] }
+          end
+        end
+
+        def emit_count
+          @event_streams.size
+        end
+
+        def record_count
+          @event_streams.reduce(0) {|a, e| a + e.es.size }
+        end
+
+        def error_events(tag: nil)
+          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.time, e.record, e.error
+            end
+          else
+            selected.map{|e| [e.tag, e.time, e.record, e.error] }
+          end
+        end
+
+        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)
+          if expect_emits
+            @run_post_conditions << ->(){ emit_count >= expect_emits }
+          end
+          if expect_records
+            @run_post_conditions << ->(){ record_count >= expect_records }
+          end
+
+          super(timeout: timeout, start: start, shutdown: shutdown, &block)
+        end
+      end
+    end
+  end
+end